diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/MlPlugin.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/MlPlugin.java new file mode 100644 index 00000000000..da054d4206b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/MlPlugin.java @@ -0,0 +1,361 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.ml.action.CloseJobAction; +import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction; +import org.elasticsearch.xpack.ml.action.DeleteFilterAction; +import org.elasticsearch.xpack.ml.action.DeleteJobAction; +import org.elasticsearch.xpack.ml.action.DeleteModelSnapshotAction; +import org.elasticsearch.xpack.ml.action.FlushJobAction; +import org.elasticsearch.xpack.ml.action.GetBucketsAction; +import org.elasticsearch.xpack.ml.action.GetCategoriesAction; +import org.elasticsearch.xpack.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.ml.action.GetDatafeedsStatsAction; +import org.elasticsearch.xpack.ml.action.GetFiltersAction; +import org.elasticsearch.xpack.ml.action.GetInfluencersAction; +import org.elasticsearch.xpack.ml.action.GetJobsAction; +import org.elasticsearch.xpack.ml.action.GetJobsStatsAction; +import org.elasticsearch.xpack.ml.action.GetModelSnapshotsAction; +import org.elasticsearch.xpack.ml.action.GetRecordsAction; +import org.elasticsearch.xpack.ml.action.InternalOpenJobAction; +import org.elasticsearch.xpack.ml.action.MlDeleteByQueryAction; +import org.elasticsearch.xpack.ml.action.OpenJobAction; +import org.elasticsearch.xpack.ml.action.PostDataAction; +import org.elasticsearch.xpack.ml.action.PutDatafeedAction; +import org.elasticsearch.xpack.ml.action.PutFilterAction; +import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.action.StopDatafeedAction; +import org.elasticsearch.xpack.ml.action.UpdateJobStateAction; +import org.elasticsearch.xpack.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction; +import org.elasticsearch.xpack.ml.action.UpdateProcessAction; +import org.elasticsearch.xpack.ml.action.ValidateDetectorAction; +import org.elasticsearch.xpack.ml.action.ValidateJobConfigAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunner; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.metadata.MlInitializationService; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; +import org.elasticsearch.xpack.ml.job.process.NativeController; +import org.elasticsearch.xpack.ml.job.process.ProcessCtrl; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessFactory; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.job.process.autodetect.BlackHoleAutodetectProcess; +import org.elasticsearch.xpack.ml.job.process.autodetect.NativeAutodetectProcessFactory; +import org.elasticsearch.xpack.ml.job.process.normalizer.MultiplyingNormalizerProcess; +import org.elasticsearch.xpack.ml.job.process.normalizer.NativeNormalizerProcessFactory; +import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; +import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerProcessFactory; +import org.elasticsearch.xpack.ml.rest.datafeeds.RestDeleteDatafeedAction; +import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedStatsAction; +import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedsAction; +import org.elasticsearch.xpack.ml.rest.datafeeds.RestPutDatafeedAction; +import org.elasticsearch.xpack.ml.rest.datafeeds.RestStartDatafeedAction; +import org.elasticsearch.xpack.ml.rest.datafeeds.RestStopDatafeedAction; +import org.elasticsearch.xpack.ml.rest.filter.RestDeleteFilterAction; +import org.elasticsearch.xpack.ml.rest.filter.RestGetFiltersAction; +import org.elasticsearch.xpack.ml.rest.filter.RestPutFilterAction; +import org.elasticsearch.xpack.ml.rest.job.RestCloseJobAction; +import org.elasticsearch.xpack.ml.rest.job.RestDeleteJobAction; +import org.elasticsearch.xpack.ml.rest.job.RestFlushJobAction; +import org.elasticsearch.xpack.ml.rest.job.RestGetJobStatsAction; +import org.elasticsearch.xpack.ml.rest.job.RestGetJobsAction; +import org.elasticsearch.xpack.ml.rest.job.RestOpenJobAction; +import org.elasticsearch.xpack.ml.rest.job.RestPostDataAction; +import org.elasticsearch.xpack.ml.rest.job.RestPostJobUpdateAction; +import org.elasticsearch.xpack.ml.rest.job.RestPutJobAction; +import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestDeleteModelSnapshotAction; +import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestGetModelSnapshotsAction; +import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestRevertModelSnapshotAction; +import org.elasticsearch.xpack.ml.rest.modelsnapshots.RestUpdateModelSnapshotAction; +import org.elasticsearch.xpack.ml.rest.results.RestGetBucketsAction; +import org.elasticsearch.xpack.ml.rest.results.RestGetCategoriesAction; +import org.elasticsearch.xpack.ml.rest.results.RestGetInfluencersAction; +import org.elasticsearch.xpack.ml.rest.results.RestGetRecordsAction; +import org.elasticsearch.xpack.ml.rest.validate.RestValidateDetectorAction; +import org.elasticsearch.xpack.ml.rest.validate.RestValidateJobConfigAction; +import org.elasticsearch.xpack.ml.utils.NamedPipeHelper; +import org.elasticsearch.xpack.persistent.CompletionPersistentTaskAction; +import org.elasticsearch.xpack.persistent.PersistentActionCoordinator; +import org.elasticsearch.xpack.persistent.PersistentActionRegistry; +import org.elasticsearch.xpack.persistent.PersistentActionRequest; +import org.elasticsearch.xpack.persistent.PersistentActionService; +import org.elasticsearch.xpack.persistent.PersistentTaskClusterService; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; +import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction; +import org.elasticsearch.xpack.persistent.StartPersistentTaskAction; +import org.elasticsearch.xpack.persistent.UpdatePersistentTaskStatusAction; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import static java.util.Collections.emptyList; + +public class MlPlugin extends Plugin implements ActionPlugin { + public static final String NAME = "ml"; + public static final String BASE_PATH = "/_xpack/ml/"; + public static final String THREAD_POOL_NAME = NAME; + public static final String DATAFEED_RUNNER_THREAD_POOL_NAME = NAME + "_datafeed_runner"; + public static final String AUTODETECT_PROCESS_THREAD_POOL_NAME = NAME + "_autodetect_process"; + + // NORELEASE - temporary solution + public static final Setting USE_NATIVE_PROCESS_OPTION = Setting.boolSetting("useNativeProcess", true, Property.NodeScope, + Property.Deprecated); + + /** Setting for enabling or disabling machine learning. Defaults to true. */ + public static final Setting ML_ENABLED = Setting.boolSetting("xpack.ml.enabled", false, Setting.Property.NodeScope); + + private final Settings settings; + private final Environment env; + private boolean enabled; + + public MlPlugin(Settings settings) { + this(settings, new Environment(settings)); + } + + public MlPlugin(Settings settings, Environment env) { + this.enabled = ML_ENABLED.get(settings); + this.settings = settings; + this.env = env; + } + + @Override + public List> getSettings() { + return Collections.unmodifiableList( + Arrays.asList(USE_NATIVE_PROCESS_OPTION, + ML_ENABLED, + ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING, + ProcessCtrl.MAX_ANOMALY_RECORDS_SETTING, + DataCountsReporter.ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING, + DataCountsReporter.ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING, + AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE)); + } + + @Override + public List getNamedWriteables() { + return Arrays.asList( + new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, "ml", MlMetadata.MlMetadataDiff::new), + new NamedWriteableRegistry.Entry(PersistentActionCoordinator.Status.class, + PersistentActionCoordinator.Status.NAME, PersistentActionCoordinator.Status::new), + new NamedWriteableRegistry.Entry(ClusterState.Custom.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::new), + new NamedWriteableRegistry.Entry(NamedDiff.class, PersistentTasksInProgress.TYPE, PersistentTasksInProgress::readDiffFrom), + new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, StartDatafeedAction.Request::new) + ); + } + + @Override + public List getNamedXContent() { + NamedXContentRegistry.Entry entry = new NamedXContentRegistry.Entry( + MetaData.Custom.class, + new ParseField("ml"), + parser -> MlMetadata.ML_METADATA_PARSER.parse(parser, null).build() + ); + return Collections.singletonList(entry); + } + + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry) { + if (false == enabled) { + return emptyList(); + } + JobResultsPersister jobResultsPersister = new JobResultsPersister(settings, client); + JobProvider jobProvider = new JobProvider(client, 0); + JobDataCountsPersister jobDataCountsPersister = new JobDataCountsPersister(settings, client); + + JobManager jobManager = new JobManager(settings, jobProvider, jobResultsPersister, clusterService); + AutodetectProcessFactory autodetectProcessFactory; + NormalizerProcessFactory normalizerProcessFactory; + if (USE_NATIVE_PROCESS_OPTION.get(settings)) { + try { + NativeController nativeController = new NativeController(env, new NamedPipeHelper()); + nativeController.tailLogsInThread(); + autodetectProcessFactory = new NativeAutodetectProcessFactory(jobProvider, env, settings, nativeController, client); + normalizerProcessFactory = new NativeNormalizerProcessFactory(env, settings, nativeController); + } catch (IOException e) { + throw new ElasticsearchException("Failed to create native process factories", e); + } + } else { + autodetectProcessFactory = (jobDetails, modelSnapshot, quantiles, filters, ignoreDowntime, executorService) -> + new BlackHoleAutodetectProcess(); + // factor of 1.0 makes renormalization a no-op + normalizerProcessFactory = (jobId, quantilesState, bucketSpan, perPartitionNormalization, + executorService) -> new MultiplyingNormalizerProcess(settings, 1.0); + } + NormalizerFactory normalizerFactory = new NormalizerFactory(normalizerProcessFactory, + threadPool.executor(MlPlugin.THREAD_POOL_NAME)); + AutodetectProcessManager dataProcessor = new AutodetectProcessManager(settings, client, threadPool, jobManager, jobProvider, + jobResultsPersister, jobDataCountsPersister, autodetectProcessFactory, normalizerFactory); + DatafeedJobRunner datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, + System::currentTimeMillis); + PersistentActionService persistentActionService = new PersistentActionService(Settings.EMPTY, clusterService, client); + PersistentActionRegistry persistentActionRegistry = new PersistentActionRegistry(Settings.EMPTY); + + return Arrays.asList( + jobProvider, + jobManager, + dataProcessor, + new MlInitializationService(settings, threadPool, clusterService, jobProvider), + jobDataCountsPersister, + datafeedJobRunner, + persistentActionService, + persistentActionRegistry, + new PersistentTaskClusterService(Settings.EMPTY, persistentActionRegistry, clusterService) + ); + } + + @Override + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + if (false == enabled) { + return emptyList(); + } + return Arrays.asList( + new RestGetJobsAction(settings, restController), + new RestGetJobStatsAction(settings, restController), + new RestPutJobAction(settings, restController), + new RestPostJobUpdateAction(settings, restController), + new RestDeleteJobAction(settings, restController), + new RestOpenJobAction(settings, restController), + new RestGetFiltersAction(settings, restController), + new RestPutFilterAction(settings, restController), + new RestDeleteFilterAction(settings, restController), + new RestGetInfluencersAction(settings, restController), + new RestGetRecordsAction(settings, restController), + new RestGetBucketsAction(settings, restController), + new RestPostDataAction(settings, restController), + new RestCloseJobAction(settings, restController), + new RestFlushJobAction(settings, restController), + new RestValidateDetectorAction(settings, restController), + new RestValidateJobConfigAction(settings, restController), + new RestGetCategoriesAction(settings, restController), + new RestGetModelSnapshotsAction(settings, restController), + new RestRevertModelSnapshotAction(settings, restController), + new RestUpdateModelSnapshotAction(settings, restController), + new RestGetDatafeedsAction(settings, restController), + new RestGetDatafeedStatsAction(settings, restController), + new RestPutDatafeedAction(settings, restController), + new RestDeleteDatafeedAction(settings, restController), + new RestStartDatafeedAction(settings, restController), + new RestStopDatafeedAction(settings, restController), + new RestDeleteModelSnapshotAction(settings, restController) + ); + } + + @Override + public List> getActions() { + if (false == enabled) { + return emptyList(); + } + return Arrays.asList( + new ActionHandler<>(GetJobsAction.INSTANCE, GetJobsAction.TransportAction.class), + new ActionHandler<>(GetJobsStatsAction.INSTANCE, GetJobsStatsAction.TransportAction.class), + new ActionHandler<>(PutJobAction.INSTANCE, PutJobAction.TransportAction.class), + new ActionHandler<>(UpdateJobAction.INSTANCE, UpdateJobAction.TransportAction.class), + new ActionHandler<>(DeleteJobAction.INSTANCE, DeleteJobAction.TransportAction.class), + new ActionHandler<>(OpenJobAction.INSTANCE, OpenJobAction.TransportAction.class), + new ActionHandler<>(InternalOpenJobAction.INSTANCE, InternalOpenJobAction.TransportAction.class), + new ActionHandler<>(UpdateJobStateAction.INSTANCE, UpdateJobStateAction.TransportAction.class), + new ActionHandler<>(GetFiltersAction.INSTANCE, GetFiltersAction.TransportAction.class), + new ActionHandler<>(PutFilterAction.INSTANCE, PutFilterAction.TransportAction.class), + new ActionHandler<>(DeleteFilterAction.INSTANCE, DeleteFilterAction.TransportAction.class), + new ActionHandler<>(GetBucketsAction.INSTANCE, GetBucketsAction.TransportAction.class), + new ActionHandler<>(GetInfluencersAction.INSTANCE, GetInfluencersAction.TransportAction.class), + new ActionHandler<>(GetRecordsAction.INSTANCE, GetRecordsAction.TransportAction.class), + new ActionHandler<>(PostDataAction.INSTANCE, PostDataAction.TransportAction.class), + new ActionHandler<>(CloseJobAction.INSTANCE, CloseJobAction.TransportAction.class), + new ActionHandler<>(FlushJobAction.INSTANCE, FlushJobAction.TransportAction.class), + new ActionHandler<>(ValidateDetectorAction.INSTANCE, ValidateDetectorAction.TransportAction.class), + new ActionHandler<>(ValidateJobConfigAction.INSTANCE, ValidateJobConfigAction.TransportAction.class), + new ActionHandler<>(GetCategoriesAction.INSTANCE, GetCategoriesAction.TransportAction.class), + new ActionHandler<>(GetModelSnapshotsAction.INSTANCE, GetModelSnapshotsAction.TransportAction.class), + new ActionHandler<>(RevertModelSnapshotAction.INSTANCE, RevertModelSnapshotAction.TransportAction.class), + new ActionHandler<>(UpdateModelSnapshotAction.INSTANCE, UpdateModelSnapshotAction.TransportAction.class), + new ActionHandler<>(GetDatafeedsAction.INSTANCE, GetDatafeedsAction.TransportAction.class), + new ActionHandler<>(GetDatafeedsStatsAction.INSTANCE, GetDatafeedsStatsAction.TransportAction.class), + new ActionHandler<>(PutDatafeedAction.INSTANCE, PutDatafeedAction.TransportAction.class), + new ActionHandler<>(DeleteDatafeedAction.INSTANCE, DeleteDatafeedAction.TransportAction.class), + new ActionHandler<>(StartDatafeedAction.INSTANCE, StartDatafeedAction.TransportAction.class), + new ActionHandler<>(StopDatafeedAction.INSTANCE, StopDatafeedAction.TransportAction.class), + new ActionHandler<>(DeleteModelSnapshotAction.INSTANCE, DeleteModelSnapshotAction.TransportAction.class), + new ActionHandler<>(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class), + new ActionHandler<>(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class), + new ActionHandler<>(CompletionPersistentTaskAction.INSTANCE, CompletionPersistentTaskAction.TransportAction.class), + new ActionHandler<>(RemovePersistentTaskAction.INSTANCE, RemovePersistentTaskAction.TransportAction.class), + new ActionHandler<>(MlDeleteByQueryAction.INSTANCE, MlDeleteByQueryAction.TransportAction.class), + new ActionHandler<>(UpdateProcessAction.INSTANCE, UpdateProcessAction.TransportAction.class) + ); + } + + public static Path resolveConfigFile(Environment env, String name) { + return env.configFile().resolve(NAME).resolve(name); + } + + @Override + public List> getExecutorBuilders(Settings settings) { + if (false == enabled) { + return emptyList(); + } + int maxNumberOfJobs = AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.get(settings); + FixedExecutorBuilder ml = new FixedExecutorBuilder(settings, THREAD_POOL_NAME, + maxNumberOfJobs * 2, 1000, "xpack.ml.thread_pool"); + + // fail quick to run autodetect process / datafeed, so no queues + // 4 threads: for c++ logging, result processing, state processing and restore state + FixedExecutorBuilder autoDetect = new FixedExecutorBuilder(settings, AUTODETECT_PROCESS_THREAD_POOL_NAME, + maxNumberOfJobs * 4, 4, "xpack.ml.autodetect_process_thread_pool"); + + // TODO: if datafeed and non datafeed jobs are considered more equal and the datafeed and + // autodetect process are created at the same time then these two different TPs can merge. + FixedExecutorBuilder datafeed = new FixedExecutorBuilder(settings, DATAFEED_RUNNER_THREAD_POOL_NAME, + maxNumberOfJobs, 1, "xpack.ml.datafeed_thread_pool"); + return Arrays.asList(ml, autoDetect, datafeed); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java new file mode 100644 index 00000000000..4908808c876 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/CloseJobAction.java @@ -0,0 +1,249 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.TransportCancelTasksAction; +import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.list.TransportListTasksAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.JobStateObserver; + +import java.io.IOException; +import java.util.Objects; + +public class CloseJobAction extends Action { + + public static final CloseJobAction INSTANCE = new CloseJobAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/close"; + + private CloseJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + private String jobId; + private TimeValue closeTimeout = TimeValue.timeValueMinutes(20); + + Request() {} + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public TimeValue getCloseTimeout() { + return closeTimeout; + } + + public void setCloseTimeout(TimeValue closeTimeout) { + this.closeTimeout = closeTimeout; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + closeTimeout = new TimeValue(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + closeTimeout.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, closeTimeout); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(closeTimeout, other.closeTimeout); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, CloseJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private boolean closed; + + Response() { + } + + Response(boolean closed) { + this.closed = closed; + } + + public boolean isClosed() { + return closed; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + closed = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(closed); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("closed", closed); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return closed == response.closed; + } + + @Override + public int hashCode() { + return Objects.hash(closed); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final ClusterService clusterService; + private final JobStateObserver jobStateObserver; + private final TransportListTasksAction listTasksAction; + private final TransportCancelTasksAction cancelTasksAction; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, TransportCancelTasksAction cancelTasksAction, + TransportListTasksAction listTasksAction) { + super(settings, CloseJobAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.clusterService = clusterService; + this.jobStateObserver = new JobStateObserver(threadPool, clusterService); + this.cancelTasksAction = cancelTasksAction; + this.listTasksAction = listTasksAction; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + MlMetadata mlMetadata = clusterService.state().metaData().custom(MlMetadata.TYPE); + validate(request.jobId, mlMetadata); + + ListTasksRequest listTasksRequest = new ListTasksRequest(); + listTasksRequest.setActions(InternalOpenJobAction.NAME); + listTasksRequest.setDetailed(true); + listTasksAction.execute(listTasksRequest, ActionListener.wrap(listTasksResponse -> { + String expectedJobDescription = "job-" + request.jobId; + for (TaskInfo taskInfo : listTasksResponse.getTasks()) { + if (expectedJobDescription.equals(taskInfo.getDescription())) { + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); + cancelTasksRequest.setTaskId(taskInfo.getTaskId()); + cancelTasksAction.execute(cancelTasksRequest, ActionListener.wrap( + cancelTasksResponse -> { + jobStateObserver.waitForState(request.jobId, request.closeTimeout, JobState.CLOSED, + e -> { + if (e != null) { + listener.onFailure(e); + } else { + listener.onResponse(new CloseJobAction.Response(true)); + } + } + ); + }, + listener::onFailure) + ); + return; + } + } + listener.onFailure(new ResourceNotFoundException("No job [" + request.jobId + "] running")); + }, listener::onFailure)); + } + + static void validate(String jobId, MlMetadata mlMetadata) { + Allocation allocation = mlMetadata.getAllocations().get(jobId); + if (allocation == null) { + throw ExceptionsHelper.missingJobException(jobId); + } + + if (allocation.getState() != JobState.OPENED) { + throw new ElasticsearchStatusException("job not opened, expected job state [{}], but got [{}]", + RestStatus.CONFLICT, JobState.OPENED, allocation.getState()); + } + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedAction.java new file mode 100644 index 00000000000..92d589e6a46 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedAction.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteDatafeedAction extends Action { + + public static final DeleteDatafeedAction INSTANCE = new DeleteDatafeedAction(); + public static final String NAME = "cluster:admin/ml/datafeeds/delete"; + + private DeleteDatafeedAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + private String datafeedId; + + public Request(String datafeedId) { + this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); + } + + Request() { + } + + public String getDatafeedId() { + return datafeedId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeedId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(datafeedId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(datafeedId, request.datafeedId); + } + + @Override + public int hashCode() { + return Objects.hash(datafeedId); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, DeleteDatafeedAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + private Response() { + } + + private Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, DeleteDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + clusterService.submitStateUpdateTask("delete-datafeed-" + request.getDatafeedId(), + new AckedClusterStateUpdateTask(request, listener) { + + @Override + protected Response newResponse(boolean acknowledged) { + return new Response(acknowledged); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + MlMetadata currentMetadata = state.getMetaData().custom(MlMetadata.TYPE); + PersistentTasksInProgress persistentTasksInProgress = state.custom(PersistentTasksInProgress.TYPE); + MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) + .removeDatafeed(request.getDatafeedId(), persistentTasksInProgress).build(); + return ClusterState.builder(state).metaData( + MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build()) + .build(); + } + }); + + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java new file mode 100644 index 00000000000..bfa4a5db4bf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.delete.DeleteResponse; +import org.elasticsearch.action.delete.TransportDeleteAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + + +public class DeleteFilterAction extends Action { + + public static final DeleteFilterAction INSTANCE = new DeleteFilterAction(); + public static final String NAME = "cluster:admin/ml/filters/delete"; + + private DeleteFilterAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + public static final ParseField FILTER_ID = new ParseField("filter_id"); + + private String filterId; + + Request() { + + } + + public Request(String filterId) { + this.filterId = ExceptionsHelper.requireNonNull(filterId, FILTER_ID.getPreferredName()); + } + + public String getFilterId() { + return filterId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + filterId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(filterId); + } + + @Override + public int hashCode() { + return Objects.hash(filterId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(filterId, other.filterId); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, DeleteFilterAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final TransportDeleteAction transportAction; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + TransportDeleteAction transportAction) { + super(settings, DeleteFilterAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.transportAction = transportAction; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + + final String filterId = request.getFilterId(); + MlMetadata currentMlMetadata = state.metaData().custom(MlMetadata.TYPE); + Map jobs = currentMlMetadata.getJobs(); + List currentlyUsedBy = new ArrayList<>(); + for (Job job : jobs.values()) { + List detectors = job.getAnalysisConfig().getDetectors(); + for (Detector detector : detectors) { + if (detector.extractReferencedFilters().contains(filterId)) { + currentlyUsedBy.add(job.getId()); + break; + } + } + } + if (!currentlyUsedBy.isEmpty()) { + throw ExceptionsHelper.conflictStatusException("Cannot delete filter, currently used by jobs: " + + currentlyUsedBy); + } + + DeleteRequest deleteRequest = new DeleteRequest(JobProvider.ML_META_INDEX, MlFilter.TYPE.getPreferredName(), filterId); + transportAction.execute(deleteRequest, new ActionListener() { + @Override + public void onResponse(DeleteResponse deleteResponse) { + if (deleteResponse.status().equals(RestStatus.NOT_FOUND)) { + listener.onFailure(new ResourceNotFoundException("Could not delete filter with ID [" + filterId + + "] because it does not exist")); + } else { + listener.onResponse(new Response(true)); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Could not delete filter with ID [" + filterId + "]", e); + listener.onFailure(new IllegalStateException("Could not delete filter with ID [" + filterId + "]", e)); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteJobAction.java new file mode 100644 index 00000000000..9d32c481995 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteJobAction.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class DeleteJobAction extends Action { + + public static final DeleteJobAction INSTANCE = new DeleteJobAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/delete"; + + private DeleteJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return new JobStorageDeletionTask(id, type, action, "delete-job-" + jobId, parentTaskId); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + DeleteJobAction.Request other = (DeleteJobAction.Request) obj; + return Objects.equals(jobId, other.jobId); + } + + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + RequestBuilder(ElasticsearchClient client, DeleteJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + private final Client client; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, + Client client) { + super(settings, DeleteJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + this.client = client; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.deleteJob(request, client, (JobStorageDeletionTask) task, listener); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + throw new UnsupportedOperationException("the Task parameter is required"); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteModelSnapshotAction.java new file mode 100644 index 00000000000..2b6c0df9316 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/DeleteModelSnapshotAction.java @@ -0,0 +1,201 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.List; + +public class DeleteModelSnapshotAction extends Action { + + public static final DeleteModelSnapshotAction INSTANCE = new DeleteModelSnapshotAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/model_snapshots/delete"; + + private DeleteModelSnapshotAction() { + super(NAME); + } + + @Override + public DeleteModelSnapshotAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public DeleteModelSnapshotAction.Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + private String jobId; + private String snapshotId; + + private Request() { + } + + public Request(String jobId, String snapshotId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.snapshotId = ExceptionsHelper.requireNonNull(snapshotId, ModelSnapshot.SNAPSHOT_ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getSnapshotId() { + return snapshotId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + snapshotId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeString(snapshotId); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, DeleteModelSnapshotAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final Client client; + private final JobProvider jobProvider; + private final JobManager jobManager; + private final ClusterService clusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobProvider jobProvider, JobManager jobManager, ClusterService clusterService, + Client client) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.client = client; + this.jobProvider = jobProvider; + this.jobManager = jobManager; + this.clusterService = clusterService; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + // Verify the snapshot exists + jobProvider.modelSnapshots( + request.getJobId(), 0, 1, null, null, null, true, request.getSnapshotId(), null, + page -> { + List deleteCandidates = page.results(); + if (deleteCandidates.size() > 1) { + logger.warn("More than one model found for [job_id: " + request.getJobId() + + ", snapshot_id: " + request.getSnapshotId() + "] tuple."); + } + + if (deleteCandidates.isEmpty()) { + listener.onFailure(new ResourceNotFoundException( + Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getJobId()))); + } + ModelSnapshot deleteCandidate = deleteCandidates.get(0); + + // Verify the snapshot is not being used + // + // NORELEASE: technically, this could be stale and refuse a delete, but I think that's acceptable + // since it is non-destructive + QueryPage job = jobManager.getJob(request.getJobId(), clusterService.state()); + if (job.count() > 0) { + String currentModelInUse = job.results().get(0).getModelSnapshotId(); + if (currentModelInUse != null && currentModelInUse.equals(request.getSnapshotId())) { + throw new IllegalArgumentException(Messages.getMessage(Messages.REST_CANNOT_DELETE_HIGHEST_PRIORITY, + request.getSnapshotId(), request.getJobId())); + } + } + + // Delete the snapshot and any associated state files + JobDataDeleter deleter = new JobDataDeleter(client, request.getJobId()); + deleter.deleteModelSnapshot(deleteCandidate); + deleter.commit(new ActionListener() { + @Override + public void onResponse(BulkResponse bulkResponse) { + // We don't care about the bulk response, just that it succeeded + listener.onResponse(new DeleteModelSnapshotAction.Response(true)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + jobManager.audit(request.getJobId()).info(Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, + deleteCandidate.getDescription())); + }, listener::onFailure); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/FlushJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/FlushJobAction.java new file mode 100644 index 00000000000..8c72eb10a18 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/FlushJobAction.java @@ -0,0 +1,287 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class FlushJobAction extends Action { + + public static final FlushJobAction INSTANCE = new FlushJobAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/flush"; + + private FlushJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends TransportJobTaskAction.JobTaskRequest implements ToXContent { + + public static final ParseField CALC_INTERIM = new ParseField("calc_interim"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField ADVANCE_TIME = new ParseField("advance_time"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareBoolean(Request::setCalcInterim, CALC_INTERIM); + PARSER.declareString(Request::setStart, START); + PARSER.declareString(Request::setEnd, END); + PARSER.declareString(Request::setAdvanceTime, ADVANCE_TIME); + } + + public static Request parseRequest(String jobId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private boolean calcInterim = false; + private String start; + private String end; + private String advanceTime; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public boolean getCalcInterim() { + return calcInterim; + } + + public void setCalcInterim(boolean calcInterim) { + this.calcInterim = calcInterim; + } + + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = start; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + + public String getAdvanceTime() { return advanceTime; } + + public void setAdvanceTime(String advanceTime) { + this.advanceTime = advanceTime; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + calcInterim = in.readBoolean(); + start = in.readOptionalString(); + end = in.readOptionalString(); + advanceTime = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(calcInterim); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeOptionalString(advanceTime); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, calcInterim, start, end, advanceTime); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + calcInterim == other.calcInterim && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(advanceTime, other.advanceTime); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(CALC_INTERIM.getPreferredName(), calcInterim); + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (advanceTime != null) { + builder.field(ADVANCE_TIME.getPreferredName(), advanceTime); + } + builder.endObject(); + return builder; + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, FlushJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends BaseTasksResponse implements Writeable, ToXContentObject { + + private boolean flushed; + + Response() { + } + + Response(boolean flushed) { + super(null, null); + this.flushed = flushed; + } + + public boolean isFlushed() { + return flushed; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + flushed = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(flushed); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("flushed", flushed); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return flushed == response.flushed; + } + + @Override + public int hashCode() { + return Objects.hash(flushed); + } + } + + public static class TransportAction extends TransportJobTaskAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ClusterService clusterService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + AutodetectProcessManager processManager, JobManager jobManager) { + super(settings, FlushJobAction.NAME, threadPool, clusterService, transportService, actionFilters, + indexNameExpressionResolver, FlushJobAction.Request::new, FlushJobAction.Response::new, MlPlugin.THREAD_POOL_NAME, + jobManager, processManager, Request::getJobId); + } + + @Override + protected FlushJobAction.Response readTaskResponse(StreamInput in) throws IOException { + Response response = new Response(); + response.readFrom(in); + return response; + } + + @Override + protected void taskOperation(Request request, InternalOpenJobAction.JobTask task, + ActionListener listener) { + jobManager.getJobOrThrowIfUnknown(request.getJobId()); + + InterimResultsParams.Builder paramsBuilder = InterimResultsParams.builder(); + paramsBuilder.calcInterim(request.getCalcInterim()); + if (request.getAdvanceTime() != null) { + paramsBuilder.advanceTime(request.getAdvanceTime()); + } + TimeRange.Builder timeRangeBuilder = TimeRange.builder(); + if (request.getStart() != null) { + timeRangeBuilder.startTime(request.getStart()); + } + if (request.getEnd() != null) { + timeRangeBuilder.endTime(request.getEnd()); + } + paramsBuilder.forTimeRange(timeRangeBuilder.build()); + processManager.flushJob(request.getJobId(), paramsBuilder.build()); + listener.onResponse(new Response(true)); + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java new file mode 100644 index 00000000000..e91b3df882c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java @@ -0,0 +1,421 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetBucketsAction extends Action { + + public static final GetBucketsAction INSTANCE = new GetBucketsAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/results/buckets/get"; + + private GetBucketsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField EXPAND = new ParseField("expand"); + public static final ParseField INCLUDE_INTERIM = new ParseField("include_interim"); + public static final ParseField PARTITION_VALUE = new ParseField("partition_value"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("max_normalized_probability"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString(Request::setTimestamp, Bucket.TIMESTAMP); + PARSER.declareString(Request::setPartitionValue, PARTITION_VALUE); + PARSER.declareBoolean(Request::setExpand, EXPAND); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareStringOrNull(Request::setStart, START); + PARSER.declareStringOrNull(Request::setEnd, END); + PARSER.declareBoolean(Request::setExpand, EXPAND); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(Request::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); + PARSER.declareString(Request::setPartitionValue, PARTITION_VALUE); + } + + public static Request parseRequest(String jobId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String timestamp; + private boolean expand = false; + private boolean includeInterim = false; + private String partitionValue; + private String start; + private String end; + private PageParams pageParams; + private Double anomalyScore; + private Double maxNormalizedProbability; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public void setTimestamp(String timestamp) { + if (pageParams != null || start != null || end != null || anomalyScore != null || maxNormalizedProbability != null) { + throw new IllegalArgumentException("Param [" + TIMESTAMP.getPreferredName() + "] is incompatible with [" + + PageParams.FROM.getPreferredName() + "," + + PageParams.SIZE.getPreferredName() + "," + + START.getPreferredName() + "," + + END.getPreferredName() + "," + + ANOMALY_SCORE.getPreferredName() + "," + + MAX_NORMALIZED_PROBABILITY.getPreferredName() + "]"); + } + this.timestamp = ExceptionsHelper.requireNonNull(timestamp, Bucket.TIMESTAMP.getPreferredName()); + } + + public String getTimestamp() { + return timestamp; + } + + public boolean isExpand() { + return expand; + } + + public void setExpand(boolean expand) { + this.expand = expand; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public void setIncludeInterim(boolean includeInterim) { + this.includeInterim = includeInterim; + } + + public String getPartitionValue() { + return partitionValue; + } + + public void setPartitionValue(String partitionValue) { + if (timestamp != null) { + throw new IllegalArgumentException("Param [" + PARTITION_VALUE.getPreferredName() + "] is incompatible with [" + + TIMESTAMP.getPreferredName() + "]."); + } + this.partitionValue = partitionValue; + } + + public String getStart() { + return start; + } + + public void setStart(String start) { + if (timestamp != null) { + throw new IllegalArgumentException("Param [" + START.getPreferredName() + "] is incompatible with [" + + TIMESTAMP.getPreferredName() + "]."); + } + this.start = start; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + if (timestamp != null) { + throw new IllegalArgumentException("Param [" + END.getPreferredName() + "] is incompatible with [" + + TIMESTAMP.getPreferredName() + "]."); + } + this.end = end; + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + if (timestamp != null) { + throw new IllegalArgumentException("Param [" + PageParams.FROM.getPreferredName() + + ", " + PageParams.SIZE.getPreferredName() + "] is incompatible with [" + TIMESTAMP.getPreferredName() + "]."); + } + this.pageParams = ExceptionsHelper.requireNonNull(pageParams, PageParams.PAGE.getPreferredName()); + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + if (timestamp != null) { + throw new IllegalArgumentException("Param [" + ANOMALY_SCORE.getPreferredName() + "] is incompatible with [" + + TIMESTAMP.getPreferredName() + "]."); + } + this.anomalyScore = anomalyScore; + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + public void setMaxNormalizedProbability(double maxNormalizedProbability) { + if (timestamp != null) { + throw new IllegalArgumentException("Param [" + MAX_NORMALIZED_PROBABILITY.getPreferredName() + "] is incompatible with [" + + TIMESTAMP.getPreferredName() + "]."); + } + this.maxNormalizedProbability = maxNormalizedProbability; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + timestamp = in.readOptionalString(); + expand = in.readBoolean(); + includeInterim = in.readBoolean(); + partitionValue = in.readOptionalString(); + start = in.readOptionalString(); + end = in.readOptionalString(); + anomalyScore = in.readOptionalDouble(); + maxNormalizedProbability = in.readOptionalDouble(); + pageParams = in.readOptionalWriteable(PageParams::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(timestamp); + out.writeBoolean(expand); + out.writeBoolean(includeInterim); + out.writeOptionalString(partitionValue); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeOptionalDouble(anomalyScore); + out.writeOptionalDouble(maxNormalizedProbability); + out.writeOptionalWriteable(pageParams); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(Bucket.TIMESTAMP.getPreferredName(), timestamp); + } + builder.field(EXPAND.getPreferredName(), expand); + builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim); + if (partitionValue != null) { + builder.field(PARTITION_VALUE.getPreferredName(), partitionValue); + } + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (pageParams != null) { + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + } + if (anomalyScore != null) { + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + } + if (maxNormalizedProbability != null) { + builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, partitionValue, expand, includeInterim, + anomalyScore, maxNormalizedProbability, pageParams, start, end); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(timestamp, other.timestamp) && + Objects.equals(partitionValue, other.partitionValue) && + Objects.equals(expand, other.expand) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(anomalyScore, other.anomalyScore) && + Objects.equals(maxNormalizedProbability, other.maxNormalizedProbability) && + Objects.equals(pageParams, other.pageParams) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage buckets; + + Response() { + } + + Response(QueryPage buckets) { + this.buckets = buckets; + } + + public QueryPage getBuckets() { + return buckets; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + buckets = new QueryPage<>(in, Bucket::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + buckets.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + buckets.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(buckets); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(buckets, other.buckets); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + BucketsQueryBuilder query = + new BucketsQueryBuilder().expand(request.expand) + .includeInterim(request.includeInterim) + .start(request.start) + .end(request.end) + .anomalyScoreThreshold(request.anomalyScore) + .normalizedProbabilityThreshold(request.maxNormalizedProbability) + .partitionValue(request.partitionValue); + + if (request.pageParams != null) { + query.from(request.pageParams.getFrom()) + .size(request.pageParams.getSize()); + } + if (request.timestamp != null) { + query.timestamp(request.timestamp); + } else { + query.start(request.start); + query.end(request.end); + } + jobProvider.buckets(request.jobId, query.build(), q -> listener.onResponse(new Response(q)), listener::onFailure); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetCategoriesAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetCategoriesAction.java new file mode 100644 index 00000000000..a14f7670d97 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetCategoriesAction.java @@ -0,0 +1,255 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + +public class GetCategoriesAction extends +Action { + + public static final GetCategoriesAction INSTANCE = new GetCategoriesAction(); + private static final String NAME = "cluster:admin/ml/anomaly_detectors/results/categories/get"; + + private GetCategoriesAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField CATEGORY_ID = new ParseField("category_id"); + public static final ParseField FROM = new ParseField("from"); + public static final ParseField SIZE = new ParseField("size"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString(Request::setCategoryId, CATEGORY_ID); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + } + + public static Request parseRequest(String jobId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String categoryId; + private PageParams pageParams; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() { + } + + public String getCategoryId() { + return categoryId; + } + + public void setCategoryId(String categoryId) { + if (pageParams != null) { + throw new IllegalArgumentException("Param [" + CATEGORY_ID.getPreferredName() + "] is incompatible with [" + + PageParams.FROM.getPreferredName() + ", " + PageParams.SIZE.getPreferredName() + "]."); + } + this.categoryId = ExceptionsHelper.requireNonNull(categoryId, CATEGORY_ID.getPreferredName()); + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + if (categoryId != null) { + throw new IllegalArgumentException("Param [" + PageParams.FROM.getPreferredName() + ", " + + PageParams.SIZE.getPreferredName() + "] is incompatible with [" + CATEGORY_ID.getPreferredName() + "]."); + } + this.pageParams = pageParams; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (pageParams == null && categoryId == null) { + validationException = addValidationError("Both [" + CATEGORY_ID.getPreferredName() + "] and [" + + PageParams.FROM.getPreferredName() + ", " + PageParams.SIZE.getPreferredName() + "] " + + "cannot be null" , validationException); + } + return validationException; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + categoryId = in.readOptionalString(); + pageParams = in.readOptionalWriteable(PageParams::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(categoryId); + out.writeOptionalWriteable(pageParams); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (categoryId != null) { + builder.field(CATEGORY_ID.getPreferredName(), categoryId); + } + if (pageParams != null) { + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Request request = (Request) o; + return Objects.equals(jobId, request.jobId) + && Objects.equals(categoryId, request.categoryId) + && Objects.equals(pageParams, request.pageParams); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, categoryId, pageParams); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetCategoriesAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage result; + + public Response(QueryPage result) { + this.result = result; + } + + Response() { + } + + public QueryPage getResult() { + return result; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + result = new QueryPage<>(in, CategoryDefinition::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + result.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + result.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Response response = (Response) o; + return Objects.equals(result, response.result); + } + + @Override + public int hashCode() { + return Objects.hash(result); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + Integer from = request.pageParams != null ? request.pageParams.getFrom() : null; + Integer size = request.pageParams != null ? request.pageParams.getSize() : null; + jobProvider.categoryDefinitions(request.jobId, request.categoryId, from, size, + r -> listener.onResponse(new Response(r)), listener::onFailure); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsAction.java new file mode 100644 index 00000000000..3e20d178dad --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsAction.java @@ -0,0 +1,221 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class GetDatafeedsAction extends Action { + + public static final GetDatafeedsAction INSTANCE = new GetDatafeedsAction(); + public static final String NAME = "cluster:admin/ml/datafeeds/get"; + + public static final String ALL = "_all"; + + private GetDatafeedsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest { + + private String datafeedId; + + public Request(String datafeedId) { + this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); + } + + Request() {} + + public String getDatafeedId() { + return datafeedId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeedId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(datafeedId); + } + + @Override + public int hashCode() { + return Objects.hash(datafeedId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(datafeedId, other.datafeedId); + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetDatafeedsAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage datafeeds; + + public Response(QueryPage datafeeds) { + this.datafeeds = datafeeds; + } + + public Response() {} + + public QueryPage getResponse() { + return datafeeds; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeeds = new QueryPage<>(in, DatafeedConfig::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + datafeeds.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + datafeeds.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(datafeeds); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(datafeeds, other.datafeeds); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends TransportMasterNodeReadAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, GetDatafeedsAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.debug("Get datafeed '{}'", request.getDatafeedId()); + + QueryPage response; + MlMetadata mlMetadata = state.metaData().custom(MlMetadata.TYPE); + if (ALL.equals(request.getDatafeedId())) { + List datafeedConfigs = new ArrayList<>(mlMetadata.getDatafeeds().values()); + response = new QueryPage<>(datafeedConfigs, datafeedConfigs.size(), DatafeedConfig.RESULTS_FIELD); + } else { + DatafeedConfig datafeed = mlMetadata.getDatafeed(request.getDatafeedId()); + if (datafeed == null) { + throw ExceptionsHelper.missingDatafeedException(request.getDatafeedId()); + } + response = new QueryPage<>(Collections.singletonList(datafeed), 1, DatafeedConfig.RESULTS_FIELD); + } + + listener.onResponse(new Response(response)); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsStatsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsStatsAction.java new file mode 100644 index 00000000000..4406e58bb0c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetDatafeedsStatsAction.java @@ -0,0 +1,303 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress.PersistentTaskInProgress; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; + +public class GetDatafeedsStatsAction extends Action { + + public static final GetDatafeedsStatsAction INSTANCE = new GetDatafeedsStatsAction(); + public static final String NAME = "cluster:admin/ml/datafeeds/stats/get"; + + public static final String ALL = "_all"; + private static final String STATE = "state"; + + private GetDatafeedsStatsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest { + + private String datafeedId; + + public Request(String datafeedId) { + this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); + } + + Request() {} + + public String getDatafeedId() { + return datafeedId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeedId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(datafeedId); + } + + @Override + public int hashCode() { + return Objects.hash(datafeedId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(datafeedId, other.datafeedId); + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetDatafeedsStatsAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + public static class DatafeedStats implements ToXContent, Writeable { + + private final String datafeedId; + private final DatafeedState datafeedState; + + DatafeedStats(String datafeedId, DatafeedState datafeedState) { + this.datafeedId = Objects.requireNonNull(datafeedId); + this.datafeedState = Objects.requireNonNull(datafeedState); + } + + DatafeedStats(StreamInput in) throws IOException { + datafeedId = in.readString(); + datafeedState = DatafeedState.fromStream(in); + } + + public String getDatafeedId() { + return datafeedId; + } + + public DatafeedState getDatafeedState() { + return datafeedState; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + builder.field(STATE, datafeedState); + builder.endObject(); + + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(datafeedId); + datafeedState.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(datafeedId, datafeedState); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + GetDatafeedsStatsAction.Response.DatafeedStats other = (GetDatafeedsStatsAction.Response.DatafeedStats) obj; + return Objects.equals(datafeedId, other.datafeedId) && Objects.equals(this.datafeedState, other.datafeedState); + } + } + + private QueryPage datafeedsStats; + + public Response(QueryPage datafeedsStats) { + this.datafeedsStats = datafeedsStats; + } + + public Response() {} + + public QueryPage getResponse() { + return datafeedsStats; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeedsStats = new QueryPage<>(in, DatafeedStats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + datafeedsStats.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + datafeedsStats.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(datafeedsStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(datafeedsStats, other.datafeedsStats); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends TransportMasterNodeReadAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, GetDatafeedsStatsAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.debug("Get stats for datafeed '{}'", request.getDatafeedId()); + + Map states = new HashMap<>(); + PersistentTasksInProgress tasksInProgress = state.custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress != null) { + Predicate> predicate = ALL.equals(request.getDatafeedId()) ? p -> true : + p -> request.getDatafeedId().equals(((StartDatafeedAction.Request) p.getRequest()).getDatafeedId()); + for (PersistentTaskInProgress taskInProgress : tasksInProgress.findTasks(StartDatafeedAction.NAME, predicate)) { + StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) taskInProgress.getRequest(); + states.put(storedRequest.getDatafeedId(), DatafeedState.STARTED); + } + } + + List stats = new ArrayList<>(); + MlMetadata mlMetadata = state.metaData().custom(MlMetadata.TYPE); + if (ALL.equals(request.getDatafeedId())) { + Collection datafeeds = mlMetadata.getDatafeeds().values(); + for (DatafeedConfig datafeed : datafeeds) { + DatafeedState datafeedState = states.getOrDefault(datafeed.getId(), DatafeedState.STOPPED); + stats.add(new Response.DatafeedStats(datafeed.getId(), datafeedState)); + } + } else { + DatafeedConfig datafeed = mlMetadata.getDatafeed(request.getDatafeedId()); + if (datafeed == null) { + throw ExceptionsHelper.missingDatafeedException(request.getDatafeedId()); + } + DatafeedState datafeedState = states.getOrDefault(datafeed.getId(), DatafeedState.STOPPED); + stats.add(new Response.DatafeedStats(datafeed.getId(), datafeedState)); + } + QueryPage statsPage = new QueryPage<>(stats, stats.size(), DatafeedConfig.RESULTS_FIELD); + listener.onResponse(new Response(statsPage)); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java new file mode 100644 index 00000000000..a51705aeed3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java @@ -0,0 +1,336 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.TransportGetAction; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.TransportSearchAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; + + +public class GetFiltersAction extends Action { + + public static final GetFiltersAction INSTANCE = new GetFiltersAction(); + public static final String NAME = "cluster:admin/ml/filters/get"; + + private GetFiltersAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest { + + private String filterId; + private PageParams pageParams; + + public Request() { + } + + public void setFilterId(String filterId) { + if (pageParams != null) { + throw new IllegalArgumentException("Param [" + MlFilter.ID.getPreferredName() + "] is incompatible with [" + + PageParams.FROM.getPreferredName()+ ", " + PageParams.SIZE.getPreferredName() + "]."); + } + this.filterId = filterId; + } + + public String getFilterId() { + return filterId; + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + if (filterId != null) { + throw new IllegalArgumentException("Param [" + PageParams.FROM.getPreferredName() + + ", " + PageParams.SIZE.getPreferredName() + "] is incompatible with [" + + MlFilter.ID.getPreferredName() + "]."); + } + this.pageParams = pageParams; + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + if (pageParams == null && filterId == null) { + validationException = addValidationError("Both [" + MlFilter.ID.getPreferredName() + "] and [" + + PageParams.FROM.getPreferredName() + ", " + PageParams.SIZE.getPreferredName() + "] " + + "cannot be null" , validationException); + } + return validationException; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + filterId = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(filterId); + } + + @Override + public int hashCode() { + return Objects.hash(filterId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(filterId, other.filterId); + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetFiltersAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements StatusToXContentObject { + + private QueryPage filters; + + public Response(QueryPage filters) { + this.filters = filters; + } + + Response() { + } + + public QueryPage getFilters() { + return filters; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + filters = new QueryPage<>(in, MlFilter::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + filters.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + filters.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(filters); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(filters, other.filters); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends TransportMasterNodeReadAction { + + private final TransportGetAction transportGetAction; + private final TransportSearchAction transportSearchAction; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + TransportGetAction transportGetAction, TransportSearchAction transportSearchAction) { + super(settings, GetFiltersAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.transportGetAction = transportGetAction; + this.transportSearchAction = transportSearchAction; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + final String filterId = request.getFilterId(); + if (!Strings.isNullOrEmpty(filterId)) { + getFilter(filterId, listener); + } else if (request.getPageParams() != null) { + getFilters(request.getPageParams(), listener); + } else { + throw new IllegalStateException("Both filterId and pageParams are null"); + } + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + + private void getFilter(String filterId, ActionListener listener) { + GetRequest getRequest = new GetRequest(JobProvider.ML_META_INDEX, MlFilter.TYPE.getPreferredName(), filterId); + transportGetAction.execute(getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getDocResponse) { + + try { + QueryPage responseBody; + if (getDocResponse.isExists()) { + BytesReference docSource = getDocResponse.getSourceAsBytesRef(); + XContentParser parser = + XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource); + MlFilter filter = MlFilter.PARSER.apply(parser, null); + responseBody = new QueryPage<>(Collections.singletonList(filter), 1, MlFilter.RESULTS_FIELD); + + Response filterResponse = new Response(responseBody); + listener.onResponse(filterResponse); + } else { + this.onFailure(QueryPage.emptyQueryPage(MlFilter.RESULTS_FIELD)); + } + + } catch (Exception e) { + this.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + private void getFilters(PageParams pageParams, ActionListener listener) { + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() + .from(pageParams.getFrom()) + .size(pageParams.getSize()); + + SearchRequest searchRequest = new SearchRequest(new String[]{JobProvider.ML_META_INDEX}, sourceBuilder) + .types(MlFilter.TYPE.getPreferredName()); + + transportSearchAction.execute(searchRequest, new ActionListener() { + @Override + public void onResponse(SearchResponse response) { + + try { + List docs = new ArrayList<>(); + for (SearchHit hit : response.getHits().getHits()) { + BytesReference docSource = hit.getSourceRef(); + XContentParser parser = + XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource); + docs.add(MlFilter.PARSER.apply(parser, null)); + } + + Response filterResponse = new Response(new QueryPage<>(docs, docs.size(), MlFilter.RESULTS_FIELD)); + listener.onResponse(filterResponse); + + } catch (Exception e) { + this.onFailure(e); + } + } + + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } + +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetInfluencersAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetInfluencersAction.java new file mode 100644 index 00000000000..58bdc7ab5ae --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetInfluencersAction.java @@ -0,0 +1,320 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetInfluencersAction +extends Action { + + public static final GetInfluencersAction INSTANCE = new GetInfluencersAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/results/influencers/get"; + + private GetInfluencersAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField INCLUDE_INTERIM = new ParseField("include_interim"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + public static final ParseField SORT_FIELD = new ParseField("sort"); + public static final ParseField DESCENDING_SORT = new ParseField("desc"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareStringOrNull(Request::setStart, START); + PARSER.declareStringOrNull(Request::setEnd, END); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareString(Request::setSort, SORT_FIELD); + PARSER.declareBoolean(Request::setDecending, DESCENDING_SORT); + } + + public static Request parseRequest(String jobId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String start; + private String end; + private boolean includeInterim = false; + private PageParams pageParams = new PageParams(); + private double anomalyScoreFilter = 0.0; + private String sort = Influencer.ANOMALY_SCORE.getPreferredName(); + private boolean decending = false; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = start; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + + public boolean isDecending() { + return decending; + } + + public void setDecending(boolean decending) { + this.decending = decending; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public void setIncludeInterim(boolean includeInterim) { + this.includeInterim = includeInterim; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + public PageParams getPageParams() { + return pageParams; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public void setAnomalyScore(double anomalyScoreFilter) { + this.anomalyScoreFilter = anomalyScoreFilter; + } + + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = ExceptionsHelper.requireNonNull(sort, SORT_FIELD.getPreferredName()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + includeInterim = in.readBoolean(); + pageParams = new PageParams(in); + start = in.readOptionalString(); + end = in.readOptionalString(); + sort = in.readOptionalString(); + decending = in.readBoolean(); + anomalyScoreFilter = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(includeInterim); + pageParams.writeTo(out); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeOptionalString(sort); + out.writeBoolean(decending); + out.writeDouble(anomalyScoreFilter); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + builder.field(START.getPreferredName(), start); + builder.field(END.getPreferredName(), end); + builder.field(SORT_FIELD.getPreferredName(), sort); + builder.field(DESCENDING_SORT.getPreferredName(), decending); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScoreFilter); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, includeInterim, pageParams, start, end, sort, decending, anomalyScoreFilter); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(start, other.start) && Objects.equals(end, other.end) + && Objects.equals(includeInterim, other.includeInterim) && Objects.equals(pageParams, other.pageParams) + && Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && Objects.equals(decending, other.decending) + && Objects.equals(sort, other.sort); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage influencers; + + Response() { + } + + Response(QueryPage influencers) { + this.influencers = influencers; + } + + public QueryPage getInfluencers() { + return influencers; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + influencers = new QueryPage<>(in, Influencer::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + influencers.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + influencers.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(influencers); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(influencers, other.influencers); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder().includeInterim(request.includeInterim) + .start(request.start).end(request.end).from(request.pageParams.getFrom()).size(request.pageParams.getSize()) + .anomalyScoreThreshold(request.anomalyScoreFilter).sortField(request.sort).sortDescending(request.decending).build(); + jobProvider.influencers(request.jobId, query, page -> listener.onResponse(new Response(page)), listener::onFailure); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsAction.java new file mode 100644 index 00000000000..d2ccc67a13b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsAction.java @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeReadOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeReadAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetJobsAction extends Action { + + public static final GetJobsAction INSTANCE = new GetJobsAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/get"; + + private GetJobsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest { + + private String jobId; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + public static class RequestBuilder extends MasterNodeReadOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetJobsAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage jobs; + + public Response(QueryPage jobs) { + this.jobs = jobs; + } + + public Response() {} + + public QueryPage getResponse() { + return jobs; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobs = new QueryPage<>(in, Job::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + jobs.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + jobs.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobs); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(jobs, other.jobs); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + + public static class TransportAction extends TransportMasterNodeReadAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, GetJobsAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.debug("Get job '{}'", request.getJobId()); + QueryPage jobs = jobManager.getJob(request.getJobId(), state); + listener.onResponse(new Response(jobs)); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java new file mode 100644 index 00000000000..9002a8ab9cc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetJobsStatsAction.java @@ -0,0 +1,413 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.TaskOperationFailure; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksRequest; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.action.support.tasks.TransportTasksAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +public class GetJobsStatsAction extends Action { + + public static final GetJobsStatsAction INSTANCE = new GetJobsStatsAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/stats/get"; + + private static final String DATA_COUNTS = "data_counts"; + private static final String MODEL_SIZE_STATS = "model_size_stats"; + private static final String STATE = "state"; + + private GetJobsStatsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends BaseTasksRequest { + + private String jobId; + + // used internally to expand _all jobid to encapsulate all jobs in cluster: + private List expandedJobsIds; + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.expandedJobsIds = Collections.singletonList(jobId); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + @Override + public boolean match(Task task) { + return jobId.equals(Job.ALL) || InternalOpenJobAction.JobTask.match(task, jobId); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + expandedJobsIds = in.readList(StreamInput::readString); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeStringList(expandedJobsIds); + } + + @Override + public int hashCode() { + return Objects.hash(jobId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetJobsStatsAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends BaseTasksResponse implements ToXContentObject { + + public static class JobStats implements ToXContent, Writeable { + private final String jobId; + private DataCounts dataCounts; + @Nullable + private ModelSizeStats modelSizeStats; + private JobState state; + + JobStats(String jobId, DataCounts dataCounts, @Nullable ModelSizeStats modelSizeStats, JobState state) { + this.jobId = Objects.requireNonNull(jobId); + this.dataCounts = Objects.requireNonNull(dataCounts); + this.modelSizeStats = modelSizeStats; + this.state = Objects.requireNonNull(state); + } + + JobStats(StreamInput in) throws IOException { + jobId = in.readString(); + dataCounts = new DataCounts(in); + modelSizeStats = in.readOptionalWriteable(ModelSizeStats::new); + state = JobState.fromStream(in); + } + + public String getJobid() { + return jobId; + } + + public DataCounts getDataCounts() { + return dataCounts; + } + + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + public JobState getState() { + return state; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(DATA_COUNTS, dataCounts); + if (modelSizeStats != null) { + builder.field(MODEL_SIZE_STATS, modelSizeStats); + } + builder.field(STATE, state); + builder.endObject(); + + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + dataCounts.writeTo(out); + out.writeOptionalWriteable(modelSizeStats); + state.writeTo(out); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, dataCounts, modelSizeStats, state); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + JobStats other = (JobStats) obj; + return Objects.equals(jobId, other.jobId) + && Objects.equals(this.dataCounts, other.dataCounts) + && Objects.equals(this.modelSizeStats, other.modelSizeStats) + && Objects.equals(this.state, other.state); + } + } + + private QueryPage jobsStats; + + public Response(QueryPage jobsStats) { + super(Collections.emptyList(), Collections.emptyList()); + this.jobsStats = jobsStats; + } + + Response(List taskFailures, List nodeFailures, + QueryPage jobsStats) { + super(taskFailures, nodeFailures); + this.jobsStats = jobsStats; + } + + public Response() { + super(Collections.emptyList(), Collections.emptyList()); + } + + public QueryPage getResponse() { + return jobsStats; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobsStats = new QueryPage<>(in, JobStats::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + jobsStats.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject();; + jobsStats.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobsStats); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(jobsStats, other.jobsStats); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends TransportTasksAction> { + + private final ClusterService clusterService; + private final AutodetectProcessManager processManager; + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + ClusterService clusterService, IndexNameExpressionResolver indexNameExpressionResolver, + AutodetectProcessManager processManager, JobProvider jobProvider) { + super(settings, GetJobsStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, + indexNameExpressionResolver, Request::new, Response::new, ThreadPool.Names.MANAGEMENT); + this.clusterService = clusterService; + this.processManager = processManager; + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + if (Job.ALL.equals(request.getJobId())) { + MlMetadata mlMetadata = clusterService.state().metaData().custom(MlMetadata.TYPE); + request.expandedJobsIds = mlMetadata.getJobs().keySet().stream().collect(Collectors.toList()); + } + + ActionListener finalListener = listener; + listener = ActionListener.wrap(response -> gatherStatsForClosedJobs(request, response, finalListener), listener::onFailure); + super.doExecute(task, request, listener); + } + + @Override + protected Response newResponse(Request request, List> tasks, + List taskOperationFailures, + List failedNodeExceptions) { + List stats = new ArrayList<>(); + for (QueryPage task : tasks) { + stats.addAll(task.results()); + } + return new Response(taskOperationFailures, failedNodeExceptions, new QueryPage<>(stats, stats.size(), Job.RESULTS_FIELD)); + } + + @Override + protected QueryPage readTaskResponse(StreamInput in) throws IOException { + return new QueryPage<>(in, Response.JobStats::new); + } + + @Override + protected boolean accumulateExceptions() { + return true; + } + + @Override + protected void taskOperation(Request request, InternalOpenJobAction.JobTask task, + ActionListener> listener) { + logger.debug("Get stats for job '{}'", request.getJobId()); + MlMetadata mlMetadata = clusterService.state().metaData().custom(MlMetadata.TYPE); + Optional> stats = processManager.getStatistics(request.getJobId()); + if (stats.isPresent()) { + JobState jobState = mlMetadata.getAllocations().get(request.jobId).getState(); + Response.JobStats jobStats = new Response.JobStats(request.jobId, stats.get().v1(), stats.get().v2(), jobState); + listener.onResponse(new QueryPage<>(Collections.singletonList(jobStats), 1, Job.RESULTS_FIELD)); + } else { + listener.onResponse(new QueryPage<>(Collections.emptyList(), 0, Job.RESULTS_FIELD)); + } + } + + // Up until now we gathered the stats for jobs that were open, + // This method will fetch the stats for missing jobs, that was stored in the jobs index + void gatherStatsForClosedJobs(Request request, Response response, ActionListener listener) { + List jobIds = determineJobIdsWithoutLiveStats(request.expandedJobsIds, response.jobsStats.results()); + if (jobIds.isEmpty()) { + listener.onResponse(response); + return; + } + + MlMetadata mlMetadata = clusterService.state().metaData().custom(MlMetadata.TYPE); + AtomicInteger counter = new AtomicInteger(jobIds.size()); + AtomicArray jobStats = new AtomicArray<>(jobIds.size()); + for (int i = 0; i < jobIds.size(); i++) { + int slot = i; + String jobId = jobIds.get(i); + gatherDataCountsAndModelSizeStats(jobId, (dataCounts, modelSizeStats) -> { + JobState jobState = mlMetadata.getAllocations().get(jobId).getState(); + jobStats.set(slot, new Response.JobStats(jobId, dataCounts, modelSizeStats, jobState)); + if (counter.decrementAndGet() == 0) { + List results = response.getResponse().results(); + results.addAll(jobStats.asList().stream() + .map(e -> e.value) + .collect(Collectors.toList())); + listener.onResponse(new Response(response.getTaskFailures(), response.getNodeFailures(), + new QueryPage<>(results, results.size(), Job.RESULTS_FIELD))); + } + }, listener::onFailure); + } + } + + void gatherDataCountsAndModelSizeStats(String jobId, BiConsumer handler, + Consumer errorHandler) { + jobProvider.dataCounts(jobId, dataCounts -> { + jobProvider.modelSizeStats(jobId, modelSizeStats -> { + handler.accept(dataCounts, modelSizeStats); + }, errorHandler); + }, errorHandler); + } + + static List determineJobIdsWithoutLiveStats(List requestedJobIds, List stats) { + List jobIds = new ArrayList<>(); + outer: for (String jobId : requestedJobIds) { + for (Response.JobStats stat : stats) { + if (stat.getJobid().equals(jobId)) { + // we already have stats, no need to get stats for this job from an index + continue outer; + } + } + jobIds.add(jobId); + } + return jobIds; + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsAction.java new file mode 100644 index 00000000000..5c9f0c4c06b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsAction.java @@ -0,0 +1,350 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetModelSnapshotsAction +extends Action { + + public static final GetModelSnapshotsAction INSTANCE = new GetModelSnapshotsAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/model_snapshots/get"; + + private GetModelSnapshotsAction() { + super(NAME); + } + + @Override + public GetModelSnapshotsAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public GetModelSnapshotsAction.Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id"); + public static final ParseField SORT = new ParseField("sort"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField DESC = new ParseField("desc"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString((request, snapshotId) -> request.snapshotId = snapshotId, SNAPSHOT_ID); + PARSER.declareString(Request::setDescriptionString, DESCRIPTION); + PARSER.declareString(Request::setStart, START); + PARSER.declareString(Request::setEnd, END); + PARSER.declareString(Request::setSort, SORT); + PARSER.declareBoolean(Request::setDescOrder, DESC); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + } + + public static Request parseRequest(String jobId, String snapshotId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + if (snapshotId != null) { + request.snapshotId = snapshotId; + } + return request; + } + + private String jobId; + private String snapshotId; + private String sort; + private String description; + private String start; + private String end; + private boolean desc; + private PageParams pageParams = new PageParams(); + + Request() { + } + + public Request(String jobId, String snapshotId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.snapshotId = snapshotId; + } + + public String getJobId() { + return jobId; + } + + @Nullable + public String getSnapshotId() { + return snapshotId; + } + + @Nullable + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = sort; + } + + public boolean getDescOrder() { + return desc; + } + + public void setDescOrder(boolean desc) { + this.desc = desc; + } + + public PageParams getPageParams() { + return pageParams; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = ExceptionsHelper.requireNonNull(pageParams, PageParams.PAGE.getPreferredName()); + } + + @Nullable + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = ExceptionsHelper.requireNonNull(start, START.getPreferredName()); + } + + @Nullable + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = ExceptionsHelper.requireNonNull(end, END.getPreferredName()); + } + + @Nullable + public String getDescriptionString() { + return description; + } + + public void setDescriptionString(String description) { + this.description = ExceptionsHelper.requireNonNull(description, DESCRIPTION.getPreferredName()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + snapshotId = in.readOptionalString(); + sort = in.readOptionalString(); + description = in.readOptionalString(); + start = in.readOptionalString(); + end = in.readOptionalString(); + desc = in.readBoolean(); + pageParams = new PageParams(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeOptionalString(snapshotId); + out.writeOptionalString(sort); + out.writeOptionalString(description); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeBoolean(desc); + pageParams.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (snapshotId != null) { + builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); + } + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (sort != null) { + builder.field(SORT.getPreferredName(), sort); + } + builder.field(DESC.getPreferredName(), desc); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, snapshotId, description, start, end, sort, desc); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(snapshotId, other.snapshotId) + && Objects.equals(description, other.description) && Objects.equals(start, other.start) + && Objects.equals(end, other.end) && Objects.equals(sort, other.sort) && Objects.equals(desc, other.desc); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage page; + + public Response(QueryPage page) { + this.page = page; + } + + Response() { + } + + public QueryPage getPage() { + return page; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + page = new QueryPage<>(in, ModelSnapshot::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + page.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + page.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(page); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(page, other.page); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, GetModelSnapshotsAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + logger.debug("Get model snapshots for job {} snapshot ID {}. from = {}, size = {}" + + " start = '{}', end='{}', sort={} descending={}, description filter={}", + request.getJobId(), request.getSnapshotId(), request.pageParams.getFrom(), request.pageParams.getSize(), + request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder(), request.getDescriptionString()); + + jobProvider.modelSnapshots(request.getJobId(), request.pageParams.getFrom(), request.pageParams.getSize(), + request.getStart(), request.getEnd(), request.getSort(), request.getDescOrder(), request.getSnapshotId(), + request.getDescriptionString(), + page -> { + clearQuantiles(page); + listener.onResponse(new Response(page)); + }, listener::onFailure); + } + + public static void clearQuantiles(QueryPage page) { + if (page.results() != null) { + for (ModelSnapshot modelSnapshot : page.results()) { + modelSnapshot.setQuantiles(null); + } + } + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetRecordsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetRecordsAction.java new file mode 100644 index 00000000000..1f234c29226 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/GetRecordsAction.java @@ -0,0 +1,364 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.RecordsQueryBuilder; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class GetRecordsAction extends Action { + + public static final GetRecordsAction INSTANCE = new GetRecordsAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/results/records/get"; + + private GetRecordsAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField INCLUDE_INTERIM = new ParseField("include_interim"); + public static final ParseField ANOMALY_SCORE_FILTER = new ParseField("anomaly_score"); + public static final ParseField SORT = new ParseField("sort"); + public static final ParseField DESCENDING = new ParseField("desc"); + public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("normalized_probability"); + public static final ParseField PARTITION_VALUE = new ParseField("partition_value"); + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareStringOrNull(Request::setStart, START); + PARSER.declareStringOrNull(Request::setEnd, END); + PARSER.declareString(Request::setPartitionValue, PARTITION_VALUE); + PARSER.declareString(Request::setSort, SORT); + PARSER.declareBoolean(Request::setDecending, DESCENDING); + PARSER.declareBoolean(Request::setIncludeInterim, INCLUDE_INTERIM); + PARSER.declareObject(Request::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(Request::setAnomalyScore, ANOMALY_SCORE_FILTER); + PARSER.declareDouble(Request::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); + } + + public static Request parseRequest(String jobId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + return request; + } + + private String jobId; + private String start; + private String end; + private boolean includeInterim = false; + private PageParams pageParams = new PageParams(); + private double anomalyScoreFilter = 0.0; + private String sort = Influencer.ANOMALY_SCORE.getPreferredName(); + private boolean decending = false; + private double maxNormalizedProbability = 0.0; + private String partitionValue; + + Request() { + } + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getStart() { + return start; + } + + public void setStart(String start) { + this.start = start; + } + + public String getEnd() { + return end; + } + + public void setEnd(String end) { + this.end = end; + } + + public boolean isDecending() { + return decending; + } + + public void setDecending(boolean decending) { + this.decending = decending; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public void setIncludeInterim(boolean includeInterim) { + this.includeInterim = includeInterim; + } + + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + public PageParams getPageParams() { + return pageParams; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public void setAnomalyScore(double anomalyScoreFilter) { + this.anomalyScoreFilter = anomalyScoreFilter; + } + + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = ExceptionsHelper.requireNonNull(sort, SORT.getPreferredName()); + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + public void setMaxNormalizedProbability(double maxNormalizedProbability) { + this.maxNormalizedProbability = maxNormalizedProbability; + } + + public String getPartitionValue() { + return partitionValue; + } + + public void setPartitionValue(String partitionValue) { + this.partitionValue = ExceptionsHelper.requireNonNull(partitionValue, PARTITION_VALUE.getPreferredName()); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + includeInterim = in.readBoolean(); + pageParams = new PageParams(in); + start = in.readOptionalString(); + end = in.readOptionalString(); + sort = in.readOptionalString(); + decending = in.readBoolean(); + anomalyScoreFilter = in.readDouble(); + maxNormalizedProbability = in.readDouble(); + partitionValue = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(includeInterim); + pageParams.writeTo(out); + out.writeOptionalString(start); + out.writeOptionalString(end); + out.writeOptionalString(sort); + out.writeBoolean(decending); + out.writeDouble(anomalyScoreFilter); + out.writeDouble(maxNormalizedProbability); + out.writeOptionalString(partitionValue); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(START.getPreferredName(), start); + builder.field(END.getPreferredName(), end); + builder.field(SORT.getPreferredName(), sort); + builder.field(DESCENDING.getPreferredName(), decending); + builder.field(ANOMALY_SCORE_FILTER.getPreferredName(), anomalyScoreFilter); + builder.field(INCLUDE_INTERIM.getPreferredName(), includeInterim); + builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + if (partitionValue != null) { + builder.field(PARTITION_VALUE.getPreferredName(), partitionValue); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, start, end, sort, decending, anomalyScoreFilter, includeInterim, maxNormalizedProbability, + pageParams, partitionValue); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(sort, other.sort) && + Objects.equals(decending, other.decending) && + Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(maxNormalizedProbability, other.maxNormalizedProbability) && + Objects.equals(pageParams, other.pageParams) && + Objects.equals(partitionValue, other.partitionValue); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private QueryPage records; + + Response() { + } + + Response(QueryPage records) { + this.records = records; + } + + public QueryPage getRecords() { + return records; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + records = new QueryPage<>(in, AnomalyRecord::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + records.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + records.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(records); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(records, other.records); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + RecordsQueryBuilder.RecordsQuery query = new RecordsQueryBuilder() + .includeInterim(request.includeInterim) + .epochStart(request.start) + .epochEnd(request.end) + .from(request.pageParams.getFrom()) + .size(request.pageParams.getSize()) + .anomalyScoreThreshold(request.anomalyScoreFilter) + .sortField(request.sort) + .sortDescending(request.decending) + .build(); + jobProvider.records(request.jobId, query, page -> listener.onResponse(new Response(page)), listener::onFailure); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/InternalOpenJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/InternalOpenJobAction.java new file mode 100644 index 00000000000..74825f2bdae --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/InternalOpenJobAction.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; + +public class InternalOpenJobAction extends Action { + + public static final InternalOpenJobAction INSTANCE = new InternalOpenJobAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/internal_open"; + + private InternalOpenJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends OpenJobAction.Request { + + public Request(String jobId) { + super(jobId); + } + + Request() { + super(); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return new JobTask(getJobId(), id, type, action, parentTaskId); + } + + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, InternalOpenJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse { + + Response() {} + + } + + public static class JobTask extends CancellableTask { + + private volatile Runnable cancelHandler; + + JobTask(String jobId, long id, String type, String action, TaskId parentTask) { + super(id, type, action, "job-" + jobId, parentTask); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + @Override + protected void onCancelled() { + cancelHandler.run(); + } + + static boolean match(Task task, String expectedJobId) { + String expectedDescription = "job-" + expectedJobId; + return task instanceof JobTask && expectedDescription.equals(task.getDescription()); + } + + } + + public static class TransportAction extends HandledTransportAction { + + private final AutodetectProcessManager autodetectProcessManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + AutodetectProcessManager autodetectProcessManager) { + super(settings, InternalOpenJobAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + Request::new); + this.autodetectProcessManager = autodetectProcessManager; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + JobTask jobTask = (JobTask) task; + autodetectProcessManager.setJobState(request.getJobId(), JobState.OPENING, aVoid -> { + jobTask.cancelHandler = () -> autodetectProcessManager.closeJob(request.getJobId()); + autodetectProcessManager.openJob(request.getJobId(), request.isIgnoreDowntime(), e -> { + if (e == null) { + listener.onResponse(new Response()); + } else { + listener.onFailure(e); + } + }); + }, listener::onFailure); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + throw new IllegalStateException("shouldn't get invoked"); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/MlDeleteByQueryAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/MlDeleteByQueryAction.java new file mode 100644 index 00000000000..5a90978ad45 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/MlDeleteByQueryAction.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.byscroll.AbstractBulkByScrollRequestBuilder; +import org.elasticsearch.action.bulk.byscroll.AsyncDeleteByQueryAction; +import org.elasticsearch.action.bulk.byscroll.BulkByScrollParallelizationHelper; +import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse; +import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest; +import org.elasticsearch.action.bulk.byscroll.ParentBulkByScrollTask; +import org.elasticsearch.action.bulk.byscroll.WorkingBulkByScrollTask; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.client.ParentTaskAssigningClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +public class MlDeleteByQueryAction extends Action { + + public static final MlDeleteByQueryAction INSTANCE = new MlDeleteByQueryAction(); + public static final String NAME = "indices:data/write/delete/mlbyquery"; + + private MlDeleteByQueryAction() { + super(NAME); + } + + @Override + public MlDeleteByQueryRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new MlDeleteByQueryRequestBuilder(client, this); + } + + @Override + public BulkByScrollResponse newResponse() { + return new BulkByScrollResponse(); + } + + public static class MlDeleteByQueryRequestBuilder extends + AbstractBulkByScrollRequestBuilder { + + public MlDeleteByQueryRequestBuilder(ElasticsearchClient client, + Action action) { + this(client, action, new SearchRequestBuilder(client, SearchAction.INSTANCE)); + } + + private MlDeleteByQueryRequestBuilder(ElasticsearchClient client, + Action action, + SearchRequestBuilder search) { + super(client, action, search, new DeleteByQueryRequest(search.request())); + } + + @Override + protected MlDeleteByQueryRequestBuilder self() { + return this; + } + + @Override + public MlDeleteByQueryRequestBuilder abortOnVersionConflict(boolean abortOnVersionConflict) { + request.setAbortOnVersionConflict(abortOnVersionConflict); + return this; + } + } + + public static class TransportAction extends HandledTransportAction { + private final Client client; + private final ScriptService scriptService; + private final ClusterService clusterService; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver resolver, Client client, TransportService transportService, + ScriptService scriptService, ClusterService clusterService) { + super(settings, MlDeleteByQueryAction.NAME, threadPool, transportService, actionFilters, resolver, DeleteByQueryRequest::new); + this.client = client; + this.scriptService = scriptService; + this.clusterService = clusterService; + } + + @Override + public void doExecute(Task task, DeleteByQueryRequest request, ActionListener listener) { + if (request.getSlices() > 1) { + BulkByScrollParallelizationHelper.startSlices(client, taskManager, MlDeleteByQueryAction.INSTANCE, + clusterService.localNode().getId(), (ParentBulkByScrollTask) task, request, listener); + } else { + ClusterState state = clusterService.state(); + ParentTaskAssigningClient client = new ParentTaskAssigningClient(this.client, clusterService.localNode(), task); + new AsyncDeleteByQueryAction((WorkingBulkByScrollTask) task, logger, client, threadPool, request, scriptService, state, + listener).start(); + } + } + + @Override + protected void doExecute(DeleteByQueryRequest request, ActionListener listener) { + throw new UnsupportedOperationException("task required"); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java new file mode 100644 index 00000000000..feb297930f5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/OpenJobAction.java @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.LoggingTaskListener; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.JobStateObserver; + +import java.io.IOException; +import java.util.Objects; + +public class OpenJobAction extends Action { + + public static final OpenJobAction INSTANCE = new OpenJobAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/open"; + + private OpenJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest { + + public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime"); + + private String jobId; + private boolean ignoreDowntime; + private TimeValue openTimeout = TimeValue.timeValueSeconds(20); + + public Request(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public boolean isIgnoreDowntime() { + return ignoreDowntime; + } + + public void setIgnoreDowntime(boolean ignoreDowntime) { + this.ignoreDowntime = ignoreDowntime; + } + + public TimeValue getOpenTimeout() { + return openTimeout; + } + + public void setOpenTimeout(TimeValue openTimeout) { + this.openTimeout = openTimeout; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + ignoreDowntime = in.readBoolean(); + openTimeout = TimeValue.timeValueMillis(in.readVLong()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeBoolean(ignoreDowntime); + out.writeVLong(openTimeout.millis()); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, ignoreDowntime, openTimeout); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + OpenJobAction.Request other = (OpenJobAction.Request) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(ignoreDowntime, other.ignoreDowntime) && + Objects.equals(openTimeout, other.openTimeout); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, OpenJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private boolean opened; + + Response() {} + + Response(boolean opened) { + this.opened = opened; + } + + public boolean isOpened() { + return opened; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + opened = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(opened); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("opened", opened); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return opened == response.opened; + } + + @Override + public int hashCode() { + return Objects.hash(opened); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobStateObserver observer; + private final ClusterService clusterService; + private final InternalOpenJobAction.TransportAction internalOpenJobAction; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, InternalOpenJobAction.TransportAction internalOpenJobAction) { + super(settings, OpenJobAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.clusterService = clusterService; + this.observer = new JobStateObserver(threadPool, clusterService); + this.internalOpenJobAction = internalOpenJobAction; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + // This validation happens also in InternalOpenJobAction, the reason we do it here too is that if it fails there + // we are unable to provide the user immediate feedback. We would create the task and the validation would fail + // in the background, whereas now the validation failure is part of the response being returned. + MlMetadata mlMetadata = clusterService.state().metaData().custom(MlMetadata.TYPE); + validate(mlMetadata, request.getJobId()); + + InternalOpenJobAction.Request internalRequest = new InternalOpenJobAction.Request(request.jobId); + internalOpenJobAction.execute(internalRequest, LoggingTaskListener.instance()); + observer.waitForState(request.getJobId(), request.openTimeout, JobState.OPENED, e -> { + if (e != null) { + listener.onFailure(e); + } else { + listener.onResponse(new Response(true)); + } + }); + } + + /** + * Fail fast before trying to update the job state on master node if the job doesn't exist or its state + * is not what it should be. + */ + public static void validate(MlMetadata mlMetadata, String jobId) { + MlMetadata.Builder builder = new MlMetadata.Builder(mlMetadata); + builder.updateState(jobId, JobState.OPENING, null); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PostDataAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PostDataAction.java new file mode 100644 index 00000000000..fc8e42f6048 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PostDataAction.java @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange; + +import java.io.IOException; +import java.util.Objects; +import java.util.Optional; + +public class PostDataAction extends Action { + + public static final PostDataAction INSTANCE = new PostDataAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/data/post"; + + private PostDataAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, PostDataAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends BaseTasksResponse implements StatusToXContentObject, Writeable { + + private DataCounts dataCounts; + + Response(String jobId) { + dataCounts = new DataCounts(jobId); + } + + private Response() { + } + + public Response(DataCounts counts) { + super(null, null); + this.dataCounts = counts; + } + + public DataCounts getDataCounts() { + return dataCounts; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + dataCounts = new DataCounts(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + dataCounts.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.ACCEPTED; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + dataCounts.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hashCode(dataCounts); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + + return Objects.equals(dataCounts, other.dataCounts); + + } + } + + public static class Request extends TransportJobTaskAction.JobTaskRequest { + + public static final ParseField RESET_START = new ParseField("reset_start"); + public static final ParseField RESET_END = new ParseField("reset_end"); + + private String resetStart = ""; + private String resetEnd = ""; + private DataDescription dataDescription; + private BytesReference content; + + Request() { + } + + public Request(String jobId) { + super(jobId); + } + + public String getResetStart() { + return resetStart; + } + + public void setResetStart(String resetStart) { + this.resetStart = resetStart; + } + + public String getResetEnd() { + return resetEnd; + } + + public void setResetEnd(String resetEnd) { + this.resetEnd = resetEnd; + } + + public DataDescription getDataDescription() { + return dataDescription; + } + + public void setDataDescription(DataDescription dataDescription) { + this.dataDescription = dataDescription; + } + + public BytesReference getContent() { return content; } + + public void setContent(BytesReference content) { + this.content = content; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + resetStart = in.readOptionalString(); + resetEnd = in.readOptionalString(); + dataDescription = in.readOptionalWriteable(DataDescription::new); + content = in.readBytesReference(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalString(resetStart); + out.writeOptionalString(resetEnd); + out.writeOptionalWriteable(dataDescription); + out.writeBytesReference(content); + } + + @Override + public int hashCode() { + // content stream not included + return Objects.hash(jobId, resetStart, resetEnd, dataDescription); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + + // content stream not included + return Objects.equals(jobId, other.jobId) && + Objects.equals(resetStart, other.resetStart) && + Objects.equals(resetEnd, other.resetEnd) && + Objects.equals(dataDescription, other.dataDescription); + } + } + + + public static class TransportAction extends TransportJobTaskAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ClusterService clusterService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager, AutodetectProcessManager processManager) { + super(settings, PostDataAction.NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + Request::new, Response::new, MlPlugin.THREAD_POOL_NAME, jobManager, processManager, Request::getJobId); + } + + @Override + protected Response readTaskResponse(StreamInput in) throws IOException { + Response response = new Response(); + response.readFrom(in); + return response; + } + + @Override + protected void taskOperation(Request request, InternalOpenJobAction.JobTask task, ActionListener listener) { + TimeRange timeRange = TimeRange.builder().startTime(request.getResetStart()).endTime(request.getResetEnd()).build(); + DataLoadParams params = new DataLoadParams(timeRange, Optional.ofNullable(request.getDataDescription())); + threadPool.executor(MlPlugin.THREAD_POOL_NAME).execute(() -> { + try { + DataCounts dataCounts = processManager.processData(request.getJobId(), request.content.streamInput(), params); + listener.onResponse(new Response(dataCounts)); + } catch (Exception e) { + listener.onFailure(e); + } + }); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutDatafeedAction.java new file mode 100644 index 00000000000..413356e1bd8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutDatafeedAction.java @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.io.IOException; +import java.util.Objects; + +public class PutDatafeedAction extends Action { + + public static final PutDatafeedAction INSTANCE = new PutDatafeedAction(); + public static final String NAME = "cluster:admin/ml/datafeeds/put"; + + private PutDatafeedAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static Request parseRequest(String datafeedId, XContentParser parser) { + DatafeedConfig.Builder datafeed = DatafeedConfig.PARSER.apply(parser, null); + datafeed.setId(datafeedId); + return new Request(datafeed.build()); + } + + private DatafeedConfig datafeed; + + public Request(DatafeedConfig datafeed) { + this.datafeed = datafeed; + } + + Request() { + } + + public DatafeedConfig getDatafeed() { + return datafeed; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeed = new DatafeedConfig(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + datafeed.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + datafeed.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(datafeed, request.datafeed); + } + + @Override + public int hashCode() { + return Objects.hash(datafeed); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PutDatafeedAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse implements ToXContentObject { + + private DatafeedConfig datafeed; + + public Response(boolean acked, DatafeedConfig datafeed) { + super(acked); + this.datafeed = datafeed; + } + + Response() { + } + + public DatafeedConfig getResponse() { + return datafeed; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + datafeed = new DatafeedConfig(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + datafeed.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + datafeed.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(datafeed, response.datafeed); + } + + @Override + public int hashCode() { + return Objects.hash(datafeed); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, PutDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + clusterService.submitStateUpdateTask("put-datafeed-" + request.getDatafeed().getId(), + new AckedClusterStateUpdateTask(request, listener) { + + @Override + protected Response newResponse(boolean acknowledged) { + if (acknowledged) { + logger.info("Created datafeed [{}]", request.getDatafeed().getId()); + } + return new Response(acknowledged, request.getDatafeed()); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return putDatafeed(request, currentState); + } + }); + } + + private ClusterState putDatafeed(Request request, ClusterState clusterState) { + MlMetadata currentMetadata = clusterState.getMetaData().custom(MlMetadata.TYPE); + MlMetadata newMetadata = new MlMetadata.Builder(currentMetadata) + .putDatafeed(request.getDatafeed()).build(); + return ClusterState.builder(clusterState).metaData( + MetaData.builder(clusterState.getMetaData()).putCustom(MlMetadata.TYPE, newMetadata).build()) + .build(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java new file mode 100644 index 00000000000..d175dbb88a9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.index.TransportIndexAction; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeReadRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + + +public class PutFilterAction extends Action { + + public static final PutFilterAction INSTANCE = new PutFilterAction(); + public static final String NAME = "cluster:admin/ml/filters/put"; + + private PutFilterAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeReadRequest implements ToXContent { + + public static Request parseRequest(XContentParser parser) { + MlFilter filter = MlFilter.PARSER.apply(parser, null); + return new Request(filter); + } + + private MlFilter filter; + + Request() { + + } + + public Request(MlFilter filter) { + this.filter = ExceptionsHelper.requireNonNull(filter, "filter"); + } + + public MlFilter getFilter() { + return this.filter; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + filter = new MlFilter(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + filter.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + filter.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(filter); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(filter, other.filter); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PutFilterAction action) { + super(client, action, new Request()); + } + } + public static class Response extends AcknowledgedResponse { + + public Response() { + super(true); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + + } + + // extends TransportMasterNodeAction, because we will store in cluster state. + public static class TransportAction extends TransportMasterNodeAction { + + private final TransportIndexAction transportIndexAction; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + TransportIndexAction transportIndexAction) { + super(settings, PutFilterAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.transportIndexAction = transportIndexAction; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + MlFilter filter = request.getFilter(); + final String filterId = filter.getId(); + IndexRequest indexRequest = new IndexRequest(JobProvider.ML_META_INDEX, MlFilter.TYPE.getPreferredName(), filterId); + XContentBuilder builder = XContentFactory.jsonBuilder(); + indexRequest.source(filter.toXContent(builder, ToXContent.EMPTY_PARAMS)); + transportIndexAction.execute(indexRequest, new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + listener.onResponse(new Response()); + } + + @Override + public void onFailure(Exception e) { + logger.error("Could not create filter with ID [" + filterId + "]", e); + throw new ResourceNotFoundException("Could not create filter with ID [" + filterId + "]", e); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutJobAction.java new file mode 100644 index 00000000000..f6525873012 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/PutJobAction.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Objects; + +public class PutJobAction extends Action { + + public static final PutJobAction INSTANCE = new PutJobAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/put"; + + private PutJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static Request parseRequest(String jobId, XContentParser parser) { + Job job = Job.PARSER.apply(parser, null).build(true, jobId); + return new Request(job); + } + + private Job job; + + public Request(Job job) { + this.job = job; + } + + Request() { + } + + public Job getJob() { + return job; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + job = new Job(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + job.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + job.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(job, request.job); + } + + @Override + public int hashCode() { + return Objects.hash(job); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, PutJobAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse implements ToXContentObject { + + private Job job; + + public Response(boolean acked, Job job) { + super(acked); + this.job = job; + } + + Response() { + } + + public Job getResponse() { + return job; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + job = new Job(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + job.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + // Don't serialize acknowledged because current api directly serializes the job details + builder.startObject(); + job.doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Response response = (Response) o; + return Objects.equals(job, response.job); + } + + @Override + public int hashCode() { + return Objects.hash(job); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager) { + super(settings, PutJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.putJob(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotAction.java new file mode 100644 index 00000000000..6e38782764a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotAction.java @@ -0,0 +1,392 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobDataDeleter; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +public class RevertModelSnapshotAction +extends Action { + + public static final RevertModelSnapshotAction INSTANCE = new RevertModelSnapshotAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/model_snapshots/revert"; + + private RevertModelSnapshotAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id"); + public static final ParseField DELETE_INTERVENING = new ParseField("delete_intervening_results"); + + private static ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString((request, snapshotId) -> request.snapshotId = snapshotId, SNAPSHOT_ID); + PARSER.declareBoolean(Request::setDeleteInterveningResults, DELETE_INTERVENING); + } + + public static Request parseRequest(String jobId, String snapshotId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + if (snapshotId != null) { + request.snapshotId = snapshotId; + } + return request; + } + + private String jobId; + private String snapshotId; + private boolean deleteInterveningResults; + + Request() { + } + + public Request(String jobId, String snapshotId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.snapshotId = ExceptionsHelper.requireNonNull(snapshotId, SNAPSHOT_ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getSnapshotId() { + return snapshotId; + } + + public boolean getDeleteInterveningResults() { + return deleteInterveningResults; + } + + public void setDeleteInterveningResults(boolean deleteInterveningResults) { + this.deleteInterveningResults = deleteInterveningResults; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + snapshotId = in.readString(); + deleteInterveningResults = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeString(snapshotId); + out.writeBoolean(deleteInterveningResults); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); + builder.field(DELETE_INTERVENING.getPreferredName(), deleteInterveningResults); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, snapshotId, deleteInterveningResults); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(snapshotId, other.snapshotId) + && Objects.equals(deleteInterveningResults, other.deleteInterveningResults); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class Response extends AcknowledgedResponse implements StatusToXContentObject { + + private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged"); + private static final ParseField MODEL = new ParseField("model"); + private ModelSnapshot model; + + Response() { + + } + + public Response(ModelSnapshot modelSnapshot) { + super(true); + model = modelSnapshot; + } + + public ModelSnapshot getModel() { + return model; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + model = new ModelSnapshot(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + model.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ACKNOWLEDGED.getPreferredName(), true); + builder.field(MODEL.getPreferredName()); + builder = model.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(model); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(model, other.model); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final Client client; + private final JobManager jobManager; + private final JobProvider jobProvider; + private final JobDataCountsPersister jobDataCountsPersister; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, JobProvider jobProvider, + ClusterService clusterService, Client client, JobDataCountsPersister jobDataCountsPersister) { + super(settings, NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, Request::new); + this.client = client; + this.jobManager = jobManager; + this.jobProvider = jobProvider; + this.jobDataCountsPersister = jobDataCountsPersister; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + logger.debug("Received request to revert to snapshot id '{}' for job '{}', deleting intervening results: {}", + request.getSnapshotId(), request.getJobId(), request.getDeleteInterveningResults()); + + QueryPage job = jobManager.getJob(request.getJobId(), clusterService.state()); + Allocation allocation = jobManager.getJobAllocation(request.getJobId()); + if (job.count() > 0 && allocation.getState().equals(JobState.CLOSED) == false) { + throw ExceptionsHelper.conflictStatusException(Messages.getMessage(Messages.REST_JOB_NOT_CLOSED_REVERT)); + } + + getModelSnapshot(request, jobProvider, modelSnapshot -> { + ActionListener wrappedListener = listener; + if (request.getDeleteInterveningResults()) { + wrappedListener = wrapDeleteOldDataListener(wrappedListener, modelSnapshot, request.getJobId()); + wrappedListener = wrapRevertDataCountsListener(wrappedListener, modelSnapshot, request.getJobId()); + } + jobManager.revertSnapshot(request, wrappedListener, modelSnapshot); + }, listener::onFailure); + } + + private void getModelSnapshot(Request request, JobProvider provider, Consumer handler, + Consumer errorHandler) { + logger.info("Reverting to snapshot '" + request.getSnapshotId() + "'"); + + provider.modelSnapshots(request.getJobId(), 0, 1, null, null, + ModelSnapshot.TIMESTAMP.getPreferredName(), true, request.getSnapshotId(), request.getDescription(), + page -> { + List revertCandidates = page.results(); + if (revertCandidates == null || revertCandidates.isEmpty()) { + throw new ResourceNotFoundException( + Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getJobId())); + } + ModelSnapshot modelSnapshot = revertCandidates.get(0); + + // The quantiles can be large, and totally dominate the output - + // it's clearer to remove them + modelSnapshot.setQuantiles(null); + handler.accept(modelSnapshot); + }, errorHandler); + } + + private ActionListener wrapDeleteOldDataListener( + ActionListener listener, + ModelSnapshot modelSnapshot, String jobId) { + + // If we need to delete buckets that occurred after the snapshot, we + // wrap the listener with one that invokes the OldDataRemover on + // acknowledged responses + return ActionListener.wrap(response -> { + if (response.isAcknowledged()) { + Date deleteAfter = modelSnapshot.getLatestResultTimeStamp(); + logger.debug("Removing intervening records: last record: " + deleteAfter + ", last result: " + + modelSnapshot.getLatestResultTimeStamp()); + + logger.info("Deleting results after '" + deleteAfter + "'"); + + // NORELEASE: JobDataDeleter is basically delete-by-query. + // We should replace this whole abstraction with DBQ eventually + JobDataDeleter dataDeleter = new JobDataDeleter(client, jobId); + dataDeleter.deleteResultsFromTime(deleteAfter.getTime() + 1, new ActionListener() { + @Override + public void onResponse(Boolean success) { + dataDeleter.commit(ActionListener.wrap( + bulkItemResponses -> {listener.onResponse(response);}, + listener::onFailure)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + }, listener::onFailure); + } + + private ActionListener wrapRevertDataCountsListener( + ActionListener listener, + ModelSnapshot modelSnapshot, String jobId) { + + + return ActionListener.wrap(response -> { + if (response.isAcknowledged()) { + jobProvider.dataCounts(jobId, counts -> { + counts.setLatestRecordTimeStamp(modelSnapshot.getLatestRecordTimeStamp()); + jobDataCountsPersister.persistDataCounts(jobId, counts, new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + listener.onResponse(response); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + }, listener::onFailure); + } + }, listener::onFailure); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/StartDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/StartDatafeedAction.java new file mode 100644 index 00000000000..49d4f8bb4d5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/StartDatafeedAction.java @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunner; +import org.elasticsearch.xpack.ml.datafeed.DatafeedJobValidator; +import org.elasticsearch.xpack.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.persistent.PersistentActionRegistry; +import org.elasticsearch.xpack.persistent.PersistentActionRequest; +import org.elasticsearch.xpack.persistent.PersistentActionResponse; +import org.elasticsearch.xpack.persistent.PersistentActionService; +import org.elasticsearch.xpack.persistent.PersistentTask; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; +import org.elasticsearch.xpack.persistent.TransportPersistentAction; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Predicate; + +public class StartDatafeedAction + extends Action { + + public static final ParseField START_TIME = new ParseField("start"); + public static final ParseField END_TIME = new ParseField("end"); + + public static final StartDatafeedAction INSTANCE = new StartDatafeedAction(); + public static final String NAME = "cluster:admin/ml/datafeeds/start"; + + private StartDatafeedAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + public static class Request extends PersistentActionRequest implements ToXContent { + + public static ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, datafeedId) -> request.datafeedId = datafeedId, DatafeedConfig.ID); + PARSER.declareLong((request, startTime) -> request.startTime = startTime, START_TIME); + PARSER.declareLong(Request::setEndTime, END_TIME); + } + + public static Request parseRequest(String datafeedId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (datafeedId != null) { + request.datafeedId = datafeedId; + } + return request; + } + + private String datafeedId; + private long startTime; + private Long endTime; + + public Request(String datafeedId, long startTime) { + this.datafeedId = ExceptionsHelper.requireNonNull(datafeedId, DatafeedConfig.ID.getPreferredName()); + this.startTime = startTime; + } + + public Request(StreamInput in) throws IOException { + readFrom(in); + } + + Request() { + } + + public String getDatafeedId() { + return datafeedId; + } + + public long getStartTime() { + return startTime; + } + + public Long getEndTime() { + return endTime; + } + + public void setEndTime(Long endTime) { + this.endTime = endTime; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return new DatafeedTask(id, type, action, parentTaskId, datafeedId); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeedId = in.readString(); + startTime = in.readVLong(); + endTime = in.readOptionalLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(datafeedId); + out.writeVLong(startTime); + out.writeOptionalLong(endTime); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DatafeedConfig.ID.getPreferredName(), datafeedId); + builder.field(START_TIME.getPreferredName(), startTime); + if (endTime != null) { + builder.field(END_TIME.getPreferredName(), endTime); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(datafeedId, startTime, endTime); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(datafeedId, other.datafeedId) && + Objects.equals(startTime, other.startTime) && + Objects.equals(endTime, other.endTime); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, StartDatafeedAction action) { + super(client, action, new Request()); + } + } + + public static class DatafeedTask extends PersistentTask { + + private volatile DatafeedJobRunner.Holder holder; + + public DatafeedTask(long id, String type, String action, TaskId parentTaskId, String datafeedId) { + super(id, type, action, "datafeed-" + datafeedId, parentTaskId); + } + + public void setHolder(DatafeedJobRunner.Holder holder) { + this.holder = holder; + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + @Override + protected void onCancelled() { + stop(); + } + + /* public for testing */ + public void stop() { + if (holder == null) { + throw new IllegalStateException("task cancel ran before datafeed runner assigned the holder"); + } + holder.stop("cancel", null); + } + } + + public static class TransportAction extends TransportPersistentAction { + + private final DatafeedJobRunner datafeedJobRunner; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + PersistentActionService persistentActionService, PersistentActionRegistry persistentActionRegistry, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + DatafeedJobRunner datafeedJobRunner) { + super(settings, NAME, false, threadPool, transportService, persistentActionService, persistentActionRegistry, + actionFilters, indexNameExpressionResolver, Request::new, ThreadPool.Names.MANAGEMENT); + this.datafeedJobRunner = datafeedJobRunner; + } + + @Override + public void validate(Request request, ClusterState clusterState) { + MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); + StartDatafeedAction.validate(request.getDatafeedId(), mlMetadata); + PersistentTasksInProgress persistentTasksInProgress = clusterState.custom(PersistentTasksInProgress.TYPE); + if (persistentTasksInProgress == null) { + return; + } + + Predicate> predicate = taskInProgress -> { + Request storedRequest = (Request) taskInProgress.getRequest(); + return storedRequest.getDatafeedId().equals(request.getDatafeedId()); + }; + if (persistentTasksInProgress.tasksExist(NAME, predicate)) { + throw new ElasticsearchStatusException("datafeed already started, expected datafeed state [{}], but got [{}]", + RestStatus.CONFLICT, DatafeedState.STOPPED, DatafeedState.STARTED); + } + } + + @Override + protected void nodeOperation(PersistentTask task, Request request, ActionListener listener) { + DatafeedTask datafeedTask = (DatafeedTask) task; + datafeedJobRunner.run(request.getDatafeedId(), request.getStartTime(), request.getEndTime(), + datafeedTask, + (error) -> { + if (error != null) { + listener.onFailure(error); + } else { + listener.onResponse(TransportResponse.Empty.INSTANCE); + } + }); + } + + } + + public static void validate(String datafeedId, MlMetadata mlMetadata) { + DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); + if (datafeed == null) { + throw ExceptionsHelper.missingDatafeedException(datafeedId); + } + Job job = mlMetadata.getJobs().get(datafeed.getJobId()); + if (job == null) { + throw ExceptionsHelper.missingJobException(datafeed.getJobId()); + } + Allocation allocation = mlMetadata.getAllocations().get(datafeed.getJobId()); + if (allocation.getState() != JobState.OPENED) { + throw new ElasticsearchStatusException("cannot start datafeed, expected job state [{}], but got [{}]", + RestStatus.CONFLICT, JobState.OPENED, allocation.getState()); + } + DatafeedJobValidator.validate(datafeed, job); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java new file mode 100644 index 00000000000..6577505bea4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/StopDatafeedAction.java @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction; + +import java.io.IOException; +import java.util.Objects; + +public class StopDatafeedAction + extends Action { + + public static final StopDatafeedAction INSTANCE = new StopDatafeedAction(); + public static final String NAME = "cluster:admin/ml/datafeeds/stop"; + + private StopDatafeedAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public RemovePersistentTaskAction.Response newResponse() { + return new RemovePersistentTaskAction.Response(); + } + + public static class Request extends MasterNodeRequest { + + private String datafeedId; + + public Request(String jobId) { + this.datafeedId = ExceptionsHelper.requireNonNull(jobId, DatafeedConfig.ID.getPreferredName()); + } + + Request() { + } + + public String getDatafeedId() { + return datafeedId; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + datafeedId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(datafeedId); + } + + @Override + public int hashCode() { + return Objects.hash(datafeedId); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(datafeedId, other.datafeedId); + } + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, StopDatafeedAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final RemovePersistentTaskAction.TransportAction removePersistentTaskAction; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, RemovePersistentTaskAction.TransportAction removePersistentTaskAction) { + super(settings, StopDatafeedAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.removePersistentTaskAction = removePersistentTaskAction; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected RemovePersistentTaskAction.Response newResponse() { + return new RemovePersistentTaskAction.Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, + ActionListener listener) throws Exception { + String datafeedId = request.getDatafeedId(); + MlMetadata mlMetadata = state.metaData().custom(MlMetadata.TYPE); + validate(datafeedId, mlMetadata); + + PersistentTasksInProgress tasksInProgress = state.custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress != null) { + for (PersistentTaskInProgress taskInProgress : tasksInProgress.findTasks(StartDatafeedAction.NAME, p -> true)) { + StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) taskInProgress.getRequest(); + if (storedRequest.getDatafeedId().equals(datafeedId)) { + RemovePersistentTaskAction.Request cancelTasksRequest = new RemovePersistentTaskAction.Request(); + cancelTasksRequest.setTaskId(taskInProgress.getId()); + removePersistentTaskAction.execute(cancelTasksRequest, listener); + return; + } + } + } + listener.onFailure(new ElasticsearchStatusException("datafeed already stopped, expected datafeed state [{}], but got [{}]", + RestStatus.CONFLICT, DatafeedState.STARTED, DatafeedState.STOPPED)); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); + } + + } + + static void validate(String datafeedId, MlMetadata mlMetadata) { + DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); + if (datafeed == null) { + throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId)); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java new file mode 100644 index 00000000000..a5e2a4fa445 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/TransportJobTaskAction.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.TaskOperationFailure; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksRequest; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.action.support.tasks.TransportTasksAction; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Base class that redirects a request to a node where the job task is running. + */ +// TODO: Hacking around here with TransportTasksAction. Ideally we should have another base class in core that +// redirects to a single node only +public abstract class TransportJobTaskAction, + Response extends BaseTasksResponse & Writeable> extends TransportTasksAction { + + protected final JobManager jobManager; + protected final AutodetectProcessManager processManager; + private final Function jobIdFromRequest; + + TransportJobTaskAction(Settings settings, String actionName, ThreadPool threadPool, ClusterService clusterService, + TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, Supplier requestSupplier, + Supplier responseSupplier, String nodeExecutor, JobManager jobManager, + AutodetectProcessManager processManager, Function jobIdFromRequest) { + super(settings, actionName, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + requestSupplier, responseSupplier, nodeExecutor); + this.jobManager = jobManager; + this.processManager = processManager; + this.jobIdFromRequest = jobIdFromRequest; + } + + @Override + protected Response newResponse(Request request, List tasks, List taskOperationFailures, + List failedNodeExceptions) { + // no need to accumulate sub responses, since we only perform an operation on one task only + // not ideal, but throwing exceptions here works, because higher up the stack there is a try-catch block delegating to + // the actionlistener's onFailure + if (tasks.isEmpty()) { + if (taskOperationFailures.isEmpty() == false) { + throw new ElasticsearchException(taskOperationFailures.get(0).getCause()); + } else if (failedNodeExceptions.isEmpty() == false) { + throw new ElasticsearchException(failedNodeExceptions.get(0).getCause()); + } else { + // the same validation that exists in AutodetectProcessManager#processData(...) and flush(...) methods + // is required here too because if the job hasn't been opened yet then no task exist for it yet and then + // #taskOperation(...) method will not be invoked, returning an empty result to the client. + // This ensures that we return an understandable error: + String jobId = jobIdFromRequest.apply(request); + jobManager.getJobOrThrowIfUnknown(jobId); + Allocation allocation = jobManager.getJobAllocation(jobId); + if (allocation.getState() != JobState.OPENED) { + throw new ElasticsearchStatusException("job [" + jobId + "] state is [" + allocation.getState() + + "], but must be [" + JobState.OPENED + "] to perform requested action", RestStatus.CONFLICT); + } else { + throw new IllegalStateException("No errors or response"); + } + } + } else { + if (tasks.size() > 1) { + throw new IllegalStateException("Expected one node level response, but got [" + tasks.size() + "]"); + } + return tasks.get(0); + } + } + + @Override + protected boolean accumulateExceptions() { + return false; + } + + public static class JobTaskRequest> extends BaseTasksRequest { + + String jobId; + + JobTaskRequest() { + } + + JobTaskRequest(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + } + + @Override + public boolean match(Task task) { + return InternalOpenJobAction.JobTask.match(task, jobId); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobAction.java new file mode 100644 index 00000000000..11799bf960e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobAction.java @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobUpdate; + +import java.io.IOException; +import java.util.Objects; + +public class UpdateJobAction extends Action { + public static final UpdateJobAction INSTANCE = new UpdateJobAction(); + public static final String NAME = "cluster:admin/ml/job/update"; + + private UpdateJobAction() { + super(NAME); + } + + @Override + public UpdateJobAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new UpdateJobAction.RequestBuilder(client, this); + } + + @Override + public PutJobAction.Response newResponse() { + return new PutJobAction.Response(); + } + + public static class Request extends AcknowledgedRequest implements ToXContent { + + public static UpdateJobAction.Request parseRequest(String jobId, XContentParser parser) { + JobUpdate update = JobUpdate.PARSER.apply(parser, null); + return new UpdateJobAction.Request(jobId, update); + } + + private String jobId; + private JobUpdate update; + + public Request(String jobId, JobUpdate update) { + this.jobId = jobId; + this.update = update; + } + + Request() { + } + + public String getJobId() { + return jobId; + } + + public JobUpdate getJobUpdate() { + return update; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + update = new JobUpdate(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + update.writeTo(out); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + update.toXContent(builder, params); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UpdateJobAction.Request request = (UpdateJobAction.Request) o; + return Objects.equals(update, request.update); + } + + @Override + public int hashCode() { + return Objects.hash(update); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, UpdateJobAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + private final Client client; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager, Client client) { + super(settings, UpdateJobAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, UpdateJobAction.Request::new); + this.jobManager = jobManager; + this.client = client; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected PutJobAction.Response newResponse() { + return new PutJobAction.Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, + ActionListener listener) throws Exception { + if (request.getJobId().equals(Job.ALL)) { + throw new IllegalArgumentException("Job Id " + Job.ALL + " cannot be for update"); + } + + ActionListener wrappedListener = listener; + if (request.getJobUpdate().isAutodetectProcessUpdate()) { + wrappedListener = ActionListener.wrap( + response -> updateProcess(request, response, listener), + listener::onFailure); + } + + jobManager.updateJob(request.getJobId(), request.getJobUpdate(), request, wrappedListener); + } + + private void updateProcess(Request request, PutJobAction.Response updateConfigResponse, + ActionListener listener) { + + UpdateProcessAction.Request updateProcessRequest = new UpdateProcessAction.Request(request.getJobId(), + request.getJobUpdate().getModelDebugConfig(), request.getJobUpdate().getDetectorUpdates()); + client.execute(UpdateProcessAction.INSTANCE, updateProcessRequest, new ActionListener() { + @Override + public void onResponse(UpdateProcessAction.Response response) { + listener.onResponse(updateConfigResponse); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + @Override + protected ClusterBlockException checkBlock(UpdateJobAction.Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobStateAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobStateAction.java new file mode 100644 index 00000000000..96da125f42b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateJobStateAction.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +public class UpdateJobStateAction + extends Action { + + public static final UpdateJobStateAction INSTANCE = new UpdateJobStateAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/state/update"; + + private UpdateJobStateAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends AcknowledgedRequest { + + private String jobId; + private JobState state; + private String reason; + + public Request(String jobId, JobState state) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.state = ExceptionsHelper.requireNonNull(state, Allocation.STATE.getPreferredName()); + } + + Request() {} + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public JobState getState() { + return state; + } + + public void setState(JobState state) { + this.state = state; + } + + public String getReason() { + return reason; + } + + public void setReason(String reason) { + this.reason = reason; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + state = JobState.fromStream(in); + reason = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + state.writeTo(out); + out.writeOptionalString(reason); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, state); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + UpdateJobStateAction.Request other = (UpdateJobStateAction.Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(state, other.state); + } + } + + static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + RequestBuilder(ElasticsearchClient client, UpdateJobStateAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends AcknowledgedResponse { + + public Response(boolean acknowledged) { + super(acknowledged); + } + + private Response() {} + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final JobManager jobManager; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager) { + super(settings, UpdateJobStateAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + jobManager.setJobState(request, listener); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateModelSnapshotAction.java new file mode 100644 index 00000000000..56c06cce1b8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateModelSnapshotAction.java @@ -0,0 +1,305 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +public class UpdateModelSnapshotAction extends +Action { + + public static final UpdateModelSnapshotAction INSTANCE = new UpdateModelSnapshotAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/model_snapshots/update"; + + private UpdateModelSnapshotAction() { + super(NAME); + } + + @Override + public UpdateModelSnapshotAction.RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public UpdateModelSnapshotAction.Response newResponse() { + return new Response(); + } + + public static class Request extends ActionRequest implements ToXContent { + + private static final ObjectParser PARSER = new ObjectParser<>(NAME, Request::new); + + static { + PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); + PARSER.declareString((request, snapshotId) -> request.snapshotId = snapshotId, ModelSnapshot.SNAPSHOT_ID); + PARSER.declareString((request, description) -> request.description = description, ModelSnapshot.DESCRIPTION); + } + + public static Request parseRequest(String jobId, String snapshotId, XContentParser parser) { + Request request = PARSER.apply(parser, null); + if (jobId != null) { + request.jobId = jobId; + } + if (snapshotId != null) { + request.snapshotId = snapshotId; + } + return request; + } + + private String jobId; + private String snapshotId; + private String description; + + Request() { + } + + public Request(String jobId, String snapshotId, String description) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + this.snapshotId = ExceptionsHelper.requireNonNull(snapshotId, ModelSnapshot.SNAPSHOT_ID.getPreferredName()); + this.description = ExceptionsHelper.requireNonNull(description, ModelSnapshot.DESCRIPTION.getPreferredName()); + } + + public String getJobId() { + return jobId; + } + + public String getSnapshotId() { + return snapshotId; + } + + public String getDescriptionString() { + return description; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + jobId = in.readString(); + snapshotId = in.readString(); + description = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(jobId); + out.writeString(snapshotId); + out.writeString(description); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(ModelSnapshot.SNAPSHOT_ID.getPreferredName(), snapshotId); + builder.field(ModelSnapshot.DESCRIPTION.getPreferredName(), description); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, snapshotId, description); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(snapshotId, other.snapshotId) + && Objects.equals(description, other.description); + } + } + + public static class Response extends ActionResponse implements StatusToXContentObject { + + private static final ParseField ACKNOWLEDGED = new ParseField("acknowledged"); + private static final ParseField MODEL = new ParseField("model"); + + private ModelSnapshot model; + + Response() { + + } + + public Response(ModelSnapshot modelSnapshot) { + model = modelSnapshot; + } + + public ModelSnapshot getModel() { + return model; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + model = new ModelSnapshot(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + model.writeTo(out); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ACKNOWLEDGED.getPreferredName(), true); + builder.field(MODEL.getPreferredName()); + builder = model.toXContent(builder, params); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(model); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + return Objects.equals(model, other.model); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client, UpdateModelSnapshotAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final JobManager jobManager; + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); + this.jobManager = jobManager; + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + logger.debug("Received request to change model snapshot description using '" + request.getDescriptionString() + + "' for snapshot ID '" + request.getSnapshotId() + "' for job '" + request.getJobId() + "'"); + getChangeCandidates(request, changeCandidates -> { + checkForClashes(request, aVoid -> { + if (changeCandidates.size() > 1) { + logger.warn("More than one model found for [{}: {}, {}: {}] tuple.", Job.ID.getPreferredName(), request.getJobId(), + ModelSnapshot.SNAPSHOT_ID.getPreferredName(), request.getSnapshotId()); + } + ModelSnapshot modelSnapshot = changeCandidates.get(0); + modelSnapshot.setDescription(request.getDescriptionString()); + jobManager.updateModelSnapshot(modelSnapshot, b -> { + modelSnapshot.setDescription(request.getDescriptionString()); + // The quantiles can be large, and totally dominate the output - + // it's clearer to remove them + modelSnapshot.setQuantiles(null); + listener.onResponse(new Response(modelSnapshot)); + }, listener::onFailure); + }, listener::onFailure); + }, listener::onFailure); + } + + private void getChangeCandidates(Request request, Consumer> handler, Consumer errorHandler) { + getModelSnapshots(request.getJobId(), request.getSnapshotId(), null, + changeCandidates -> { + if (changeCandidates == null || changeCandidates.isEmpty()) { + errorHandler.accept(new ResourceNotFoundException( + Messages.getMessage(Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getJobId()))); + } else { + handler.accept(changeCandidates); + } + }, errorHandler); + } + + private void checkForClashes(Request request, Consumer handler, Consumer errorHandler) { + getModelSnapshots(request.getJobId(), null, request.getDescriptionString(), clashCandidates -> { + if (clashCandidates != null && !clashCandidates.isEmpty()) { + errorHandler.accept(new IllegalArgumentException(Messages.getMessage( + Messages.REST_DESCRIPTION_ALREADY_USED, request.getDescriptionString(), request.getJobId()))); + } else { + handler.accept(null); + } + }, errorHandler); + } + + private void getModelSnapshots(String jobId, String snapshotId, String description, + Consumer> handler, Consumer errorHandler) { + jobProvider.modelSnapshots(jobId, 0, 1, null, null, null, true, snapshotId, description, + page -> handler.accept(page.results()), errorHandler); + } + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateProcessAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateProcessAction.java new file mode 100644 index 00000000000..c10084afac6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/UpdateProcessAction.java @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.tasks.BaseTasksResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class UpdateProcessAction extends + Action { + + + public static final UpdateProcessAction INSTANCE = new UpdateProcessAction(); + public static final String NAME = "cluster:admin/ml/job/update/process"; + + private UpdateProcessAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + static class RequestBuilder extends ActionRequestBuilder { + + RequestBuilder(ElasticsearchClient client, UpdateProcessAction action) { + super(client, action, new Request()); + } + } + + public static class Response extends BaseTasksResponse implements StatusToXContentObject, Writeable { + + private boolean isUpdated; + + private Response() { + this.isUpdated = true; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + isUpdated = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(isUpdated); + } + + @Override + public RestStatus status() { + return RestStatus.ACCEPTED; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("updated", isUpdated); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hashCode(isUpdated); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Response other = (Response) obj; + + return this.isUpdated == other.isUpdated; + } + } + + public static class Request extends TransportJobTaskAction.JobTaskRequest { + + private ModelDebugConfig modelDebugConfig; + private List detectorUpdates; + + Request() { + } + + public Request(String jobId, ModelDebugConfig modelDebugConfig, List detectorUpdates) { + super(jobId); + this.modelDebugConfig = modelDebugConfig; + this.detectorUpdates = detectorUpdates; + } + + public ModelDebugConfig getModelDebugConfig() { + return modelDebugConfig; + } + + public List getDetectorUpdates() { + return detectorUpdates; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + modelDebugConfig = in.readOptionalWriteable(ModelDebugConfig::new); + if (in.readBoolean()) { + in.readList(JobUpdate.DetectorUpdate::new); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(modelDebugConfig); + boolean hasDetectorUpdates = detectorUpdates != null; + out.writeBoolean(hasDetectorUpdates); + if (hasDetectorUpdates) { + out.writeList(detectorUpdates); + } + } + + @Override + public int hashCode() { + return Objects.hash(getJobId(), modelDebugConfig, detectorUpdates); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + + return Objects.equals(getJobId(), other.getJobId()) && + Objects.equals(modelDebugConfig, other.modelDebugConfig) && + Objects.equals(detectorUpdates, other.detectorUpdates); + } + } + + public static class TransportAction extends TransportJobTaskAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ClusterService clusterService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + JobManager jobManager, AutodetectProcessManager processManager) { + super(settings, NAME, threadPool, clusterService, transportService, actionFilters, indexNameExpressionResolver, + Request::new, Response::new, MlPlugin.THREAD_POOL_NAME, jobManager, processManager, Request::getJobId); + } + + @Override + protected Response readTaskResponse(StreamInput in) throws IOException { + Response response = new Response(); + response.readFrom(in); + return response; + } + + @Override + protected void taskOperation(Request request, InternalOpenJobAction.JobTask task, ActionListener listener) { + threadPool.executor(MlPlugin.THREAD_POOL_NAME).execute(() -> { + try { + if (request.getModelDebugConfig() != null) { + processManager.writeUpdateModelDebugMessage(request.getJobId(), request.getModelDebugConfig()); + } + if (request.getDetectorUpdates() != null) { + for (JobUpdate.DetectorUpdate update : request.getDetectorUpdates()) { + processManager.writeUpdateDetectorRulesMessage(request.getJobId(), update.getIndex(), update.getRules()); + } + } + + listener.onResponse(new Response()); + } catch (Exception e) { + listener.onFailure(e); + } + }); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/ValidateDetectorAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/ValidateDetectorAction.java new file mode 100644 index 00000000000..a5f78a1511b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/ValidateDetectorAction.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Detector; + +import java.io.IOException; +import java.util.Objects; + +public class ValidateDetectorAction +extends Action { + + public static final ValidateDetectorAction INSTANCE = new ValidateDetectorAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/validate/detector"; + + protected ValidateDetectorAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, INSTANCE); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class RequestBuilder extends ActionRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, ValidateDetectorAction action) { + super(client, action, new Request()); + } + + } + + public static class Request extends ActionRequest implements ToXContent { + + private Detector detector; + + // NORELEASE this needs to change so the body is not directly the + // detector but and object that contains a field for the detector + public static Request parseRequest(XContentParser parser) { + Detector detector = Detector.PARSER.apply(parser, null).build(); + return new Request(detector); + } + + Request() { + this.detector = null; + } + + public Request(Detector detector) { + this.detector = detector; + } + + public Detector getDetector() { + return detector; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + detector.writeTo(out); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + detector = new Detector(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + detector.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(detector); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(detector, other.detector); + } + + } + + public static class Response extends AcknowledgedResponse { + + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends HandledTransportAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, ValidateDetectorAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + Request::new); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + listener.onResponse(new Response(true)); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/ValidateJobConfigAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/ValidateJobConfigAction.java new file mode 100644 index 00000000000..916afb35b54 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/ValidateJobConfigAction.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Objects; + +public class ValidateJobConfigAction +extends Action { + + public static final ValidateJobConfigAction INSTANCE = new ValidateJobConfigAction(); + public static final String NAME = "cluster:admin/ml/anomaly_detectors/validate"; + + protected ValidateJobConfigAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, INSTANCE); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class RequestBuilder extends ActionRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, ValidateJobConfigAction action) { + super(client, action, new Request()); + } + + } + + public static class Request extends ActionRequest implements ToXContent { + + private Job job; + + public static Request parseRequest(XContentParser parser) { + Job.Builder job = Job.PARSER.apply(parser, null); + // When jobs are PUT their ID must be supplied in the URL - assume this will + // be valid unless an invalid job ID is specified in the JSON to be validated + return new Request(job.build(true, (job.getId() != null) ? job.getId() : "ok")); + } + + Request() { + this.job = null; + } + + public Request(Job job) { + this.job = job; + } + + public Job getJob() { + return job; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + job.writeTo(out); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + job = new Job(in); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + job.toXContent(builder, params); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(job); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(job, other.job); + } + + } + + public static class Response extends AcknowledgedResponse { + + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + writeAcknowledged(out); + } + } + + public static class TransportAction extends HandledTransportAction { + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, ValidateJobConfigAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, + Request::new); + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + listener.onResponse(new Response(true)); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/util/PageParams.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/util/PageParams.java new file mode 100644 index 00000000000..102d8e8ad3c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/util/PageParams.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action.util; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class PageParams extends ToXContentToBytes implements Writeable { + + public static final ParseField PAGE = new ParseField("page"); + public static final ParseField FROM = new ParseField("from"); + public static final ParseField SIZE = new ParseField("size"); + + public static final int DEFAULT_FROM = 0; + public static final int DEFAULT_SIZE = 100; + + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + PAGE.getPreferredName(), a -> new PageParams((int) a[0], (int) a[1])); + + public static final int MAX_FROM_SIZE_SUM = 10000; + + static { + PARSER.declareInt(ConstructingObjectParser.constructorArg(), FROM); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE); + } + + private final int from; + private final int size; + + public PageParams(StreamInput in) throws IOException { + this(in.readVInt(), in.readVInt()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(from); + out.writeVInt(size); + } + + public PageParams() { + this.from = DEFAULT_FROM; + this.size = DEFAULT_SIZE; + } + + public PageParams(int from, int size) { + if (from < 0) { + throw new IllegalArgumentException("Parameter [" + FROM.getPreferredName() + "] cannot be < 0"); + } + if (size < 0) { + throw new IllegalArgumentException("Parameter [" + PageParams.SIZE.getPreferredName() + "] cannot be < 0"); + } + if (from + size > MAX_FROM_SIZE_SUM) { + throw new IllegalArgumentException("The sum of parameters [" + FROM.getPreferredName() + "] and [" + + PageParams.SIZE.getPreferredName() + "] cannot be higher than " + MAX_FROM_SIZE_SUM + "."); + } + this.from = from; + this.size = size; + } + + public int getFrom() { + return from; + } + + public int getSize() { + return size; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FROM.getPreferredName(), from); + builder.field(SIZE.getPreferredName(), size); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(from, size); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + PageParams other = (PageParams) obj; + return Objects.equals(from, other.from) && + Objects.equals(size, other.size); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/util/QueryPage.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/util/QueryPage.java new file mode 100644 index 00000000000..3a18aea828a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/action/util/QueryPage.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action.util; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Generic wrapper class for a page of query results and the total number of + * query results.
+ * {@linkplain #count()} is the total number of results but that value may + * not be equal to the actual length of the {@linkplain #results()} list if from + * & take or some cursor was used in the database query. + */ +public final class QueryPage extends ToXContentToBytes implements Writeable { + + public static final ParseField COUNT = new ParseField("count"); + public static final ParseField DEFAULT_RESULTS_FIELD = new ParseField("results_field"); + + private final ParseField resultsField; + private final List results; + private final long count; + + public QueryPage(List results, long count, ParseField resultsField) { + this.results = results; + this.count = count; + this.resultsField = ExceptionsHelper.requireNonNull(resultsField, DEFAULT_RESULTS_FIELD.getPreferredName()); + } + + public QueryPage(StreamInput in, Reader hitReader) throws IOException { + resultsField = new ParseField(in.readString()); + results = in.readList(hitReader); + count = in.readLong(); + } + + public static ResourceNotFoundException emptyQueryPage(ParseField resultsField) { + return new ResourceNotFoundException("Could not find requested " + resultsField.getPreferredName()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(resultsField.getPreferredName()); + out.writeList(results); + out.writeLong(count); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(COUNT.getPreferredName(), count); + builder.field(resultsField.getPreferredName(), results); + return builder; + } + + public List results() { + return results; + } + + public long count() { + return count; + } + + @Override + public int hashCode() { + return Objects.hash(results, count); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + @SuppressWarnings("unchecked") + QueryPage other = (QueryPage) obj; + return Objects.equals(results, other.results) && + Objects.equals(count, other.count); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/ChunkingConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/ChunkingConfig.java new file mode 100644 index 00000000000..8196c40851a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/ChunkingConfig.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +/** + * The description of how searches should be chunked. + */ +public class ChunkingConfig extends ToXContentToBytes implements Writeable { + + public static final ParseField MODE_FIELD = new ParseField("mode"); + public static final ParseField TIME_SPAN_FIELD = new ParseField("time_span"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "chunking_config", a -> new ChunkingConfig((Mode) a[0], (Long) a[1])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Mode.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, MODE_FIELD, ValueType.STRING); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), TIME_SPAN_FIELD); + } + + private final Mode mode; + private final Long timeSpan; + + public ChunkingConfig(StreamInput in) throws IOException { + mode = Mode.readFromStream(in); + timeSpan = in.readOptionalLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + mode.writeTo(out); + out.writeOptionalLong(timeSpan); + } + + ChunkingConfig(Mode mode, @Nullable Long timeSpan) { + this.mode = ExceptionsHelper.requireNonNull(mode, MODE_FIELD.getPreferredName()); + this.timeSpan = timeSpan; + if (mode == Mode.MANUAL) { + if (timeSpan == null) { + throw new IllegalArgumentException("when chunk mode is manual time_span is required"); + } + if (timeSpan <= 0) { + throw new IllegalArgumentException("chunk time_span has to be positive"); + } + } else { + if (timeSpan != null) { + throw new IllegalArgumentException("chunk time_span may only be set when mode is manual"); + } + } + } + + @Nullable + public Long getTimeSpan() { + return timeSpan; + } + + public boolean isEnabled() { + return mode != Mode.OFF; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(MODE_FIELD.getPreferredName(), mode); + if (timeSpan != null) { + builder.field(TIME_SPAN_FIELD.getPreferredName(), timeSpan); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(mode, timeSpan); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + ChunkingConfig other = (ChunkingConfig) obj; + return Objects.equals(this.mode, other.mode) && + Objects.equals(this.timeSpan, other.timeSpan); + } + + public static ChunkingConfig newAuto() { + return new ChunkingConfig(Mode.AUTO, null); + } + + public static ChunkingConfig newOff() { + return new ChunkingConfig(Mode.OFF, null); + } + + public static ChunkingConfig newManual(long timeSpan) { + return new ChunkingConfig(Mode.MANUAL, timeSpan); + } + + public enum Mode implements Writeable { + AUTO, MANUAL, OFF; + + public static Mode fromString(String value) { + return Mode.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static Mode readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Mode ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfig.java new file mode 100644 index 00000000000..addb1a89989 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfig.java @@ -0,0 +1,487 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.utils.DomainSplitFunction; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.MlStrings; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Datafeed configuration options. Describes where to proactively pull input + * data from. + *

+ * If a value has not been set it will be null. Object wrappers are + * used around integral types and booleans so they can take null + * values. + */ +public class DatafeedConfig extends AbstractDiffable implements ToXContent { + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("datafeeds"); + + /** + * The field name used to specify document counts in Elasticsearch + * aggregations + */ + public static final String DOC_COUNT = "doc_count"; + + public static final ParseField ID = new ParseField("datafeed_id"); + public static final ParseField QUERY_DELAY = new ParseField("query_delay"); + public static final ParseField FREQUENCY = new ParseField("frequency"); + public static final ParseField INDEXES = new ParseField("indexes"); + public static final ParseField TYPES = new ParseField("types"); + public static final ParseField QUERY = new ParseField("query"); + public static final ParseField SCROLL_SIZE = new ParseField("scroll_size"); + public static final ParseField AGGREGATIONS = new ParseField("aggregations"); + public static final ParseField AGGS = new ParseField("aggs"); + public static final ParseField SCRIPT_FIELDS = new ParseField("script_fields"); + public static final ParseField SOURCE = new ParseField("_source"); + public static final ParseField CHUNKING_CONFIG = new ParseField("chunking_config"); + + public static final ObjectParser PARSER = new ObjectParser<>("datafeed_config", Builder::new); + + static { + PARSER.declareString(Builder::setId, ID); + PARSER.declareString(Builder::setJobId, Job.ID); + PARSER.declareStringArray(Builder::setIndexes, INDEXES); + PARSER.declareStringArray(Builder::setTypes, TYPES); + PARSER.declareLong(Builder::setQueryDelay, QUERY_DELAY); + PARSER.declareLong(Builder::setFrequency, FREQUENCY); + PARSER.declareObject(Builder::setQuery, + (p, c) -> new QueryParseContext(p).parseInnerQueryBuilder(), QUERY); + PARSER.declareObject(Builder::setAggregations, (p, c) -> AggregatorFactories.parseAggregators(new QueryParseContext(p)), + AGGREGATIONS); + PARSER.declareObject(Builder::setAggregations,(p, c) -> AggregatorFactories.parseAggregators(new QueryParseContext(p)), AGGS); + PARSER.declareObject(Builder::setScriptFields, (p, c) -> { + List parsedScriptFields = new ArrayList<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + parsedScriptFields.add(new SearchSourceBuilder.ScriptField(new QueryParseContext(p))); + } + parsedScriptFields.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName)); + return parsedScriptFields; + }, SCRIPT_FIELDS); + PARSER.declareInt(Builder::setScrollSize, SCROLL_SIZE); + PARSER.declareBoolean(Builder::setSource, SOURCE); + PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, CHUNKING_CONFIG); + } + + private final String id; + private final String jobId; + + /** + * The delay in seconds before starting to query a period of time + */ + private final Long queryDelay; + + /** + * The frequency in seconds with which queries are executed + */ + private final Long frequency; + + private final List indexes; + private final List types; + private final QueryBuilder query; + private final AggregatorFactories.Builder aggregations; + private final List scriptFields; + private final Integer scrollSize; + private final boolean source; + private final ChunkingConfig chunkingConfig; + + private DatafeedConfig(String id, String jobId, Long queryDelay, Long frequency, List indexes, List types, + QueryBuilder query, AggregatorFactories.Builder aggregations, List scriptFields, + Integer scrollSize, boolean source, ChunkingConfig chunkingConfig) { + this.id = id; + this.jobId = jobId; + this.queryDelay = queryDelay; + this.frequency = frequency; + this.indexes = indexes; + this.types = types; + this.query = query; + this.aggregations = aggregations; + this.scriptFields = scriptFields; + this.scrollSize = scrollSize; + this.source = source; + this.chunkingConfig = chunkingConfig; + } + + public DatafeedConfig(StreamInput in) throws IOException { + this.id = in.readString(); + this.jobId = in.readString(); + this.queryDelay = in.readOptionalLong(); + this.frequency = in.readOptionalLong(); + if (in.readBoolean()) { + this.indexes = in.readList(StreamInput::readString); + } else { + this.indexes = null; + } + if (in.readBoolean()) { + this.types = in.readList(StreamInput::readString); + } else { + this.types = null; + } + this.query = in.readNamedWriteable(QueryBuilder.class); + this.aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new); + if (in.readBoolean()) { + this.scriptFields = in.readList(SearchSourceBuilder.ScriptField::new); + } else { + this.scriptFields = null; + } + this.scrollSize = in.readOptionalVInt(); + this.source = in.readBoolean(); + this.chunkingConfig = in.readOptionalWriteable(ChunkingConfig::new); + } + + public String getId() { + return id; + } + + public String getJobId() { + return jobId; + } + + public Long getQueryDelay() { + return queryDelay; + } + + public Long getFrequency() { + return frequency; + } + + /** + * For the ELASTICSEARCH data source only, one or more indexes to search for + * input data. + * + * @return The indexes to search, or null if not set. + */ + public List getIndexes() { + return indexes; + } + + /** + * For the ELASTICSEARCH data source only, one or more types to search for + * input data. + * + * @return The types to search, or null if not set. + */ + public List getTypes() { + return types; + } + + public Integer getScrollSize() { + return scrollSize; + } + + public boolean isSource() { + return source; + } + + public QueryBuilder getQuery() { + return query; + } + + public AggregatorFactories.Builder getAggregations() { + return aggregations; + } + + public List getScriptFields() { + return scriptFields == null ? Collections.emptyList() : scriptFields; + } + + public ChunkingConfig getChunkingConfig() { + return chunkingConfig; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(jobId); + out.writeOptionalLong(queryDelay); + out.writeOptionalLong(frequency); + if (indexes != null) { + out.writeBoolean(true); + out.writeStringList(indexes); + } else { + out.writeBoolean(false); + } + if (types != null) { + out.writeBoolean(true); + out.writeStringList(types); + } else { + out.writeBoolean(false); + } + out.writeNamedWriteable(query); + out.writeOptionalWriteable(aggregations); + if (scriptFields != null) { + out.writeBoolean(true); + out.writeList(scriptFields); + } else { + out.writeBoolean(false); + } + out.writeOptionalVInt(scrollSize); + out.writeBoolean(source); + out.writeOptionalWriteable(chunkingConfig); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(ID.getPreferredName(), id); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(QUERY_DELAY.getPreferredName(), queryDelay); + if (frequency != null) { + builder.field(FREQUENCY.getPreferredName(), frequency); + } + builder.field(INDEXES.getPreferredName(), indexes); + builder.field(TYPES.getPreferredName(), types); + builder.field(QUERY.getPreferredName(), query); + if (aggregations != null) { + builder.field(AGGREGATIONS.getPreferredName(), aggregations); + } + if (scriptFields != null) { + builder.startObject(SCRIPT_FIELDS.getPreferredName()); + for (SearchSourceBuilder.ScriptField scriptField : scriptFields) { + scriptField.toXContent(builder, params); + } + builder.endObject(); + } + builder.field(SCROLL_SIZE.getPreferredName(), scrollSize); + if (source) { + builder.field(SOURCE.getPreferredName(), source); + } + if (chunkingConfig != null) { + builder.field(CHUNKING_CONFIG.getPreferredName(), chunkingConfig); + } + return builder; + } + + /** + * The lists of indexes and types are compared for equality but they are not + * sorted first so this test could fail simply because the indexes and types + * lists are in different orders. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof DatafeedConfig == false) { + return false; + } + + DatafeedConfig that = (DatafeedConfig) other; + + return Objects.equals(this.id, that.id) + && Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.frequency, that.frequency) + && Objects.equals(this.queryDelay, that.queryDelay) + && Objects.equals(this.indexes, that.indexes) + && Objects.equals(this.types, that.types) + && Objects.equals(this.query, that.query) + && Objects.equals(this.scrollSize, that.scrollSize) + && Objects.equals(this.aggregations, that.aggregations) + && Objects.equals(this.scriptFields, that.scriptFields) + && Objects.equals(this.source, that.source) + && Objects.equals(this.chunkingConfig, that.chunkingConfig); + } + + @Override + public int hashCode() { + return Objects.hash(id, jobId, frequency, queryDelay, indexes, types, query, scrollSize, aggregations, scriptFields, source, + chunkingConfig); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public static class Builder { + + private static final int DEFAULT_SCROLL_SIZE = 1000; + private static final long DEFAULT_ELASTICSEARCH_QUERY_DELAY = 60L; + + private String id; + private String jobId; + private Long queryDelay = DEFAULT_ELASTICSEARCH_QUERY_DELAY; + private Long frequency; + private List indexes = Collections.emptyList(); + private List types = Collections.emptyList(); + private QueryBuilder query = QueryBuilders.matchAllQuery(); + private AggregatorFactories.Builder aggregations; + private List scriptFields; + private Integer scrollSize = DEFAULT_SCROLL_SIZE; + private boolean source = false; + private ChunkingConfig chunkingConfig; + + public Builder() { + } + + public Builder(String id, String jobId) { + this(); + this.id = ExceptionsHelper.requireNonNull(id, ID.getPreferredName()); + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public Builder(DatafeedConfig config) { + this.id = config.id; + this.jobId = config.jobId; + this.queryDelay = config.queryDelay; + this.frequency = config.frequency; + this.indexes = config.indexes; + this.types = config.types; + this.query = config.query; + this.aggregations = config.aggregations; + this.scriptFields = config.scriptFields; + this.scrollSize = config.scrollSize; + this.source = config.source; + this.chunkingConfig = config.chunkingConfig; + } + + public void setId(String datafeedId) { + id = ExceptionsHelper.requireNonNull(datafeedId, ID.getPreferredName()); + } + + public void setJobId(String jobId) { + this.jobId = ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + } + + public void setIndexes(List indexes) { + this.indexes = ExceptionsHelper.requireNonNull(indexes, INDEXES.getPreferredName()); + } + + public void setTypes(List types) { + this.types = ExceptionsHelper.requireNonNull(types, TYPES.getPreferredName()); + } + + public void setQueryDelay(long queryDelay) { + if (queryDelay < 0) { + String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, + DatafeedConfig.QUERY_DELAY.getPreferredName(), queryDelay); + throw new IllegalArgumentException(msg); + } + this.queryDelay = queryDelay; + } + + public void setFrequency(long frequency) { + if (frequency <= 0) { + String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, + DatafeedConfig.FREQUENCY.getPreferredName(), frequency); + throw new IllegalArgumentException(msg); + } + this.frequency = frequency; + } + + public void setQuery(QueryBuilder query) { + this.query = ExceptionsHelper.requireNonNull(query, QUERY.getPreferredName()); + } + + public void setAggregations(AggregatorFactories.Builder aggregations) { + this.aggregations = aggregations; + } + + public void setScriptFields(List scriptFields) { + List sorted = new ArrayList<>(); + for (SearchSourceBuilder.ScriptField scriptField : scriptFields) { + String script = scriptField.script().getIdOrCode(); + + if (script.contains("domainSplit(")) { + String modifiedCode = DomainSplitFunction.function + "\n" + script; + Map modifiedParams = new HashMap<>(scriptField.script().getParams().size() + + DomainSplitFunction.params.size()); + + modifiedParams.putAll(scriptField.script().getParams()); + modifiedParams.putAll(DomainSplitFunction.params); + + Script newScript = new Script(scriptField.script().getType(), scriptField.script().getLang(), + modifiedCode, modifiedParams); + + sorted.add(new SearchSourceBuilder.ScriptField(scriptField.fieldName(), newScript, scriptField.ignoreFailure())); + } else { + sorted.add(scriptField); + } + + } + sorted.sort(Comparator.comparing(SearchSourceBuilder.ScriptField::fieldName)); + this.scriptFields = sorted; + } + + public void setScrollSize(int scrollSize) { + if (scrollSize < 0) { + String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, + DatafeedConfig.SCROLL_SIZE.getPreferredName(), scrollSize); + throw new IllegalArgumentException(msg); + } + this.scrollSize = scrollSize; + } + + public void setSource(boolean enabled) { + this.source = enabled; + } + + public void setChunkingConfig(ChunkingConfig chunkingConfig) { + this.chunkingConfig = chunkingConfig; + } + + public DatafeedConfig build() { + ExceptionsHelper.requireNonNull(id, ID.getPreferredName()); + ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); + if (!MlStrings.isValidId(id)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_ID, ID.getPreferredName())); + } + if (indexes == null || indexes.isEmpty() || indexes.contains(null) || indexes.contains("")) { + throw invalidOptionValue(INDEXES.getPreferredName(), indexes); + } + if (types == null || types.isEmpty() || types.contains(null) || types.contains("")) { + throw invalidOptionValue(TYPES.getPreferredName(), types); + } + if (aggregations != null && (scriptFields != null && !scriptFields.isEmpty())) { + throw new IllegalArgumentException(Messages.getMessage(Messages.DATAFEED_CONFIG_CANNOT_USE_SCRIPT_FIELDS_WITH_AGGS)); + } + return new DatafeedConfig(id, jobId, queryDelay, frequency, indexes, types, query, aggregations, scriptFields, scrollSize, + source, chunkingConfig); + } + + private static ElasticsearchException invalidOptionValue(String fieldName, Object value) { + String msg = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, fieldName, value); + throw new IllegalArgumentException(msg); + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java new file mode 100644 index 00000000000..42fad786a23 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJob.java @@ -0,0 +1,236 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.xpack.ml.action.FlushJobAction; +import org.elasticsearch.xpack.ml.action.PostDataAction; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; + +class DatafeedJob { + + private static final Logger LOGGER = Loggers.getLogger(DatafeedJob.class); + private static final int NEXT_TASK_DELAY_MS = 100; + + private final Auditor auditor; + private final String jobId; + private final DataDescription dataDescription; + private final long frequencyMs; + private final long queryDelayMs; + private final Client client; + private final DataExtractorFactory dataExtractorFactory; + private final Supplier currentTimeSupplier; + + private volatile long lookbackStartTimeMs; + private volatile Long lastEndTimeMs; + private AtomicBoolean running = new AtomicBoolean(true); + + DatafeedJob(String jobId, DataDescription dataDescription, long frequencyMs, long queryDelayMs, + DataExtractorFactory dataExtractorFactory, Client client, Auditor auditor, Supplier currentTimeSupplier, + long latestFinalBucketEndTimeMs, long latestRecordTimeMs) { + this.jobId = jobId; + this.dataDescription = Objects.requireNonNull(dataDescription); + this.frequencyMs = frequencyMs; + this.queryDelayMs = queryDelayMs; + this.dataExtractorFactory = dataExtractorFactory; + this.client = client; + this.auditor = auditor; + this.currentTimeSupplier = currentTimeSupplier; + + long lastEndTime = Math.max(latestFinalBucketEndTimeMs, latestRecordTimeMs); + if (lastEndTime > 0) { + lastEndTimeMs = lastEndTime; + } + } + + Long runLookBack(long startTime, Long endTime) throws Exception { + lookbackStartTimeMs = (lastEndTimeMs != null && lastEndTimeMs + 1 > startTime) ? lastEndTimeMs + 1 : startTime; + Optional endMs = Optional.ofNullable(endTime); + long lookbackEnd = endMs.orElse(currentTimeSupplier.get() - queryDelayMs); + boolean isLookbackOnly = endMs.isPresent(); + if (lookbackEnd <= lookbackStartTimeMs) { + if (isLookbackOnly) { + return null; + } else { + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STARTED_REALTIME)); + return nextRealtimeTimestamp(); + } + } + + String msg = Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STARTED_FROM_TO, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackStartTimeMs), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(lookbackEnd)); + auditor.info(msg); + + FlushJobAction.Request request = new FlushJobAction.Request(jobId); + request.setCalcInterim(true); + run(lookbackStartTimeMs, lookbackEnd, request); + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_LOOKBACK_COMPLETED)); + LOGGER.info("[{}] Lookback has finished", jobId); + if (isLookbackOnly) { + return null; + } else { + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_CONTINUED_REALTIME)); + return nextRealtimeTimestamp(); + } + } + + long runRealtime() throws Exception { + long start = lastEndTimeMs == null ? lookbackStartTimeMs : lastEndTimeMs + 1; + long nowMinusQueryDelay = currentTimeSupplier.get() - queryDelayMs; + long end = toIntervalStartEpochMs(nowMinusQueryDelay); + FlushJobAction.Request request = new FlushJobAction.Request(jobId); + request.setCalcInterim(true); + request.setAdvanceTime(String.valueOf(lastEndTimeMs)); + run(start, end, request); + return nextRealtimeTimestamp(); + } + + /** + * Stops the datafeed job + * + * @return true when the datafeed was running and this method invocation stopped it, + * otherwise false is returned + */ + public boolean stop() { + if (running.compareAndSet(true, false)) { + auditor.info(Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_STOPPED)); + return true; + } else { + return false; + } + } + + public boolean isRunning() { + return running.get(); + } + + private void run(long start, long end, FlushJobAction.Request flushRequest) throws IOException { + if (end <= start) { + return; + } + + LOGGER.trace("[{}] Searching data in: [{}, {})", jobId, start, end); + + RuntimeException error = null; + long recordCount = 0; + DataExtractor dataExtractor = dataExtractorFactory.newExtractor(start, end); + while (dataExtractor.hasNext()) { + if (!isRunning() && !dataExtractor.isCancelled()) { + dataExtractor.cancel(); + } + + Optional extractedData; + try { + extractedData = dataExtractor.next(); + } catch (Exception e) { + error = new ExtractionProblemException(e); + break; + } + if (extractedData.isPresent()) { + DataCounts counts; + try (InputStream in = extractedData.get()) { + counts = postData(in); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + error = new AnalysisProblemException(e); + break; + } + recordCount += counts.getProcessedRecordCount(); + if (counts.getLatestRecordTimeStamp() != null) { + lastEndTimeMs = counts.getLatestRecordTimeStamp().getTime(); + } + } + } + + lastEndTimeMs = Math.max(lastEndTimeMs == null ? 0 : lastEndTimeMs, end - 1); + + // Ensure time is always advanced in order to avoid importing duplicate data. + // This is the reason we store the error rather than throw inline. + if (error != null) { + throw error; + } + + if (recordCount == 0) { + throw new EmptyDataCountException(); + } + + try { + client.execute(FlushJobAction.INSTANCE, flushRequest).get(); + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + throw new RuntimeException(e); + } + } + + private DataCounts postData(InputStream inputStream) throws IOException, ExecutionException, InterruptedException { + PostDataAction.Request request = new PostDataAction.Request(jobId); + request.setDataDescription(dataDescription); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + Streams.copy(inputStream, outputStream); + request.setContent(new BytesArray(outputStream.toByteArray())); + PostDataAction.Response response = client.execute(PostDataAction.INSTANCE, request).get(); + return response.getDataCounts(); + } + + private long nextRealtimeTimestamp() { + long epochMs = currentTimeSupplier.get() + frequencyMs; + return toIntervalStartEpochMs(epochMs) + NEXT_TASK_DELAY_MS; + } + + private long toIntervalStartEpochMs(long epochMs) { + return (epochMs / frequencyMs) * frequencyMs; + } + + class AnalysisProblemException extends RuntimeException { + + final long nextDelayInMsSinceEpoch = nextRealtimeTimestamp(); + + AnalysisProblemException(Throwable cause) { + super(cause); + } + } + + class ExtractionProblemException extends RuntimeException { + + final long nextDelayInMsSinceEpoch = nextRealtimeTimestamp(); + + ExtractionProblemException(Throwable cause) { + super(cause); + } + } + + class EmptyDataCountException extends RuntimeException { + + final long nextDelayInMsSinceEpoch = nextRealtimeTimestamp(); + + EmptyDataCountException() {} + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java new file mode 100644 index 00000000000..06e903a39b6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunner.java @@ -0,0 +1,248 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.FutureUtils; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationDataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.chunked.ChunkedDataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.scroll.ScrollDataExtractorFactory; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.DefaultFrequency; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.notifications.Auditor; + +import java.time.Duration; +import java.util.Collections; +import java.util.Objects; +import java.util.concurrent.Future; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Supplier; + +public class DatafeedJobRunner extends AbstractComponent { + + private static final String INF_SYMBOL = "\u221E"; + + private final Client client; + private final ClusterService clusterService; + private final JobProvider jobProvider; + private final ThreadPool threadPool; + private final Supplier currentTimeSupplier; + + public DatafeedJobRunner(ThreadPool threadPool, Client client, ClusterService clusterService, JobProvider jobProvider, + Supplier currentTimeSupplier) { + super(Settings.EMPTY); + this.client = Objects.requireNonNull(client); + this.clusterService = Objects.requireNonNull(clusterService); + this.jobProvider = Objects.requireNonNull(jobProvider); + this.threadPool = threadPool; + this.currentTimeSupplier = Objects.requireNonNull(currentTimeSupplier); + } + + public void run(String datafeedId, long startTime, Long endTime, StartDatafeedAction.DatafeedTask task, + Consumer handler) { + MlMetadata mlMetadata = clusterService.state().metaData().custom(MlMetadata.TYPE); + DatafeedConfig datafeed = mlMetadata.getDatafeed(datafeedId); + Job job = mlMetadata.getJobs().get(datafeed.getJobId()); + gatherInformation(job.getId(), (buckets, dataCounts) -> { + long latestFinalBucketEndMs = -1L; + Duration bucketSpan = Duration.ofSeconds(job.getAnalysisConfig().getBucketSpan()); + if (buckets.results().size() == 1) { + latestFinalBucketEndMs = buckets.results().get(0).getTimestamp().getTime() + bucketSpan.toMillis() - 1; + } + long latestRecordTimeMs = -1L; + if (dataCounts.getLatestRecordTimeStamp() != null) { + latestRecordTimeMs = dataCounts.getLatestRecordTimeStamp().getTime(); + } + Holder holder = createJobDatafeed(datafeed, job, latestFinalBucketEndMs, latestRecordTimeMs, handler, task); + innerRun(holder, startTime, endTime); + }, handler); + } + + // Important: Holder must be created and assigned to DatafeedTask before setting state to started, + // otherwise if a stop datafeed call is made immediately after the start datafeed call we could cancel + // the DatafeedTask without stopping datafeed, which causes the datafeed to keep on running. + private void innerRun(Holder holder, long startTime, Long endTime) { + logger.info("Starting datafeed [{}] for job [{}] in [{}, {})", holder.datafeed.getId(), holder.datafeed.getJobId(), + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(startTime), + endTime == null ? INF_SYMBOL : DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.printer().print(endTime)); + holder.future = threadPool.executor(MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME).submit(() -> { + Long next = null; + try { + next = holder.datafeedJob.runLookBack(startTime, endTime); + } catch (DatafeedJob.ExtractionProblemException e) { + if (endTime == null) { + next = e.nextDelayInMsSinceEpoch; + } + holder.problemTracker.reportExtractionProblem(e.getCause().getMessage()); + } catch (DatafeedJob.AnalysisProblemException e) { + if (endTime == null) { + next = e.nextDelayInMsSinceEpoch; + } + holder.problemTracker.reportAnalysisProblem(e.getCause().getMessage()); + } catch (DatafeedJob.EmptyDataCountException e) { + if (endTime == null && holder.problemTracker.updateEmptyDataCount(true) == false) { + next = e.nextDelayInMsSinceEpoch; + } + } catch (Exception e) { + logger.error("Failed lookback import for job [" + holder.datafeed.getJobId() + "]", e); + holder.stop("general_lookback_failure", e); + return; + } + if (next != null) { + doDatafeedRealtime(next, holder.datafeed.getJobId(), holder); + } else { + holder.stop("no_realtime", null); + holder.problemTracker.finishReport(); + } + }); + } + + private void doDatafeedRealtime(long delayInMsSinceEpoch, String jobId, Holder holder) { + if (holder.isRunning()) { + TimeValue delay = computeNextDelay(delayInMsSinceEpoch); + logger.debug("Waiting [{}] before executing next realtime import for job [{}]", delay, jobId); + holder.future = threadPool.schedule(delay, MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME, () -> { + long nextDelayInMsSinceEpoch; + try { + nextDelayInMsSinceEpoch = holder.datafeedJob.runRealtime(); + } catch (DatafeedJob.ExtractionProblemException e) { + nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch; + holder.problemTracker.reportExtractionProblem(e.getCause().getMessage()); + } catch (DatafeedJob.AnalysisProblemException e) { + nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch; + holder.problemTracker.reportAnalysisProblem(e.getCause().getMessage()); + } catch (DatafeedJob.EmptyDataCountException e) { + nextDelayInMsSinceEpoch = e.nextDelayInMsSinceEpoch; + if (holder.problemTracker.updateEmptyDataCount(true)) { + holder.problemTracker.finishReport(); + holder.stop("empty_data", e); + return; + } + } catch (Exception e) { + logger.error("Unexpected datafeed failure for job [" + jobId + "] stopping...", e); + holder.stop("general_realtime_error", e); + return; + } + holder.problemTracker.finishReport(); + doDatafeedRealtime(nextDelayInMsSinceEpoch, jobId, holder); + }); + } + } + + private Holder createJobDatafeed(DatafeedConfig datafeed, Job job, long finalBucketEndMs, long latestRecordTimeMs, + Consumer handler, StartDatafeedAction.DatafeedTask task) { + Auditor auditor = jobProvider.audit(job.getId()); + Duration frequency = getFrequencyOrDefault(datafeed, job); + Duration queryDelay = Duration.ofSeconds(datafeed.getQueryDelay()); + DataExtractorFactory dataExtractorFactory = createDataExtractorFactory(datafeed, job); + DatafeedJob datafeedJob = new DatafeedJob(job.getId(), buildDataDescription(job), frequency.toMillis(), queryDelay.toMillis(), + dataExtractorFactory, client, auditor, currentTimeSupplier, finalBucketEndMs, latestRecordTimeMs); + Holder holder = new Holder(datafeed, datafeedJob, new ProblemTracker(() -> auditor), handler); + task.setHolder(holder); + return holder; + } + + DataExtractorFactory createDataExtractorFactory(DatafeedConfig datafeedConfig, Job job) { + boolean isScrollSearch = datafeedConfig.getAggregations() == null; + DataExtractorFactory dataExtractorFactory = isScrollSearch ? new ScrollDataExtractorFactory(client, datafeedConfig, job) + : new AggregationDataExtractorFactory(client, datafeedConfig, job); + ChunkingConfig chunkingConfig = datafeedConfig.getChunkingConfig(); + if (chunkingConfig == null) { + chunkingConfig = isScrollSearch ? ChunkingConfig.newAuto() : ChunkingConfig.newOff(); + } + + return chunkingConfig.isEnabled() ? new ChunkedDataExtractorFactory(client, datafeedConfig, job, dataExtractorFactory) + : dataExtractorFactory; + } + + private static DataDescription buildDataDescription(Job job) { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.JSON); + if (job.getDataDescription() != null) { + dataDescription.setTimeField(job.getDataDescription().getTimeField()); + } + dataDescription.setTimeFormat(DataDescription.EPOCH_MS); + return dataDescription.build(); + } + + private void gatherInformation(String jobId, BiConsumer, DataCounts> handler, Consumer errorHandler) { + BucketsQueryBuilder.BucketsQuery latestBucketQuery = new BucketsQueryBuilder() + .sortField(Bucket.TIMESTAMP.getPreferredName()) + .sortDescending(true).size(1) + .includeInterim(false) + .build(); + jobProvider.buckets(jobId, latestBucketQuery, buckets -> { + jobProvider.dataCounts(jobId, dataCounts -> handler.accept(buckets, dataCounts), errorHandler); + }, e -> { + if (e instanceof ResourceNotFoundException) { + QueryPage empty = new QueryPage<>(Collections.emptyList(), 0, Bucket.RESULT_TYPE_FIELD); + jobProvider.dataCounts(jobId, dataCounts -> handler.accept(empty, dataCounts), errorHandler); + } else { + errorHandler.accept(e); + } + }); + } + + private static Duration getFrequencyOrDefault(DatafeedConfig datafeed, Job job) { + Long frequency = datafeed.getFrequency(); + Long bucketSpan = job.getAnalysisConfig().getBucketSpan(); + return frequency == null ? DefaultFrequency.ofBucketSpan(bucketSpan) : Duration.ofSeconds(frequency); + } + + private TimeValue computeNextDelay(long next) { + return new TimeValue(Math.max(1, next - currentTimeSupplier.get())); + } + + public class Holder { + + private final DatafeedConfig datafeed; + private final DatafeedJob datafeedJob; + private final ProblemTracker problemTracker; + private final Consumer handler; + volatile Future future; + + private Holder(DatafeedConfig datafeed, DatafeedJob datafeedJob, ProblemTracker problemTracker, Consumer handler) { + this.datafeed = datafeed; + this.datafeedJob = datafeedJob; + this.problemTracker = problemTracker; + this.handler = handler; + } + + boolean isRunning() { + return datafeedJob.isRunning(); + } + + public void stop(String source, Exception e) { + logger.info("[{}] attempt to stop datafeed [{}] for job [{}]", source, datafeed.getId(), datafeed.getJobId()); + if (datafeedJob.stop()) { + FutureUtils.cancel(future); + handler.accept(e); + logger.info("[{}] datafeed [{}] for job [{}] has been stopped", source, datafeed.getId(), datafeed.getJobId()); + } else { + logger.info("[{}] datafeed [{}] for job [{}] was already stopped", source, datafeed.getId(), datafeed.getJobId()); + } + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidator.java new file mode 100644 index 00000000000..93fc633ffa9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidator.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +public final class DatafeedJobValidator { + + private DatafeedJobValidator() {} + + /** + * Validates a datafeedConfig in relation to the job it refers to + * @param datafeedConfig the datafeed config + * @param job the job + */ + public static void validate(DatafeedConfig datafeedConfig, Job job) { + AnalysisConfig analysisConfig = job.getAnalysisConfig(); + if (analysisConfig.getLatency() != null && analysisConfig.getLatency() > 0) { + throw new IllegalArgumentException(Messages.getMessage(Messages.DATAFEED_DOES_NOT_SUPPORT_JOB_WITH_LATENCY)); + } + if (datafeedConfig.getAggregations() != null && !DatafeedConfig.DOC_COUNT.equals(analysisConfig.getSummaryCountFieldName())) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.DATAFEED_AGGREGATIONS_REQUIRES_JOB_WITH_SUMMARY_COUNT_FIELD, DatafeedConfig.DOC_COUNT)); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedState.java new file mode 100644 index 00000000000..75bb3346e81 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedState.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum DatafeedState implements Writeable { + + STARTED, STOPPED; + + public static DatafeedState fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static DatafeedState fromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown public enum DatafeedState ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java new file mode 100644 index 00000000000..a2072d3f15f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/ProblemTracker.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + *

+ * Keeps track of problems the datafeed encounters and audits + * messages appropriately. + *

+ *

+ * The {@code ProblemTracker} is expected to interact with multiple + * threads (lookback executor, real-time executor). However, each + * thread will be accessing in a sequential manner therefore we + * only need to ensure correct visibility. + *

+ */ +class ProblemTracker { + + private static final int EMPTY_DATA_WARN_COUNT = 10; + + private final Supplier auditor; + + private volatile boolean hasProblems; + private volatile boolean hadProblems; + private volatile String previousProblem; + + private volatile int emptyDataCount; + + ProblemTracker(Supplier auditor) { + this.auditor = Objects.requireNonNull(auditor); + } + + /** + * Reports as analysis problem if it is different than the last seen problem + * + * @param problemMessage the problem message + */ + public void reportAnalysisProblem(String problemMessage) { + reportProblem(Messages.JOB_AUDIT_DATAFEED_DATA_ANALYSIS_ERROR, problemMessage); + } + + /** + * Reports as extraction problem if it is different than the last seen problem + * + * @param problemMessage the problem message + */ + public void reportExtractionProblem(String problemMessage) { + reportProblem(Messages.JOB_AUDIT_DATAFEED_DATA_EXTRACTION_ERROR, problemMessage); + } + + /** + * Reports the problem if it is different than the last seen problem + * + * @param problemMessage the problem message + */ + private void reportProblem(String template, String problemMessage) { + hasProblems = true; + if (!Objects.equals(previousProblem, problemMessage)) { + previousProblem = problemMessage; + auditor.get().error(Messages.getMessage(template, problemMessage)); + } + } + + /** + * Updates the tracking of empty data cycles. If the number of consecutive empty data + * cycles reaches {@code EMPTY_DATA_WARN_COUNT}, a warning is reported. If non-empty + * is reported and a warning was issued previously, a recovery info is reported. + * + * @param empty Whether data was seen since last report + * @return {@code true} if an empty data warning was issued, {@code false} otherwise + */ + public boolean updateEmptyDataCount(boolean empty) { + if (empty && emptyDataCount < EMPTY_DATA_WARN_COUNT) { + emptyDataCount++; + if (emptyDataCount == EMPTY_DATA_WARN_COUNT) { + auditor.get().warning(Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_NO_DATA)); + return true; + } + } else if (!empty) { + if (emptyDataCount >= EMPTY_DATA_WARN_COUNT) { + auditor.get().info(Messages.getMessage(Messages.JOB_AUDIR_DATAFEED_DATA_SEEN_AGAIN)); + } + emptyDataCount = 0; + } + return false; + } + + public boolean hasProblems() { + return hasProblems; + } + + /** + * Issues a recovery message if appropriate and prepares for next report + */ + public void finishReport() { + if (!hasProblems && hadProblems) { + auditor.get().info(Messages.getMessage(Messages.JOB_AUDIT_DATAFEED_RECOVERED)); + } + + hadProblems = hasProblems; + hasProblems = false; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java new file mode 100644 index 00000000000..31cd7fbd500 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractor.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +public interface DataExtractor { + + /** + * @return {@code true} if the search has not finished yet, or {@code false} otherwise + */ + boolean hasNext(); + + /** + * Returns the next available extracted data. Note that it is possible for the + * extracted data to be empty the last time this method can be called. + * @return an optional input stream with the next available extracted data + * @throws IOException if an error occurs while extracting the data + */ + Optional next() throws IOException; + + /** + * @return {@code true} if the extractor has been cancelled, or {@code false} otherwise + */ + boolean isCancelled(); + + /** + * Cancel the current search. + */ + void cancel(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java new file mode 100644 index 00000000000..6db53f812ab --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor; + +public interface DataExtractorFactory { + DataExtractor newExtractor(long start, long end); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/ExtractorUtils.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/ExtractorUtils.java new file mode 100644 index 00000000000..3634f636850 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/ExtractorUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.Arrays; + +/** + * Collects common utility methods needed by various {@link DataExtractor} implementations + */ +public final class ExtractorUtils { + + private static final Logger LOGGER = Loggers.getLogger(ExtractorUtils.class); + private static final String EPOCH_MILLIS = "epoch_millis"; + + private ExtractorUtils() {} + + /** + * Combines a user query with a time range query. + */ + public static QueryBuilder wrapInTimeRangeQuery(QueryBuilder userQuery, String timeField, long start, long end) { + QueryBuilder timeQuery = new RangeQueryBuilder(timeField).gte(start).lt(end).format(EPOCH_MILLIS); + return new BoolQueryBuilder().filter(userQuery).filter(timeQuery); + } + + /** + * Checks that a {@link SearchResponse} has an OK status code and no shard failures + */ + public static void checkSearchWasSuccessful(String jobId, SearchResponse searchResponse) throws IOException { + if (searchResponse.status() != RestStatus.OK) { + throw new IOException("[" + jobId + "] Search request returned status code: " + searchResponse.status() + + ". Response was:\n" + searchResponse.toString()); + } + ShardSearchFailure[] shardFailures = searchResponse.getShardFailures(); + if (shardFailures != null && shardFailures.length > 0) { + LOGGER.error("[{}] Search request returned shard failures: {}", jobId, Arrays.toString(shardFailures)); + throw new IOException("[" + jobId + "] Search request returned shard failures; see more info in the logs"); + } + int unavailableShards = searchResponse.getTotalShards() - searchResponse.getSuccessfulShards(); + if (unavailableShards > 0) { + throw new IOException("[" + jobId + "] Search request encountered [" + unavailableShards + "] unavailable shards"); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java new file mode 100644 index 00000000000..0dd0a28ea23 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.ExtractorUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; + +/** + * An implementation that extracts data from elasticsearch using search with aggregations on a client. + * Cancellation is effective only when it is called before the first time {@link #next()} is called. + * Note that this class is NOT thread-safe. + */ +class AggregationDataExtractor implements DataExtractor { + + private static final Logger LOGGER = Loggers.getLogger(AggregationDataExtractor.class); + + private final Client client; + private final AggregationDataExtractorContext context; + private boolean hasNext; + private boolean isCancelled; + + AggregationDataExtractor(Client client, AggregationDataExtractorContext dataExtractorContext) { + this.client = Objects.requireNonNull(client); + this.context = Objects.requireNonNull(dataExtractorContext); + this.hasNext = true; + } + + @Override + public boolean hasNext() { + return hasNext && !isCancelled; + } + + @Override + public boolean isCancelled() { + return isCancelled; + } + + @Override + public void cancel() { + LOGGER.trace("[{}] Data extractor received cancel request", context.jobId); + isCancelled = true; + } + + @Override + public Optional next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Optional stream = Optional.ofNullable(search()); + hasNext = false; + return stream; + } + + private InputStream search() throws IOException { + LOGGER.debug("[{}] Executing aggregated search", context.jobId); + SearchResponse searchResponse = executeSearchRequest(buildSearchRequest()); + ExtractorUtils.checkSearchWasSuccessful(context.jobId, searchResponse); + return processSearchResponse(searchResponse); + } + + protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { + return searchRequestBuilder.get(); + } + + private SearchRequestBuilder buildSearchRequest() { + SearchRequestBuilder searchRequestBuilder = SearchAction.INSTANCE.newRequestBuilder(client) + .setIndices(context.indexes) + .setTypes(context.types) + .setSize(0) + .setQuery(ExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, context.start, context.end)); + + context.aggs.getAggregatorFactories().forEach(a -> searchRequestBuilder.addAggregation(a)); + context.aggs.getPipelineAggregatorFactories().forEach(a -> searchRequestBuilder.addAggregation(a)); + return searchRequestBuilder; + } + + private InputStream processSearchResponse(SearchResponse searchResponse) throws IOException { + if (searchResponse.getAggregations() == null) { + return null; + } + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (AggregationToJsonProcessor processor = new AggregationToJsonProcessor(outputStream)) { + for (Aggregation agg : searchResponse.getAggregations().asList()) { + processor.process(agg); + } + } + return new ByteArrayInputStream(outputStream.toByteArray()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java new file mode 100644 index 00000000000..860ce25b690 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; + +import java.util.List; +import java.util.Objects; + +class AggregationDataExtractorContext { + + final String jobId; + final String timeField; + final String[] indexes; + final String[] types; + final QueryBuilder query; + final AggregatorFactories.Builder aggs; + final long start; + final long end; + + AggregationDataExtractorContext(String jobId, String timeField, List indexes, List types, QueryBuilder query, + AggregatorFactories.Builder aggs, long start, long end) { + this.jobId = Objects.requireNonNull(jobId); + this.timeField = Objects.requireNonNull(timeField); + this.indexes = indexes.toArray(new String[indexes.size()]); + this.types = types.toArray(new String[types.size()]); + this.query = Objects.requireNonNull(query); + this.aggs = Objects.requireNonNull(aggs); + this.start = start; + this.end = end; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java new file mode 100644 index 00000000000..757ee0503c9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.elasticsearch.client.Client; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.util.Objects; + +public class AggregationDataExtractorFactory implements DataExtractorFactory { + + private final Client client; + private final DatafeedConfig datafeedConfig; + private final Job job; + + public AggregationDataExtractorFactory(Client client, DatafeedConfig datafeedConfig, Job job) { + this.client = Objects.requireNonNull(client); + this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.job = Objects.requireNonNull(job); + } + + @Override + public DataExtractor newExtractor(long start, long end) { + AggregationDataExtractorContext dataExtractorContext = new AggregationDataExtractorContext( + job.getId(), + job.getDataDescription().getTimeField(), + datafeedConfig.getIndexes(), + datafeedConfig.getTypes(), + datafeedConfig.getQuery(), + datafeedConfig.getAggregations(), + start, + end); + return new AggregationDataExtractor(client, dataExtractorContext); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java new file mode 100644 index 00000000000..e8ff3cf2316 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessor.java @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.joda.time.base.BaseDateTime; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Processes {@link Aggregation} objects and writes flat JSON documents for each leaf aggregation. + */ +class AggregationToJsonProcessor implements Releasable { + + private final XContentBuilder jsonBuilder; + private final Map keyValuePairs; + + AggregationToJsonProcessor(OutputStream outputStream) throws IOException { + jsonBuilder = new XContentBuilder(JsonXContent.jsonXContent, outputStream); + keyValuePairs = new LinkedHashMap<>(); + } + + /** + * Processes an {@link Aggregation} and writes a flat JSON document for each of its leaf aggregations. + * It expects aggregations to have 0..1 sub-aggregations. + * It expects the top level aggregation to be {@link Histogram}. + * It expects that all sub-aggregations of the top level are either {@link Terms} or {@link NumericMetricsAggregation.SingleValue}. + */ + public void process(Aggregation aggregation) throws IOException { + if (aggregation instanceof Histogram) { + processHistogram((Histogram) aggregation); + } else { + throw new IllegalArgumentException("Top level aggregation should be [histogram]"); + } + } + + private void processHistogram(Histogram histogram) throws IOException { + for (Histogram.Bucket bucket : histogram.getBuckets()) { + Object timestamp = bucket.getKey(); + if (timestamp instanceof BaseDateTime) { + timestamp = ((BaseDateTime) timestamp).getMillis(); + } + keyValuePairs.put(histogram.getName(), timestamp); + processNestedAggs(bucket.getDocCount(), bucket.getAggregations()); + } + } + + private void processNestedAggs(long docCount, Aggregations subAggs) throws IOException { + List aggs = subAggs == null ? Collections.emptyList() : subAggs.asList(); + if (aggs.isEmpty()) { + writeJsonObject(docCount); + return; + } + if (aggs.size() > 1) { + throw new IllegalArgumentException("Multiple nested aggregations are not supported"); + } + Aggregation nestedAgg = aggs.get(0); + if (nestedAgg instanceof Terms) { + processTerms((Terms) nestedAgg); + } else if (nestedAgg instanceof NumericMetricsAggregation.SingleValue) { + processSingleValue(docCount, (NumericMetricsAggregation.SingleValue) nestedAgg); + } else { + throw new IllegalArgumentException("Unsupported aggregation type [" + nestedAgg.getName() + "]"); + } + } + + private void processTerms(Terms termsAgg) throws IOException { + for (Terms.Bucket bucket : termsAgg.getBuckets()) { + keyValuePairs.put(termsAgg.getName(), bucket.getKey()); + processNestedAggs(bucket.getDocCount(), bucket.getAggregations()); + } + } + + private void processSingleValue(long docCount, NumericMetricsAggregation.SingleValue singleValue) throws IOException { + keyValuePairs.put(singleValue.getName(), singleValue.value()); + writeJsonObject(docCount); + } + + private void writeJsonObject(long docCount) throws IOException { + if (docCount > 0) { + jsonBuilder.startObject(); + for (Map.Entry keyValue : keyValuePairs.entrySet()) { + jsonBuilder.field(keyValue.getKey(), keyValue.getValue()); + } + jsonBuilder.field(DatafeedConfig.DOC_COUNT, docCount); + jsonBuilder.endObject(); + } + } + + @Override + public void close() { + jsonBuilder.close(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java new file mode 100644 index 00000000000..7304e0eedbf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractor.java @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.ExtractorUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; + +/** + * A wrapper {@link DataExtractor} that can be used with other extractors in order to perform + * searches in smaller chunks of the time range. + * + *

The chunk span can be either specified or not. When not specified, + * a heuristic is employed (see {@link DataSummary#estimateChunk()}) to automatically determine the chunk span. + * The search is set up (see {@link #setUpChunkedSearch()} by querying a data summary for the given time range + * that includes the number of total hits and the earliest/latest times. Those are then used to determine the chunk span, + * when necessary, and to jump the search forward to the time where the earliest data can be found. + * If a search for a chunk returns empty, the set up is performed again for the remaining time. + * + *

Cancellation's behaviour depends on the delegate extractor. + * + *

Note that this class is NOT thread-safe. + */ +public class ChunkedDataExtractor implements DataExtractor { + + private static final Logger LOGGER = Loggers.getLogger(ChunkedDataExtractor.class); + + private static final String EARLIEST_TIME = "earliest_time"; + private static final String LATEST_TIME = "latest_time"; + private static final String VALUE_SUFFIX = ".value"; + + /** Let us set a minimum chunk span of 1 minute */ + private static final long MIN_CHUNK_SPAN = 60000L; + + private final Client client; + private final DataExtractorFactory dataExtractorFactory; + private final ChunkedDataExtractorContext context; + private long currentStart; + private long currentEnd; + private long chunkSpan; + private boolean isCancelled; + private DataExtractor currentExtractor; + + public ChunkedDataExtractor(Client client, DataExtractorFactory dataExtractorFactory, ChunkedDataExtractorContext context) { + this.client = Objects.requireNonNull(client); + this.dataExtractorFactory = Objects.requireNonNull(dataExtractorFactory); + this.context = Objects.requireNonNull(context); + this.currentStart = context.start; + this.currentEnd = context.start; + this.isCancelled = false; + } + + @Override + public boolean hasNext() { + boolean currentHasNext = currentExtractor != null && currentExtractor.hasNext(); + if (isCancelled()) { + return currentHasNext; + } + return currentHasNext || currentEnd < context.end; + } + + @Override + public Optional next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + if (currentExtractor == null) { + // This is the first time next is called + setUpChunkedSearch(); + } + + return getNextStream(); + } + + private void setUpChunkedSearch() throws IOException { + DataSummary dataSummary = requestDataSummary(); + if (dataSummary.totalHits > 0) { + currentStart = dataSummary.earliestTime; + currentEnd = currentStart; + chunkSpan = context.chunkSpan == null ? dataSummary.estimateChunk() : context.chunkSpan; + LOGGER.info("Chunked search configured: totalHits = {}, dataTimeSpread = {} ms, chunk span = {} ms", + dataSummary.totalHits, dataSummary.getDataTimeSpread(), chunkSpan); + } else { + // search is over + currentEnd = context.end; + } + } + + private DataSummary requestDataSummary() throws IOException { + SearchRequestBuilder searchRequestBuilder = SearchAction.INSTANCE.newRequestBuilder(client) + .setSize(0) + .setIndices(context.indexes) + .setTypes(context.types) + .setQuery(ExtractorUtils.wrapInTimeRangeQuery(context.query, context.timeField, currentStart, context.end)) + .addAggregation(AggregationBuilders.min(EARLIEST_TIME).field(context.timeField)) + .addAggregation(AggregationBuilders.max(LATEST_TIME).field(context.timeField)); + + SearchResponse response = executeSearchRequest(searchRequestBuilder); + + ExtractorUtils.checkSearchWasSuccessful(context.jobId, response); + + Aggregations aggregations = response.getAggregations(); + long earliestTime = 0; + long latestTime = 0; + long totalHits = response.getHits().getTotalHits(); + if (totalHits > 0) { + earliestTime = (long) Double.parseDouble(aggregations.getProperty(EARLIEST_TIME + VALUE_SUFFIX).toString()); + latestTime = (long) Double.parseDouble(aggregations.getProperty(LATEST_TIME + VALUE_SUFFIX).toString()); + } + return new DataSummary(earliestTime, latestTime, totalHits); + } + + protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { + return searchRequestBuilder.get(); + } + + private Optional getNextStream() throws IOException { + while (hasNext()) { + boolean isNewSearch = false; + + if (currentExtractor == null || currentExtractor.hasNext() == false) { + // First search or the current search finished; we can advance to the next search + advanceTime(); + isNewSearch = true; + } + + Optional nextStream = currentExtractor.next(); + if (nextStream.isPresent()) { + return nextStream; + } + + if (isNewSearch && hasNext()) { + // If it was a new search it means it returned 0 results. Thus, + // we reconfigure and jump to the next time interval where there are data. + setUpChunkedSearch(); + } + } + return Optional.empty(); + } + + private void advanceTime() { + currentStart = currentEnd; + currentEnd = Math.min(currentStart + chunkSpan, context.end); + currentExtractor = dataExtractorFactory.newExtractor(currentStart, currentEnd); + LOGGER.trace("advances time to [{}, {})", currentStart, currentEnd); + } + + @Override + public boolean isCancelled() { + return isCancelled; + } + + @Override + public void cancel() { + if (currentExtractor != null) { + currentExtractor.cancel(); + } + isCancelled = true; + } + + private class DataSummary { + + private long earliestTime; + private long latestTime; + private long totalHits; + + private DataSummary(long earliestTime, long latestTime, long totalHits) { + this.earliestTime = earliestTime; + this.latestTime = latestTime; + this.totalHits = totalHits; + } + + private long getDataTimeSpread() { + return latestTime - earliestTime; + } + + /** + * The heuristic here is that we want a time interval where we expect roughly scrollSize documents + * (assuming data are uniformly spread over time). + * We have totalHits documents over dataTimeSpread (latestTime - earliestTime), we want scrollSize documents over chunk. + * Thus, the interval would be (scrollSize * dataTimeSpread) / totalHits. + * However, assuming this as the chunk span may often lead to half-filled pages or empty searches. + * It is beneficial to take a multiple of that. Based on benchmarking, we set this to 10x. + */ + private long estimateChunk() { + long dataTimeSpread = getDataTimeSpread(); + if (totalHits <= 0 || dataTimeSpread <= 0) { + return context.end - currentEnd; + } + long estimatedChunk = 10 * (context.scrollSize * getDataTimeSpread()) / totalHits; + return Math.max(estimatedChunk, MIN_CHUNK_SPAN); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java new file mode 100644 index 00000000000..67d18300df3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; + +import org.elasticsearch.common.inject.internal.Nullable; +import org.elasticsearch.index.query.QueryBuilder; + +import java.util.List; +import java.util.Objects; + +class ChunkedDataExtractorContext { + + final String jobId; + final String timeField; + final String[] indexes; + final String[] types; + final QueryBuilder query; + final int scrollSize; + final long start; + final long end; + final Long chunkSpan; + + ChunkedDataExtractorContext(String jobId, String timeField, List indexes, List types, + QueryBuilder query, int scrollSize, long start, long end, @Nullable Long chunkSpan) { + this.jobId = Objects.requireNonNull(jobId); + this.timeField = Objects.requireNonNull(timeField); + this.indexes = indexes.toArray(new String[indexes.size()]); + this.types = types.toArray(new String[types.size()]); + this.query = Objects.requireNonNull(query); + this.scrollSize = scrollSize; + this.start = start; + this.end = end; + this.chunkSpan = chunkSpan; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java new file mode 100644 index 00000000000..20417edb22d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; + +import org.elasticsearch.client.Client; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.util.Objects; + +public class ChunkedDataExtractorFactory implements DataExtractorFactory { + + private final Client client; + private final DatafeedConfig datafeedConfig; + private final Job job; + private final DataExtractorFactory dataExtractorFactory; + + public ChunkedDataExtractorFactory(Client client, DatafeedConfig datafeedConfig, Job job, DataExtractorFactory dataExtractorFactory) { + this.client = Objects.requireNonNull(client); + this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.job = Objects.requireNonNull(job); + this.dataExtractorFactory = Objects.requireNonNull(dataExtractorFactory); + } + + @Override + public DataExtractor newExtractor(long start, long end) { + ChunkedDataExtractorContext dataExtractorContext = new ChunkedDataExtractorContext( + job.getId(), + job.getDataDescription().getTimeField(), + datafeedConfig.getIndexes(), + datafeedConfig.getTypes(), + datafeedConfig.getQuery(), + datafeedConfig.getScrollSize(), + start, + end, + datafeedConfig.getChunkingConfig() == null ? null : datafeedConfig.getChunkingConfig().getTimeSpan()); + return new ChunkedDataExtractor(client, dataExtractorFactory, dataExtractorContext); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java new file mode 100644 index 00000000000..0200a8d4db1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.joda.time.base.BaseDateTime; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +abstract class ExtractedField { + + public enum ExtractionMethod { + SOURCE, DOC_VALUE, SCRIPT_FIELD + } + + protected final String name; + private final ExtractionMethod extractionMethod; + + protected ExtractedField(String name, ExtractionMethod extractionMethod) { + this.name = Objects.requireNonNull(name); + this.extractionMethod = Objects.requireNonNull(extractionMethod); + } + + public String getName() { + return name; + } + + public ExtractionMethod getExtractionMethod() { + return extractionMethod; + } + + public abstract Object[] value(SearchHit hit); + + public static ExtractedField newTimeField(String name, ExtractionMethod extractionMethod) { + if (extractionMethod == ExtractionMethod.SOURCE) { + throw new IllegalArgumentException("time field cannot be extracted from source"); + } + return new TimeField(name, extractionMethod); + } + + public static ExtractedField newField(String name, ExtractionMethod extractionMethod) { + switch (extractionMethod) { + case DOC_VALUE: + case SCRIPT_FIELD: + return new FromFields(name, extractionMethod); + case SOURCE: + return new FromSource(name, extractionMethod); + default: + throw new IllegalArgumentException("Invalid extraction method [" + extractionMethod + "]"); + } + } + + private static class FromFields extends ExtractedField { + + FromFields(String name, ExtractionMethod extractionMethod) { + super(name, extractionMethod); + } + + @Override + public Object[] value(SearchHit hit) { + SearchHitField keyValue = hit.field(name); + if (keyValue != null) { + List values = keyValue.getValues(); + return values.toArray(new Object[values.size()]); + } + return new Object[0]; + } + } + + private static class TimeField extends FromFields { + + TimeField(String name, ExtractionMethod extractionMethod) { + super(name, extractionMethod); + } + + @Override + public Object[] value(SearchHit hit) { + Object[] value = super.value(hit); + if (value.length != 1) { + return value; + } + value[0] = ((BaseDateTime) value[0]).getMillis(); + return value; + } + } + + private static class FromSource extends ExtractedField { + + private String[] namePath; + + FromSource(String name, ExtractionMethod extractionMethod) { + super(name, extractionMethod); + namePath = name.split("\\."); + } + + @Override + public Object[] value(SearchHit hit) { + Map source = hit.getSourceAsMap(); + int level = 0; + while (source != null && level < namePath.length - 1) { + source = getNextLevel(source, namePath[level]); + level++; + } + if (source != null) { + Object values = source.get(namePath[level]); + if (values != null) { + if (values instanceof List) { + @SuppressWarnings("unchecked") + List asList = (List) values; + return asList.toArray(new Object[asList.size()]); + } else { + return new Object[]{values}; + } + } + } + return new Object[0]; + } + + @SuppressWarnings("unchecked") + private static Map getNextLevel(Map source, String key) { + Object nextLevel = source.get(key); + if (nextLevel instanceof Map) { + return (Map) source.get(key); + } + return null; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFields.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFields.java new file mode 100644 index 00000000000..e74d2d71bc1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFields.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +class ExtractedFields { + + private final ExtractedField timeField; + private final List allFields; + + ExtractedFields(ExtractedField timeField, List allFields) { + if (!allFields.contains(timeField)) { + throw new IllegalArgumentException("timeField should also be contained in allFields"); + } + this.timeField = Objects.requireNonNull(timeField); + this.allFields = Collections.unmodifiableList(allFields); + } + + public List getAllFields() { + return allFields; + } + + public String[] getSourceFields() { + return filterFields(ExtractedField.ExtractionMethod.SOURCE); + } + + public String[] getDocValueFields() { + return filterFields(ExtractedField.ExtractionMethod.DOC_VALUE); + } + + private String[] filterFields(ExtractedField.ExtractionMethod method) { + List result = new ArrayList<>(); + for (ExtractedField field : allFields) { + if (field.getExtractionMethod() == method) { + result.add(field.getName()); + } + } + return result.toArray(new String[result.size()]); + } + + public String timeField() { + return timeField.getName(); + } + + public Long timeFieldValue(SearchHit hit) { + Object[] value = timeField.value(hit); + if (value.length != 1) { + throw new RuntimeException("Time field [" + timeField.getName() + "] expected a single value; actual was: " + + Arrays.toString(value)); + } + if (value[0] instanceof Long) { + return (Long) value[0]; + } + throw new RuntimeException("Time field [" + timeField.getName() + "] expected a long value; actual was: " + value[0]); + } + + public static ExtractedFields build(Job job, DatafeedConfig datafeedConfig) { + Set scriptFields = datafeedConfig.getScriptFields().stream().map(sf -> sf.fieldName()).collect(Collectors.toSet()); + String timeField = job.getDataDescription().getTimeField(); + ExtractedField timeExtractedField = ExtractedField.newTimeField(timeField, scriptFields.contains(timeField) ? + ExtractedField.ExtractionMethod.SCRIPT_FIELD : ExtractedField.ExtractionMethod.DOC_VALUE); + List remainingFields = job.allFields().stream().filter(f -> !f.equals(timeField)).collect(Collectors.toList()); + List allExtractedFields = new ArrayList<>(remainingFields.size()); + allExtractedFields.add(timeExtractedField); + for (String field : remainingFields) { + ExtractedField.ExtractionMethod method = scriptFields.contains(field) ? ExtractedField.ExtractionMethod.SCRIPT_FIELD : + datafeedConfig.isSource() ? ExtractedField.ExtractionMethod.SOURCE : ExtractedField.ExtractionMethod.DOC_VALUE; + allExtractedFields.add(ExtractedField.newField(field, method)); + } + return new ExtractedFields(timeExtractedField, allExtractedFields); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java new file mode 100644 index 00000000000..a7df09b0dcc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.ClearScrollAction; +import org.elasticsearch.action.search.SearchAction; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.fetch.StoredFieldsContext; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.ExtractorUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** + * An implementation that extracts data from elasticsearch using search and scroll on a client. + * It supports safe and responsive cancellation by continuing the scroll until a new timestamp + * is seen. + * Note that this class is NOT thread-safe. + */ +class ScrollDataExtractor implements DataExtractor { + + private static final Logger LOGGER = Loggers.getLogger(ScrollDataExtractor.class); + private static final TimeValue SCROLL_TIMEOUT = new TimeValue(10, TimeUnit.MINUTES); + + private final Client client; + private final ScrollDataExtractorContext context; + private String scrollId; + private boolean isCancelled; + private boolean hasNext; + private Long timestampOnCancel; + + ScrollDataExtractor(Client client, ScrollDataExtractorContext dataExtractorContext) { + this.client = Objects.requireNonNull(client); + this.context = Objects.requireNonNull(dataExtractorContext); + this.hasNext = true; + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public boolean isCancelled() { + return isCancelled; + } + + @Override + public void cancel() { + LOGGER.trace("[{}] Data extractor received cancel request", context.jobId); + isCancelled = true; + } + + @Override + public Optional next() throws IOException { + if (!hasNext()) { + throw new NoSuchElementException(); + } + Optional stream = scrollId == null ? Optional.ofNullable(initScroll()) : Optional.ofNullable(continueScroll()); + if (!stream.isPresent()) { + hasNext = false; + } + return stream; + } + + private InputStream initScroll() throws IOException { + LOGGER.debug("[{}] Initializing scroll", context.jobId); + SearchResponse searchResponse = executeSearchRequest(buildSearchRequest()); + return processSearchResponse(searchResponse); + } + + protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { + return searchRequestBuilder.get(); + } + + private SearchRequestBuilder buildSearchRequest() { + SearchRequestBuilder searchRequestBuilder = SearchAction.INSTANCE.newRequestBuilder(client) + .setScroll(SCROLL_TIMEOUT) + .addSort(context.extractedFields.timeField(), SortOrder.ASC) + .setIndices(context.indexes) + .setTypes(context.types) + .setSize(context.scrollSize) + .setQuery(ExtractorUtils.wrapInTimeRangeQuery( + context.query, context.extractedFields.timeField(), context.start, context.end)); + + for (String docValueField : context.extractedFields.getDocValueFields()) { + searchRequestBuilder.addDocValueField(docValueField); + } + String[] sourceFields = context.extractedFields.getSourceFields(); + if (sourceFields.length == 0) { + searchRequestBuilder.setFetchSource(false); + searchRequestBuilder.storedFields(StoredFieldsContext._NONE_); + } else { + searchRequestBuilder.setFetchSource(sourceFields, null); + } + context.scriptFields.forEach(f -> searchRequestBuilder.addScriptField(f.fieldName(), f.script())); + return searchRequestBuilder; + } + + private InputStream processSearchResponse(SearchResponse searchResponse) throws IOException { + ExtractorUtils.checkSearchWasSuccessful(context.jobId, searchResponse); + scrollId = searchResponse.getScrollId(); + if (searchResponse.getHits().getHits().length == 0) { + hasNext = false; + clearScroll(scrollId); + return null; + } + + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (SearchHitToJsonProcessor hitProcessor = new SearchHitToJsonProcessor(context.extractedFields, outputStream)) { + for (SearchHit hit : searchResponse.getHits().getHits()) { + if (isCancelled) { + Long timestamp = context.extractedFields.timeFieldValue(hit); + if (timestamp != null) { + if (timestampOnCancel == null) { + timestampOnCancel = timestamp; + } else if (timestamp != timestampOnCancel) { + hasNext = false; + clearScroll(scrollId); + break; + } + } + } + hitProcessor.process(hit); + } + } + return new ByteArrayInputStream(outputStream.toByteArray()); + } + + private InputStream continueScroll() throws IOException { + LOGGER.debug("[{}] Continuing scroll with id [{}]", context.jobId, scrollId); + SearchResponse searchResponse = executeSearchScrollRequest(scrollId); + return processSearchResponse(searchResponse); + } + + protected SearchResponse executeSearchScrollRequest(String scrollId) { + return SearchScrollAction.INSTANCE.newRequestBuilder(client) + .setScroll(SCROLL_TIMEOUT) + .setScrollId(scrollId) + .get(); + } + + void clearScroll(String scrollId) { + ClearScrollAction.INSTANCE.newRequestBuilder(client).addScrollId(scrollId).get(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java new file mode 100644 index 00000000000..80fe0922c8c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.util.List; +import java.util.Objects; + +class ScrollDataExtractorContext { + + final String jobId; + final ExtractedFields extractedFields; + final String[] indexes; + final String[] types; + final QueryBuilder query; + final List scriptFields; + final int scrollSize; + final long start; + final long end; + + ScrollDataExtractorContext(String jobId, ExtractedFields extractedFields, List indexes, List types, + QueryBuilder query, List scriptFields, int scrollSize, + long start, long end) { + this.jobId = Objects.requireNonNull(jobId); + this.extractedFields = Objects.requireNonNull(extractedFields); + this.indexes = indexes.toArray(new String[indexes.size()]); + this.types = types.toArray(new String[types.size()]); + this.query = Objects.requireNonNull(query); + this.scriptFields = Objects.requireNonNull(scriptFields); + this.scrollSize = scrollSize; + this.start = start; + this.end = end; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java new file mode 100644 index 00000000000..b50e30846bc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.client.Client; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; + +import java.util.Objects; + +public class ScrollDataExtractorFactory implements DataExtractorFactory { + + private final Client client; + private final DatafeedConfig datafeedConfig; + private final Job job; + private final ExtractedFields extractedFields; + + public ScrollDataExtractorFactory(Client client, DatafeedConfig datafeedConfig, Job job) { + this.client = Objects.requireNonNull(client); + this.datafeedConfig = Objects.requireNonNull(datafeedConfig); + this.job = Objects.requireNonNull(job); + this.extractedFields = ExtractedFields.build(job, datafeedConfig); + } + + @Override + public DataExtractor newExtractor(long start, long end) { + ScrollDataExtractorContext dataExtractorContext = new ScrollDataExtractorContext( + job.getId(), + extractedFields, + datafeedConfig.getIndexes(), + datafeedConfig.getTypes(), + datafeedConfig.getQuery(), + datafeedConfig.getScriptFields(), + datafeedConfig.getScrollSize(), + start, + end); + return new ScrollDataExtractor(client, dataExtractorContext); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java new file mode 100644 index 00000000000..193d92a13dc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessor.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Objects; + +class SearchHitToJsonProcessor implements Releasable { + + private final ExtractedFields fields; + private final XContentBuilder jsonBuilder; + + SearchHitToJsonProcessor(ExtractedFields fields, OutputStream outputStream) throws IOException { + this.fields = Objects.requireNonNull(fields); + this.jsonBuilder = new XContentBuilder(JsonXContent.jsonXContent, outputStream); + } + + public void process(SearchHit hit) throws IOException { + jsonBuilder.startObject(); + for (ExtractedField field : fields.getAllFields()) { + writeKeyValue(field.getName(), field.value(hit)); + } + jsonBuilder.endObject(); + } + + private void writeKeyValue(String key, Object... values) throws IOException { + if (values.length == 0) { + return; + } + if (values.length == 1) { + jsonBuilder.field(key, values[0]); + } else { + jsonBuilder.array(key, values); + } + } + + @Override + public void close() { + jsonBuilder.close(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java new file mode 100644 index 00000000000..83f31b621ac --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -0,0 +1,382 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ack.AckedRequest; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.ml.action.DeleteJobAction; +import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction; +import org.elasticsearch.xpack.ml.action.UpdateJobStateAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.IgnoreDowntime; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Allows interactions with jobs. The managed interactions include: + *
    + *
  • creation
    • + *
  • deletion
    • + *
  • updating
    • + *
  • starting/stopping of datafeed jobs
    • + *
+ */ +public class JobManager extends AbstractComponent { + + /** + * Field name in which to store the API version in the usage info + */ + public static final String APP_VER_FIELDNAME = "appVer"; + + public static final String DEFAULT_RECORD_SORT_FIELD = AnomalyRecord.PROBABILITY.getPreferredName(); + + private final JobProvider jobProvider; + private final ClusterService clusterService; + private final JobResultsPersister jobResultsPersister; + + + /** + * Create a JobManager + */ + public JobManager(Settings settings, JobProvider jobProvider, JobResultsPersister jobResultsPersister, + ClusterService clusterService) { + super(settings); + this.jobProvider = Objects.requireNonNull(jobProvider); + this.clusterService = clusterService; + this.jobResultsPersister = jobResultsPersister; + } + + /** + * Get the jobs that match the given {@code jobId}. + * Note that when the {@code jocId} is {@link Job#ALL} all jobs are returned. + * + * @param jobId + * the jobId + * @return A {@link QueryPage} containing the matching {@code Job}s + */ + public QueryPage getJob(String jobId, ClusterState clusterState) { + if (jobId.equals(Job.ALL)) { + return getJobs(clusterState); + } + MlMetadata mlMetadata = clusterState.getMetaData().custom(MlMetadata.TYPE); + Job job = mlMetadata.getJobs().get(jobId); + if (job == null) { + logger.debug(String.format(Locale.ROOT, "Cannot find job '%s'", jobId)); + throw ExceptionsHelper.missingJobException(jobId); + } + + logger.debug("Returning job [" + jobId + "]"); + return new QueryPage<>(Collections.singletonList(job), 1, Job.RESULTS_FIELD); + } + + /** + * Get details of all Jobs. + * + * @return A query page object with hitCount set to the total number of jobs + * not the only the number returned here as determined by the + * size parameter. + */ + public QueryPage getJobs(ClusterState clusterState) { + MlMetadata mlMetadata = clusterState.getMetaData().custom(MlMetadata.TYPE); + List jobs = mlMetadata.getJobs().entrySet().stream() + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + return new QueryPage<>(jobs, mlMetadata.getJobs().size(), Job.RESULTS_FIELD); + } + + /** + * Returns the non-null {@code Job} object for the given + * {@code jobId} or throws + * {@link org.elasticsearch.ResourceNotFoundException} + * + * @param jobId + * the jobId + * @return the {@code Job} if a job with the given {@code jobId} + * exists + * @throws org.elasticsearch.ResourceNotFoundException + * if there is no job with matching the given {@code jobId} + */ + public Job getJobOrThrowIfUnknown(String jobId) { + return getJobOrThrowIfUnknown(clusterService.state(), jobId); + } + + public Allocation getJobAllocation(String jobId) { + return getAllocation(clusterService.state(), jobId); + } + + /** + * Returns the non-null {@code Job} object for the given + * {@code jobId} or throws + * {@link org.elasticsearch.ResourceNotFoundException} + * + * @param jobId + * the jobId + * @return the {@code Job} if a job with the given {@code jobId} + * exists + * @throws org.elasticsearch.ResourceNotFoundException + * if there is no job with matching the given {@code jobId} + */ + Job getJobOrThrowIfUnknown(ClusterState clusterState, String jobId) { + MlMetadata mlMetadata = clusterState.metaData().custom(MlMetadata.TYPE); + Job job = mlMetadata.getJobs().get(jobId); + if (job == null) { + throw ExceptionsHelper.missingJobException(jobId); + } + return job; + } + + /** + * Stores a job in the cluster state + */ + public void putJob(PutJobAction.Request request, ActionListener actionListener) { + Job job = request.getJob(); + + ActionListener createResultsIndexListener = ActionListener.wrap(jobSaved -> + jobProvider.createJobResultIndex(job, new ActionListener() { + @Override + public void onResponse(Boolean indicesCreated) { + audit(job.getId()).info(Messages.getMessage(Messages.JOB_AUDIT_CREATED)); + + // Also I wonder if we need to audit log infra + // structure in ml as when we merge into xpack + // we can use its audit trailing. See: + // https://github.com/elastic/prelert-legacy/issues/48 + actionListener.onResponse(new PutJobAction.Response(jobSaved && indicesCreated, job)); + } + + @Override + public void onFailure(Exception e) { + actionListener.onFailure(e); + } + }), actionListener::onFailure); + + clusterService.submitStateUpdateTask("put-job-" + job.getId(), + new AckedClusterStateUpdateTask(request, createResultsIndexListener) { + @Override + protected Boolean newResponse(boolean acknowledged) { + return acknowledged; + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState cs = updateClusterState(job, false, currentState); + if (currentState.metaData().index(AnomalyDetectorsIndex.jobResultsIndexName(job.getIndexName())) != null) { + throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_INDEX_ALREADY_EXISTS, + AnomalyDetectorsIndex.jobResultsIndexName(job.getIndexName()))); + } + return cs; + } + }); + } + + public void updateJob(String jobId, JobUpdate jobUpdate, AckedRequest request, ActionListener actionListener) { + clusterService.submitStateUpdateTask("update-job-" + jobId, + new AckedClusterStateUpdateTask(request, actionListener) { + private Job updatedJob; + + @Override + protected PutJobAction.Response newResponse(boolean acknowledged) { + return new PutJobAction.Response(acknowledged, updatedJob); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + Job job = getJob(jobId, currentState).results().get(0); + updatedJob = jobUpdate.mergeWithJob(job); + return updateClusterState(updatedJob, true, currentState); + } + }); + } + + ClusterState updateClusterState(Job job, boolean overwrite, ClusterState currentState) { + MlMetadata.Builder builder = createMlMetadataBuilder(currentState); + builder.putJob(job, overwrite); + return buildNewClusterState(currentState, builder); + } + + + public void deleteJob(DeleteJobAction.Request request, Client client, JobStorageDeletionTask task, + ActionListener actionListener) { + + String jobId = request.getJobId(); + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + logger.debug("Deleting job '" + jobId + "'"); + + // Step 3. When the job has been removed from the cluster state, return a response + // ------- + CheckedConsumer apiResponseHandler = jobDeleted -> { + if (jobDeleted) { + logger.info("Job [" + jobId + "] deleted."); + actionListener.onResponse(new DeleteJobAction.Response(true)); + audit(jobId).info(Messages.getMessage(Messages.JOB_AUDIT_DELETED)); + } else { + actionListener.onResponse(new DeleteJobAction.Response(false)); + } + }; + + // Step 2. When the physical storage has been deleted, remove from Cluster State + // ------- + CheckedConsumer deleteJobStateHandler = response -> clusterService.submitStateUpdateTask("delete-job-" + jobId, + new AckedClusterStateUpdateTask(request, ActionListener.wrap(apiResponseHandler, actionListener::onFailure)) { + + @Override + protected Boolean newResponse(boolean acknowledged) { + return acknowledged && response; + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + return removeJobFromState(jobId, currentState); + } + }); + + // Step 1. When the job's status updates to DELETING, begin deleting the physical storage + // ------- + CheckedConsumer updateHandler = response -> { + // Successfully updated the status to DELETING, begin actually deleting + if (response.isAcknowledged()) { + logger.info("Job [" + jobId + "] set to [" + JobState.DELETING + "]"); + } else { + logger.warn("Job [" + jobId + "] change to [" + JobState.DELETING + "] was not acknowledged."); + } + + // This task manages the physical deletion of the job (removing the results, then the index) + task.delete(jobId, indexName, client, deleteJobStateHandler::accept, actionListener::onFailure); + + }; + + // Step 0. Kick off the chain of callbacks with the initial UpdateStatus call + // ------- + UpdateJobStateAction.Request updateStateListener = new UpdateJobStateAction.Request(jobId, JobState.DELETING); + setJobState(updateStateListener, ActionListener.wrap(updateHandler, actionListener::onFailure)); + + } + + ClusterState removeJobFromState(String jobId, ClusterState currentState) { + MlMetadata.Builder builder = createMlMetadataBuilder(currentState); + builder.deleteJob(jobId); + return buildNewClusterState(currentState, builder); + } + + private Allocation getAllocation(ClusterState state, String jobId) { + MlMetadata mlMetadata = state.metaData().custom(MlMetadata.TYPE); + Allocation allocation = mlMetadata.getAllocations().get(jobId); + if (allocation == null) { + throw new ResourceNotFoundException("No allocation found for job with id [" + jobId + "]"); + } + return allocation; + } + + public Auditor audit(String jobId) { + return jobProvider.audit(jobId); + } + + public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionListener actionListener, + ModelSnapshot modelSnapshot) { + + clusterService.submitStateUpdateTask("revert-snapshot-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + + @Override + protected RevertModelSnapshotAction.Response newResponse(boolean acknowledged) { + if (acknowledged) { + audit(request.getJobId()) + .info(Messages.getMessage(Messages.JOB_AUDIT_REVERTED, modelSnapshot.getDescription())); + return new RevertModelSnapshotAction.Response(modelSnapshot); + } + throw new IllegalStateException("Could not revert modelSnapshot on job [" + + request.getJobId() + "], not acknowledged by master."); + } + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + Job job = getJobOrThrowIfUnknown(currentState, request.getJobId()); + Job.Builder builder = new Job.Builder(job); + builder.setModelSnapshotId(modelSnapshot.getSnapshotId()); + if (request.getDeleteInterveningResults()) { + builder.setIgnoreDowntime(IgnoreDowntime.NEVER); + } else { + builder.setIgnoreDowntime(IgnoreDowntime.ONCE); + } + + return updateClusterState(builder.build(), true, currentState); + } + }); + } + + public void setJobState(UpdateJobStateAction.Request request, ActionListener actionListener) { + clusterService.submitStateUpdateTask("set-job-state-" + request.getState() + "-" + request.getJobId(), + new AckedClusterStateUpdateTask(request, actionListener) { + + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + MlMetadata.Builder builder = new MlMetadata.Builder(currentState.metaData().custom(MlMetadata.TYPE)); + builder.updateState(request.getJobId(), request.getState(), request.getReason()); + return ClusterState.builder(currentState) + .metaData(MetaData.builder(currentState.metaData()).putCustom(MlMetadata.TYPE, builder.build())) + .build(); + } + + @Override + protected UpdateJobStateAction.Response newResponse(boolean acknowledged) { + return new UpdateJobStateAction.Response(acknowledged); + } + }); + } + + /** + * Update a persisted model snapshot metadata document to match the + * argument supplied. + * + * @param modelSnapshot the updated model snapshot object to be stored + */ + public void updateModelSnapshot(ModelSnapshot modelSnapshot, Consumer handler, Consumer errorHandler) { + jobResultsPersister.updateModelSnapshot(modelSnapshot, handler, errorHandler); + } + + private static MlMetadata.Builder createMlMetadataBuilder(ClusterState currentState) { + MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE); + return new MlMetadata.Builder(currentMlMetadata); + } + + private static ClusterState buildNewClusterState(ClusterState currentState, MlMetadata.Builder builder) { + ClusterState.Builder newState = ClusterState.builder(currentState); + newState.metaData(MetaData.builder(currentState.getMetaData()).putCustom(MlMetadata.TYPE, builder.build()).build()); + return newState.build(); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java new file mode 100644 index 00000000000..174051c3671 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfig.java @@ -0,0 +1,713 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; +import java.util.stream.Collectors; + + +/** + * Autodetect analysis configuration options describes which fields are + * analysed and the functions to use. + *

+ * The configuration can contain multiple detectors, a new anomaly detector will + * be created for each detector configuration. The fields + * bucketSpan, batchSpan, summaryCountFieldName and categorizationFieldName + * apply to all detectors. + *

+ * If a value has not been set it will be null + * Object wrappers are used around integral types & booleans so they can take + * null values. + */ +public class AnalysisConfig extends ToXContentToBytes implements Writeable { + /** + * Serialisation names + */ + private static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config"); + private static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + private static final ParseField BATCH_SPAN = new ParseField("batch_span"); + private static final ParseField CATEGORIZATION_FIELD_NAME = new ParseField("categorization_field_name"); + public static final ParseField CATEGORIZATION_FILTERS = new ParseField("categorization_filters"); + private static final ParseField LATENCY = new ParseField("latency"); + private static final ParseField PERIOD = new ParseField("period"); + private static final ParseField SUMMARY_COUNT_FIELD_NAME = new ParseField("summary_count_field_name"); + private static final ParseField DETECTORS = new ParseField("detectors"); + private static final ParseField INFLUENCERS = new ParseField("influencers"); + private static final ParseField OVERLAPPING_BUCKETS = new ParseField("overlapping_buckets"); + private static final ParseField RESULT_FINALIZATION_WINDOW = new ParseField("result_finalization_window"); + private static final ParseField MULTIVARIATE_BY_FIELDS = new ParseField("multivariate_by_fields"); + private static final ParseField MULTIPLE_BUCKET_SPANS = new ParseField("multiple_bucket_spans"); + private static final ParseField USER_PER_PARTITION_NORMALIZATION = new ParseField("use_per_partition_normalization"); + + private static final String ML_CATEGORY_FIELD = "mlcategory"; + public static final Set AUTO_CREATED_FIELDS = new HashSet<>(Arrays.asList(ML_CATEGORY_FIELD)); + + public static final long DEFAULT_RESULT_FINALIZATION_WINDOW = 2L; + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(ANALYSIS_CONFIG.getPreferredName(), a -> new AnalysisConfig.Builder((List) a[0])); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> Detector.PARSER.apply(p, c).build(), DETECTORS); + PARSER.declareLong(Builder::setBucketSpan, BUCKET_SPAN); + PARSER.declareLong(Builder::setBatchSpan, BATCH_SPAN); + PARSER.declareString(Builder::setCategorizationFieldName, CATEGORIZATION_FIELD_NAME); + PARSER.declareStringArray(Builder::setCategorizationFilters, CATEGORIZATION_FILTERS); + PARSER.declareLong(Builder::setLatency, LATENCY); + PARSER.declareLong(Builder::setPeriod, PERIOD); + PARSER.declareString(Builder::setSummaryCountFieldName, SUMMARY_COUNT_FIELD_NAME); + PARSER.declareStringArray(Builder::setInfluencers, INFLUENCERS); + PARSER.declareBoolean(Builder::setOverlappingBuckets, OVERLAPPING_BUCKETS); + PARSER.declareLong(Builder::setResultFinalizationWindow, RESULT_FINALIZATION_WINDOW); + PARSER.declareBoolean(Builder::setMultivariateByFields, MULTIVARIATE_BY_FIELDS); + PARSER.declareLongArray(Builder::setMultipleBucketSpans, MULTIPLE_BUCKET_SPANS); + PARSER.declareBoolean(Builder::setUsePerPartitionNormalization, USER_PER_PARTITION_NORMALIZATION); + } + + /** + * These values apply to all detectors + */ + private final long bucketSpan; + private final Long batchSpan; + private final String categorizationFieldName; + private final List categorizationFilters; + private final long latency; + private final Long period; + private final String summaryCountFieldName; + private final List detectors; + private final List influencers; + private final Boolean overlappingBuckets; + private final Long resultFinalizationWindow; + private final Boolean multivariateByFields; + private final List multipleBucketSpans; + private final boolean usePerPartitionNormalization; + + private AnalysisConfig(Long bucketSpan, Long batchSpan, String categorizationFieldName, List categorizationFilters, + long latency, Long period, String summaryCountFieldName, List detectors, List influencers, + Boolean overlappingBuckets, Long resultFinalizationWindow, Boolean multivariateByFields, + List multipleBucketSpans, boolean usePerPartitionNormalization) { + this.detectors = detectors; + this.bucketSpan = bucketSpan; + this.batchSpan = batchSpan; + this.latency = latency; + this.period = period; + this.categorizationFieldName = categorizationFieldName; + this.categorizationFilters = categorizationFilters; + this.summaryCountFieldName = summaryCountFieldName; + this.influencers = influencers; + this.overlappingBuckets = overlappingBuckets; + this.resultFinalizationWindow = resultFinalizationWindow; + this.multivariateByFields = multivariateByFields; + this.multipleBucketSpans = multipleBucketSpans; + this.usePerPartitionNormalization = usePerPartitionNormalization; + } + + public AnalysisConfig(StreamInput in) throws IOException { + bucketSpan = in.readLong(); + batchSpan = in.readOptionalLong(); + categorizationFieldName = in.readOptionalString(); + categorizationFilters = in.readBoolean() ? in.readList(StreamInput::readString) : null; + latency = in.readLong(); + period = in.readOptionalLong(); + summaryCountFieldName = in.readOptionalString(); + detectors = in.readList(Detector::new); + influencers = in.readList(StreamInput::readString); + overlappingBuckets = in.readOptionalBoolean(); + resultFinalizationWindow = in.readOptionalLong(); + multivariateByFields = in.readOptionalBoolean(); + multipleBucketSpans = in.readBoolean() ? in.readList(StreamInput::readLong) : null; + usePerPartitionNormalization = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(bucketSpan); + out.writeOptionalLong(batchSpan); + out.writeOptionalString(categorizationFieldName); + if (categorizationFilters != null) { + out.writeBoolean(true); + out.writeStringList(categorizationFilters); + } else { + out.writeBoolean(false); + } + out.writeLong(latency); + out.writeOptionalLong(period); + out.writeOptionalString(summaryCountFieldName); + out.writeList(detectors); + out.writeStringList(influencers); + out.writeOptionalBoolean(overlappingBuckets); + out.writeOptionalLong(resultFinalizationWindow); + out.writeOptionalBoolean(multivariateByFields); + if (multipleBucketSpans != null) { + out.writeBoolean(true); + out.writeVInt(multipleBucketSpans.size()); + for (Long bucketSpan : multipleBucketSpans) { + out.writeLong(bucketSpan); + } + } else { + out.writeBoolean(false); + } + out.writeBoolean(usePerPartitionNormalization); + } + + /** + * The size of the interval the analysis is aggregated into measured in + * seconds + * + * @return The bucketspan or null if not set + */ + public Long getBucketSpan() { + return bucketSpan; + } + + public long getBucketSpanOrDefault() { + return bucketSpan; + } + + /** + * Interval into which to batch seasonal data measured in seconds + * + * @return The batchspan or null if not set + */ + public Long getBatchSpan() { + return batchSpan; + } + + public String getCategorizationFieldName() { + return categorizationFieldName; + } + + public List getCategorizationFilters() { + return categorizationFilters; + } + + /** + * The latency interval (seconds) during which out-of-order records should be handled. + * + * @return The latency interval (seconds) or null if not set + */ + public Long getLatency() { + return latency; + } + + /** + * The repeat interval for periodic data in multiples of + * {@linkplain #getBatchSpan()} + * + * @return The period or null if not set + */ + public Long getPeriod() { + return period; + } + + /** + * The name of the field that contains counts for pre-summarised input + * + * @return The field name or null if not set + */ + public String getSummaryCountFieldName() { + return summaryCountFieldName; + } + + /** + * The list of analysis detectors. In a valid configuration the list should + * contain at least 1 {@link Detector} + * + * @return The Detectors used in this job + */ + public List getDetectors() { + return detectors; + } + + /** + * The list of influence field names + */ + public List getInfluencers() { + return influencers; + } + + /** + * Return the list of term fields. + * These are the influencer fields, partition field, + * by field and over field of each detector. + * null and empty strings are filtered from the + * config. + * + * @return Set of term fields - never null + */ + public Set termFields() { + Set termFields = new TreeSet<>(); + + for (Detector d : getDetectors()) { + addIfNotNull(termFields, d.getByFieldName()); + addIfNotNull(termFields, d.getOverFieldName()); + addIfNotNull(termFields, d.getPartitionFieldName()); + } + + for (String i : getInfluencers()) { + addIfNotNull(termFields, i); + } + + // remove empty strings + termFields.remove(""); + + return termFields; + } + + public Set extractReferencedFilters() { + return detectors.stream().map(Detector::extractReferencedFilters) + .flatMap(Set::stream).collect(Collectors.toSet()); + } + + public Boolean getOverlappingBuckets() { + return overlappingBuckets; + } + + public Long getResultFinalizationWindow() { + return resultFinalizationWindow; + } + + public Boolean getMultivariateByFields() { + return multivariateByFields; + } + + public List getMultipleBucketSpans() { + return multipleBucketSpans; + } + + public boolean getUsePerPartitionNormalization() { + return usePerPartitionNormalization; + } + + /** + * Return the list of fields required by the analysis. + * These are the influencer fields, metric field, partition field, + * by field and over field of each detector, plus the summary count + * field and the categorization field name of the job. + * null and empty strings are filtered from the + * config. + * + * @return List of required analysis fields - never null + */ + public List analysisFields() { + Set analysisFields = termFields(); + + addIfNotNull(analysisFields, categorizationFieldName); + addIfNotNull(analysisFields, summaryCountFieldName); + + for (Detector d : getDetectors()) { + addIfNotNull(analysisFields, d.getFieldName()); + } + + // remove empty strings + analysisFields.remove(""); + + return new ArrayList<>(analysisFields); + } + + private static void addIfNotNull(Set fields, String field) { + if (field != null) { + fields.add(field); + } + } + + public List fields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getFieldName()); + } + + private List collectNonNullAndNonEmptyDetectorFields( + Function fieldGetter) { + Set fields = new HashSet<>(); + + for (Detector d : getDetectors()) { + addIfNotNull(fields, fieldGetter.apply(d)); + } + + // remove empty strings + fields.remove(""); + + return new ArrayList<>(fields); + } + + public List byFields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getByFieldName()); + } + + public List overFields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getOverFieldName()); + } + + + public List partitionFields() { + return collectNonNullAndNonEmptyDetectorFields(d -> d.getPartitionFieldName()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + if (batchSpan != null) { + builder.field(BATCH_SPAN.getPreferredName(), batchSpan); + } + if (categorizationFieldName != null) { + builder.field(CATEGORIZATION_FIELD_NAME.getPreferredName(), categorizationFieldName); + } + if (categorizationFilters != null) { + builder.field(CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); + } + builder.field(LATENCY.getPreferredName(), latency); + if (period != null) { + builder.field(PERIOD.getPreferredName(), period); + } + if (summaryCountFieldName != null) { + builder.field(SUMMARY_COUNT_FIELD_NAME.getPreferredName(), summaryCountFieldName); + } + builder.field(DETECTORS.getPreferredName(), detectors); + builder.field(INFLUENCERS.getPreferredName(), influencers); + if (overlappingBuckets != null) { + builder.field(OVERLAPPING_BUCKETS.getPreferredName(), overlappingBuckets); + } + if (resultFinalizationWindow != null) { + builder.field(RESULT_FINALIZATION_WINDOW.getPreferredName(), resultFinalizationWindow); + } + if (multivariateByFields != null) { + builder.field(MULTIVARIATE_BY_FIELDS.getPreferredName(), multivariateByFields); + } + if (multipleBucketSpans != null) { + builder.field(MULTIPLE_BUCKET_SPANS.getPreferredName(), multipleBucketSpans); + } + builder.field(USER_PER_PARTITION_NORMALIZATION.getPreferredName(), usePerPartitionNormalization); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AnalysisConfig that = (AnalysisConfig) o; + return latency == that.latency && + usePerPartitionNormalization == that.usePerPartitionNormalization && + Objects.equals(bucketSpan, that.bucketSpan) && + Objects.equals(batchSpan, that.batchSpan) && + Objects.equals(categorizationFieldName, that.categorizationFieldName) && + Objects.equals(categorizationFilters, that.categorizationFilters) && + Objects.equals(period, that.period) && + Objects.equals(summaryCountFieldName, that.summaryCountFieldName) && + Objects.equals(detectors, that.detectors) && + Objects.equals(influencers, that.influencers) && + Objects.equals(overlappingBuckets, that.overlappingBuckets) && + Objects.equals(resultFinalizationWindow, that.resultFinalizationWindow) && + Objects.equals(multivariateByFields, that.multivariateByFields) && + Objects.equals(multipleBucketSpans, that.multipleBucketSpans); + } + + @Override + public int hashCode() { + return Objects.hash( + bucketSpan, batchSpan, categorizationFieldName, categorizationFilters, latency, period, + summaryCountFieldName, detectors, influencers, overlappingBuckets, resultFinalizationWindow, + multivariateByFields, multipleBucketSpans, usePerPartitionNormalization + ); + } + + public static class Builder { + + public static final long DEFAULT_BUCKET_SPAN = 300L; + + private List detectors; + private long bucketSpan = DEFAULT_BUCKET_SPAN; + private Long batchSpan; + private long latency = 0L; + private Long period; + private String categorizationFieldName; + private List categorizationFilters; + private String summaryCountFieldName; + private List influencers = new ArrayList<>(); + private Boolean overlappingBuckets; + private Long resultFinalizationWindow; + private Boolean multivariateByFields; + private List multipleBucketSpans; + private boolean usePerPartitionNormalization = false; + + public Builder(List detectors) { + this.detectors = detectors; + } + + public Builder(AnalysisConfig analysisConfig) { + this.detectors = analysisConfig.detectors; + this.bucketSpan = analysisConfig.bucketSpan; + this.batchSpan = analysisConfig.batchSpan; + this.latency = analysisConfig.latency; + this.period = analysisConfig.period; + this.categorizationFieldName = analysisConfig.categorizationFieldName; + this.categorizationFilters = analysisConfig.categorizationFilters; + this.summaryCountFieldName = analysisConfig.summaryCountFieldName; + this.influencers = analysisConfig.influencers; + this.overlappingBuckets = analysisConfig.overlappingBuckets; + this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow; + this.multivariateByFields = analysisConfig.multivariateByFields; + this.multipleBucketSpans = analysisConfig.multipleBucketSpans; + this.usePerPartitionNormalization = analysisConfig.usePerPartitionNormalization; + } + + public void setDetectors(List detectors) { + this.detectors = detectors; + } + + public void setBucketSpan(long bucketSpan) { + this.bucketSpan = bucketSpan; + } + + public void setBatchSpan(long batchSpan) { + this.batchSpan = batchSpan; + } + + public void setLatency(long latency) { + this.latency = latency; + } + + public void setPeriod(long period) { + this.period = period; + } + + public void setCategorizationFieldName(String categorizationFieldName) { + this.categorizationFieldName = categorizationFieldName; + } + + public void setCategorizationFilters(List categorizationFilters) { + this.categorizationFilters = categorizationFilters; + } + + public void setSummaryCountFieldName(String summaryCountFieldName) { + this.summaryCountFieldName = summaryCountFieldName; + } + + public void setInfluencers(List influencers) { + this.influencers = influencers; + } + + public void setOverlappingBuckets(Boolean overlappingBuckets) { + this.overlappingBuckets = overlappingBuckets; + } + + public void setResultFinalizationWindow(Long resultFinalizationWindow) { + this.resultFinalizationWindow = resultFinalizationWindow; + } + + public void setMultivariateByFields(Boolean multivariateByFields) { + this.multivariateByFields = multivariateByFields; + } + + public void setMultipleBucketSpans(List multipleBucketSpans) { + this.multipleBucketSpans = multipleBucketSpans; + } + + public void setUsePerPartitionNormalization(boolean usePerPartitionNormalization) { + this.usePerPartitionNormalization = usePerPartitionNormalization; + } + + /** + * Checks the configuration is valid + *

    + *
  1. Check that if non-null BucketSpan, BatchSpan, Latency and Period are + * >= 0
    2. + *
  2. Check that if non-null Latency is <= MAX_LATENCY
    3. + *
  3. Check there is at least one detector configured
    4. + *
  4. Check all the detectors are configured correctly
    5. + *
  5. Check that OVERLAPPING_BUCKETS is set appropriately
    6. + *
  6. Check that MULTIPLE_BUCKETSPANS are set appropriately
    7. + *
  7. If Per Partition normalization is configured at least one detector + * must have a partition field and no influences can be used
    8. + *
+ */ + public AnalysisConfig build() { + checkFieldIsNotNegativeIfSpecified(BUCKET_SPAN.getPreferredName(), bucketSpan); + checkFieldIsNotNegativeIfSpecified(BATCH_SPAN.getPreferredName(), batchSpan); + checkFieldIsNotNegativeIfSpecified(LATENCY.getPreferredName(), latency); + checkFieldIsNotNegativeIfSpecified(PERIOD.getPreferredName(), period); + + verifyDetectorAreDefined(detectors); + verifyFieldName(summaryCountFieldName); + verifyFieldName(categorizationFieldName); + + verifyCategorizationFilters(categorizationFilters, categorizationFieldName); + verifyMultipleBucketSpans(multipleBucketSpans, bucketSpan); + + overlappingBuckets = verifyOverlappingBucketsConfig(overlappingBuckets, detectors); + + if (usePerPartitionNormalization) { + checkDetectorsHavePartitionFields(detectors); + checkNoInfluencersAreSet(influencers); + } + + return new AnalysisConfig(bucketSpan, batchSpan, categorizationFieldName, categorizationFilters, + latency, period, summaryCountFieldName, detectors, influencers, overlappingBuckets, + resultFinalizationWindow, multivariateByFields, multipleBucketSpans, usePerPartitionNormalization); + } + + private static void checkFieldIsNotNegativeIfSpecified(String fieldName, Long value) { + if (value != null && value < 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, fieldName, 0, value); + throw new IllegalArgumentException(msg); + } + } + + private static void verifyDetectorAreDefined(List detectors) { + if (detectors == null || detectors.isEmpty()) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS)); + } + } + + private static void verifyCategorizationFilters(List filters, String categorizationFieldName) { + if (filters == null || filters.isEmpty()) { + return; + } + + verifyCategorizationFieldNameSetIfFiltersAreSet(categorizationFieldName); + verifyCategorizationFiltersAreDistinct(filters); + verifyCategorizationFiltersContainNoneEmpty(filters); + verifyCategorizationFiltersAreValidRegex(filters); + } + + private static void verifyCategorizationFieldNameSetIfFiltersAreSet(String categorizationFieldName) { + if (categorizationFieldName == null) { + throw new IllegalArgumentException(Messages.getMessage( + Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_REQUIRE_CATEGORIZATION_FIELD_NAME)); + } + } + + private static void verifyCategorizationFiltersAreDistinct(List filters) { + if (filters.stream().distinct().count() != filters.size()) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES)); + } + } + + private static void verifyCategorizationFiltersContainNoneEmpty(List filters) { + if (filters.stream().anyMatch(f -> f.isEmpty())) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY)); + } + } + + private static void verifyCategorizationFiltersAreValidRegex(List filters) { + for (String filter : filters) { + if (!isValidRegex(filter)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_INVALID_REGEX, filter)); + } + } + } + + private static void verifyMultipleBucketSpans(List multipleBucketSpans, Long bucketSpan) { + if (multipleBucketSpans == null) { + return; + } + + if (bucketSpan == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_REQUIRE_BUCKETSPAN)); + } + for (Long span : multipleBucketSpans) { + if ((span % bucketSpan != 0L) || (span <= bucketSpan)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, span, bucketSpan)); + } + } + } + + private static boolean checkDetectorsHavePartitionFields(List detectors) { + for (Detector detector : detectors) { + if (!Strings.isNullOrEmpty(detector.getPartitionFieldName())) { + return true; + } + } + throw new IllegalArgumentException(Messages.getMessage( + Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD)); + } + + private static boolean checkNoInfluencersAreSet(List influencers) { + if (!influencers.isEmpty()) { + throw new IllegalArgumentException(Messages.getMessage( + Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS)); + } + + return true; + } + + /** + * Check that the characters used in a field name will not cause problems. + * + * @param field The field name to be validated + * @return true + */ + public static boolean verifyFieldName(String field) throws ElasticsearchParseException { + if (field != null && containsInvalidChar(field)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_INVALID_FIELDNAME_CHARS, field, Detector.PROHIBITED)); + } + return true; + } + + private static boolean containsInvalidChar(String field) { + for (Character ch : Detector.PROHIBITED_FIELDNAME_CHARACTERS) { + if (field.indexOf(ch) >= 0) { + return true; + } + } + return field.chars().anyMatch(ch -> Character.isISOControl(ch)); + } + + private static boolean isValidRegex(String exp) { + try { + Pattern.compile(exp); + return true; + } catch (PatternSyntaxException e) { + return false; + } + } + + private static Boolean verifyOverlappingBucketsConfig(Boolean overlappingBuckets, List detectors) { + // If any detector function is rare/freq_rare, mustn't use overlapping buckets + boolean mustNotUse = false; + + List illegalFunctions = new ArrayList<>(); + for (Detector d : detectors) { + if (Detector.NO_OVERLAPPING_BUCKETS_FUNCTIONS.contains(d.getFunction())) { + illegalFunctions.add(d.getFunction()); + mustNotUse = true; + } + } + + if (Boolean.TRUE.equals(overlappingBuckets) && mustNotUse) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION, illegalFunctions.toString())); + } + + return overlappingBuckets; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimits.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimits.java new file mode 100644 index 00000000000..a8645a29a86 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimits.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.Objects; + +/** + * Analysis limits for autodetect + *

+ * If an option has not been set it shouldn't be used so the default value is picked up instead. + */ +public class AnalysisLimits extends ToXContentToBytes implements Writeable { + /** + * Serialisation field names + */ + public static final ParseField MODEL_MEMORY_LIMIT = new ParseField("model_memory_limit"); + public static final ParseField CATEGORIZATION_EXAMPLES_LIMIT = new ParseField("categorization_examples_limit"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "analysis_limits", a -> new AnalysisLimits((Long) a[0], (Long) a[1])); + + static { + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), MODEL_MEMORY_LIMIT); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), CATEGORIZATION_EXAMPLES_LIMIT); + } + + /** + * It is initialised to null. + * A value of null or 0 will result to the default being used. + */ + private final Long modelMemoryLimit; + + /** + * It is initialised to null. + * A value of null will result to the default being used. + */ + private final Long categorizationExamplesLimit; + + public AnalysisLimits(Long modelMemoryLimit, Long categorizationExamplesLimit) { + this.modelMemoryLimit = modelMemoryLimit; + if (categorizationExamplesLimit != null && categorizationExamplesLimit < 0) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, CATEGORIZATION_EXAMPLES_LIMIT, 0, + categorizationExamplesLimit); + throw new IllegalArgumentException(msg); + } + this.categorizationExamplesLimit = categorizationExamplesLimit; + } + + public AnalysisLimits(StreamInput in) throws IOException { + this(in.readOptionalLong(), in.readOptionalLong()); + } + + /** + * Maximum size of the model in MB before the anomaly detector + * will drop new samples to prevent the model using any more + * memory + * + * @return The set memory limit or null if not set + */ + @Nullable + public Long getModelMemoryLimit() { + return modelMemoryLimit; + } + + /** + * Gets the limit to the number of examples that are stored per category + * + * @return the limit or null if not set + */ + @Nullable + public Long getCategorizationExamplesLimit() { + return categorizationExamplesLimit; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalLong(modelMemoryLimit); + out.writeOptionalLong(categorizationExamplesLimit); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (modelMemoryLimit != null) { + builder.field(MODEL_MEMORY_LIMIT.getPreferredName(), modelMemoryLimit); + } + if (categorizationExamplesLimit != null) { + builder.field(CATEGORIZATION_EXAMPLES_LIMIT.getPreferredName(), categorizationExamplesLimit); + } + builder.endObject(); + return builder; + } + + /** + * Overridden equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof AnalysisLimits == false) { + return false; + } + + AnalysisLimits that = (AnalysisLimits) other; + return Objects.equals(this.modelMemoryLimit, that.modelMemoryLimit) && + Objects.equals(this.categorizationExamplesLimit, that.categorizationExamplesLimit); + } + + @Override + public int hashCode() { + return Objects.hash(modelMemoryLimit, categorizationExamplesLimit); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Condition.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Condition.java new file mode 100644 index 00000000000..6a6c67a29df --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Condition.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * A class that describes a condition. + * The {@linkplain Operator} enum defines the available + * comparisons a condition can use. + */ +public class Condition extends ToXContentToBytes implements Writeable { + public static final ParseField CONDITION_FIELD = new ParseField("condition"); + public static final ParseField FILTER_VALUE_FIELD = new ParseField("value"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + CONDITION_FIELD.getPreferredName(), a -> new Condition((Operator) a[0], (String) a[1])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Operator.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, Operator.OPERATOR_FIELD, ValueType.STRING); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return p.text(); + } + if (p.currentToken() == XContentParser.Token.VALUE_NULL) { + return null; + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, FILTER_VALUE_FIELD, ValueType.STRING_OR_NULL); + } + + private final Operator op; + private final String filterValue; + + public Condition(StreamInput in) throws IOException { + op = Operator.readFromStream(in); + filterValue = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + op.writeTo(out); + out.writeOptionalString(filterValue); + } + + public Condition(Operator op, String filterValue) { + if (filterValue == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NULL)); + } + + if (op.expectsANumericArgument()) { + try { + Double.parseDouble(filterValue); + } catch (NumberFormatException nfe) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, filterValue); + throw new IllegalArgumentException(msg); + } + } else { + try { + Pattern.compile(filterValue); + } catch (PatternSyntaxException e) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX, filterValue); + throw new IllegalArgumentException(msg); + } + } + this.op = op; + this.filterValue = filterValue; + } + + public Operator getOperator() { + return op; + } + + public String getValue() { + return filterValue; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Operator.OPERATOR_FIELD.getPreferredName(), op); + builder.field(FILTER_VALUE_FIELD.getPreferredName(), filterValue); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(op, filterValue); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + Condition other = (Condition) obj; + return Objects.equals(this.op, other.op) && + Objects.equals(this.filterValue, other.filterValue); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Connective.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Connective.java new file mode 100644 index 00000000000..d99bc72d85b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Connective.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import java.io.IOException; +import java.util.Locale; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +public enum Connective implements Writeable { + OR, AND; + + /** + * Case-insensitive from string method. + * + * @param value + * String representation + * @return The connective type + */ + public static Connective fromString(String value) { + return Connective.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static Connective readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Connective ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DataDescription.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DataDescription.java new file mode 100644 index 00000000000..d2b60825f07 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DataDescription.java @@ -0,0 +1,352 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.DateTimeFormatterTimestampConverter; + +import java.io.IOException; +import java.time.ZoneOffset; +import java.util.Locale; +import java.util.Objects; + +/** + * Describes the format of the data used in the job and how it should + * be interpreted by autodetect. + *

+ * Data must either be in a textual delineated format (e.g. csv, tsv) or JSON + * the {@linkplain DataFormat} enum indicates which. {@link #getTimeField()} + * is the name of the field containing the timestamp and {@link #getTimeFormat()} + * is the format code for the date string in as described by + * {@link java.time.format.DateTimeFormatter}. The default quote character for + * delineated formats is {@value #DEFAULT_QUOTE_CHAR} but any other character can be + * used. + */ +public class DataDescription extends ToXContentToBytes implements Writeable { + /** + * Enum of the acceptable data formats. + */ + public enum DataFormat implements Writeable { + JSON, + DELIMITED; + + /** + * Delimited used to be called delineated. We keep supporting that for backwards + * compatibility. + */ + private static final String DEPRECATED_DELINEATED = "DELINEATED"; + + /** + * Case-insensitive from string method. + * Works with either JSON, json, etc. + * + * @param value String representation + * @return The data format + */ + public static DataFormat forString(String value) { + String valueUpperCase = value.toUpperCase(Locale.ROOT); + return DEPRECATED_DELINEATED.equals(valueUpperCase) ? DELIMITED : DataFormat + .valueOf(valueUpperCase); + } + + public static DataFormat readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown DataFormat ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + private static final ParseField DATA_DESCRIPTION_FIELD = new ParseField("data_description"); + private static final ParseField FORMAT_FIELD = new ParseField("format"); + private static final ParseField TIME_FIELD_NAME_FIELD = new ParseField("time_field"); + private static final ParseField TIME_FORMAT_FIELD = new ParseField("time_format"); + private static final ParseField FIELD_DELIMITER_FIELD = new ParseField("field_delimiter"); + private static final ParseField QUOTE_CHARACTER_FIELD = new ParseField("quote_character"); + + /** + * Special time format string for epoch times (seconds) + */ + public static final String EPOCH = "epoch"; + + /** + * Special time format string for epoch times (milli-seconds) + */ + public static final String EPOCH_MS = "epoch_ms"; + + /** + * By default autodetect expects the timestamp in a field with this name + */ + public static final String DEFAULT_TIME_FIELD = "time"; + + /** + * The default field delimiter expected by the native autodetect + * program. + */ + public static final char DEFAULT_DELIMITER = '\t'; + + /** + * Csv data must have this line ending + */ + public static final char LINE_ENDING = '\n'; + + /** + * The default quote character used to escape text in + * delineated data formats + */ + public static final char DEFAULT_QUOTE_CHAR = '"'; + + private final DataFormat dataFormat; + private final String timeFieldName; + private final String timeFormat; + private final char fieldDelimiter; + private final char quoteCharacter; + + public static final ObjectParser PARSER = + new ObjectParser<>(DATA_DESCRIPTION_FIELD.getPreferredName(), Builder::new); + + static { + PARSER.declareString(Builder::setFormat, FORMAT_FIELD); + PARSER.declareString(Builder::setTimeField, TIME_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setTimeFormat, TIME_FORMAT_FIELD); + PARSER.declareField(Builder::setFieldDelimiter, DataDescription::extractChar, FIELD_DELIMITER_FIELD, ValueType.STRING); + PARSER.declareField(Builder::setQuoteCharacter, DataDescription::extractChar, QUOTE_CHARACTER_FIELD, ValueType.STRING); + } + + public DataDescription(DataFormat dataFormat, String timeFieldName, String timeFormat, char fieldDelimiter, char quoteCharacter) { + this.dataFormat = dataFormat; + this.timeFieldName = timeFieldName; + this.timeFormat = timeFormat; + this.fieldDelimiter = fieldDelimiter; + this.quoteCharacter = quoteCharacter; + } + + public DataDescription(StreamInput in) throws IOException { + dataFormat = DataFormat.readFromStream(in); + timeFieldName = in.readString(); + timeFormat = in.readString(); + fieldDelimiter = (char) in.read(); + quoteCharacter = (char) in.read(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + dataFormat.writeTo(out); + out.writeString(timeFieldName); + out.writeString(timeFormat); + out.write(fieldDelimiter); + out.write(quoteCharacter); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FORMAT_FIELD.getPreferredName(), dataFormat); + builder.field(TIME_FIELD_NAME_FIELD.getPreferredName(), timeFieldName); + builder.field(TIME_FORMAT_FIELD.getPreferredName(), timeFormat); + builder.field(FIELD_DELIMITER_FIELD.getPreferredName(), String.valueOf(fieldDelimiter)); + builder.field(QUOTE_CHARACTER_FIELD.getPreferredName(), String.valueOf(quoteCharacter)); + builder.endObject(); + return builder; + } + + /** + * The format of the data to be processed. + * Defaults to {@link DataDescription.DataFormat#DELIMITED} + * + * @return The data format + */ + public DataFormat getFormat() { + return dataFormat; + } + + /** + * The name of the field containing the timestamp + * + * @return A String if set or null + */ + public String getTimeField() { + return timeFieldName; + } + + /** + * Either {@value #EPOCH}, {@value #EPOCH_MS} or a SimpleDateTime format string. + * If not set (is null or an empty string) or set to + * {@value #EPOCH} (the default) then the date is assumed to be in + * seconds from the epoch. + * + * @return A String if set or null + */ + public String getTimeFormat() { + return timeFormat; + } + + /** + * If the data is in a delineated format with a header e.g. csv or tsv + * this is the delimiter character used. This is only applicable if + * {@linkplain #getFormat()} is {@link DataDescription.DataFormat#DELIMITED}. + * The default value is {@value #DEFAULT_DELIMITER} + * + * @return A char + */ + public char getFieldDelimiter() { + return fieldDelimiter; + } + + /** + * The quote character used in delineated formats. + * Defaults to {@value #DEFAULT_QUOTE_CHAR} + * + * @return The delineated format quote character + */ + public char getQuoteCharacter() { + return quoteCharacter; + } + + /** + * Returns true if the data described by this object needs + * transforming before processing by autodetect. + * A transformation must be applied if either a timeformat is + * not in seconds since the epoch or the data is in Json format. + * + * @return True if the data should be transformed. + */ + public boolean transform() { + return dataFormat == DataFormat.JSON || + isTransformTime(); + } + + /** + * Return true if the time is in a format that needs transforming. + * Anytime format this isn't {@value #EPOCH} or null + * needs transforming. + * + * @return True if the time field needs to be transformed. + */ + public boolean isTransformTime() { + return timeFormat != null && !EPOCH.equals(timeFormat); + } + + /** + * Return true if the time format is {@value #EPOCH_MS} + * + * @return True if the date is in milli-seconds since the epoch. + */ + public boolean isEpochMs() { + return EPOCH_MS.equals(timeFormat); + } + + private static char extractChar(XContentParser parser) throws IOException { + if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { + String charStr = parser.text(); + if (charStr.length() != 1) { + throw new IllegalArgumentException("String must be a single character, found [" + charStr + "]"); + } + return charStr.charAt(0); + } + throw new IllegalArgumentException("Unsupported token [" + parser.currentToken() + "]"); + } + + /** + * Overridden equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof DataDescription == false) { + return false; + } + + DataDescription that = (DataDescription) other; + + return this.dataFormat == that.dataFormat && + this.quoteCharacter == that.quoteCharacter && + Objects.equals(this.timeFieldName, that.timeFieldName) && + Objects.equals(this.timeFormat, that.timeFormat) && + Objects.equals(this.fieldDelimiter, that.fieldDelimiter); + } + + @Override + public int hashCode() { + return Objects.hash(dataFormat, quoteCharacter, timeFieldName, + timeFormat, fieldDelimiter); + } + + public static class Builder { + + private DataFormat dataFormat = DataFormat.DELIMITED; + private String timeFieldName = DEFAULT_TIME_FIELD; + private String timeFormat = EPOCH; + private char fieldDelimiter = DEFAULT_DELIMITER; + private char quoteCharacter = DEFAULT_QUOTE_CHAR; + + public void setFormat(DataFormat format) { + dataFormat = ExceptionsHelper.requireNonNull(format, FORMAT_FIELD.getPreferredName() + " must not be null"); + } + + private void setFormat(String format) { + setFormat(DataFormat.forString(format)); + } + + public void setTimeField(String fieldName) { + timeFieldName = ExceptionsHelper.requireNonNull(fieldName, TIME_FIELD_NAME_FIELD.getPreferredName() + " must not be null"); + } + + public void setTimeFormat(String format) { + ExceptionsHelper.requireNonNull(format, TIME_FORMAT_FIELD.getPreferredName() + " must not be null"); + switch (format) { + case EPOCH: + case EPOCH_MS: + break; + default: + try { + DateTimeFormatterTimestampConverter.ofPattern(format, ZoneOffset.UTC); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, format)); + } + } + timeFormat = format; + } + + public void setFieldDelimiter(char delimiter) { + fieldDelimiter = delimiter; + } + + public void setQuoteCharacter(char value) { + quoteCharacter = value; + } + + public DataDescription build() { + return new DataDescription(dataFormat, timeFieldName, timeFormat, fieldDelimiter,quoteCharacter); + } + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DefaultDetectorDescription.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DefaultDetectorDescription.java new file mode 100644 index 00000000000..89179e0df88 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DefaultDetectorDescription.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.ml.utils.MlStrings; + + +public final class DefaultDetectorDescription { + private static final String BY_TOKEN = " by "; + private static final String OVER_TOKEN = " over "; + + private static final String USE_NULL_OPTION = " usenull="; + private static final String PARTITION_FIELD_OPTION = " partitionfield="; + private static final String EXCLUDE_FREQUENT_OPTION = " excludefrequent="; + + private DefaultDetectorDescription() { + // do nothing + } + + /** + * Returns the default description for the given {@code detector} + * + * @param detector the {@code Detector} for which a default description is requested + * @return the default description + */ + public static String of(Detector detector) { + StringBuilder sb = new StringBuilder(); + appendOn(detector, sb); + return sb.toString(); + } + + /** + * Appends to the given {@code StringBuilder} the default description + * for the given {@code detector} + * + * @param detector the {@code Detector} for which a default description is requested + * @param sb the {@code StringBuilder} to append to + */ + public static void appendOn(Detector detector, StringBuilder sb) { + if (isNotNullOrEmpty(detector.getFunction())) { + sb.append(detector.getFunction()); + if (isNotNullOrEmpty(detector.getFieldName())) { + sb.append('(').append(quoteField(detector.getFieldName())) + .append(')'); + } + } else if (isNotNullOrEmpty(detector.getFieldName())) { + sb.append(quoteField(detector.getFieldName())); + } + + if (isNotNullOrEmpty(detector.getByFieldName())) { + sb.append(BY_TOKEN).append(quoteField(detector.getByFieldName())); + } + + if (isNotNullOrEmpty(detector.getOverFieldName())) { + sb.append(OVER_TOKEN).append(quoteField(detector.getOverFieldName())); + } + + if (detector.isUseNull()) { + sb.append(USE_NULL_OPTION).append(detector.isUseNull()); + } + + if (isNotNullOrEmpty(detector.getPartitionFieldName())) { + sb.append(PARTITION_FIELD_OPTION).append(quoteField(detector.getPartitionFieldName())); + } + + if (detector.getExcludeFrequent() != null) { + sb.append(EXCLUDE_FREQUENT_OPTION).append(detector.getExcludeFrequent()); + } + } + + private static String quoteField(String field) { + return MlStrings.doubleQuoteIfNotAlphaNumeric(field); + } + + private static boolean isNotNullOrEmpty(String arg) { + return !Strings.isNullOrEmpty(arg); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DefaultFrequency.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DefaultFrequency.java new file mode 100644 index 00000000000..26ccebace87 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DefaultFrequency.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import java.time.Duration; + +/** + * Factory methods for a sensible default for the datafeed frequency + */ +public final class DefaultFrequency { + private static final int SECONDS_IN_MINUTE = 60; + private static final int TWO_MINS_SECONDS = 2 * SECONDS_IN_MINUTE; + private static final int TWENTY_MINS_SECONDS = 20 * SECONDS_IN_MINUTE; + private static final int HALF_DAY_SECONDS = 12 * 60 * SECONDS_IN_MINUTE; + private static final Duration TEN_MINUTES = Duration.ofMinutes(10); + private static final Duration ONE_HOUR = Duration.ofHours(1); + + private DefaultFrequency() { + // Do nothing + } + + /** + * Creates a sensible default frequency for a given bucket span. + *

+ * The default depends on the bucket span: + *

    + *
  • <= 2 mins -> 1 min
    • + *
  • <= 20 mins -> bucket span / 2
    • + *
  • <= 12 hours -> 10 mins
    • + *
  • > 12 hours -> 1 hour
    • + *
+ * + * @param bucketSpanSeconds the bucket span in seconds + * @return the default frequency + */ + public static Duration ofBucketSpan(long bucketSpanSeconds) { + if (bucketSpanSeconds <= 0) { + throw new IllegalArgumentException("Bucket span has to be > 0"); + } + + if (bucketSpanSeconds <= TWO_MINS_SECONDS) { + return Duration.ofSeconds(SECONDS_IN_MINUTE); + } + if (bucketSpanSeconds <= TWENTY_MINS_SECONDS) { + return Duration.ofSeconds(bucketSpanSeconds / 2); + } + if (bucketSpanSeconds <= HALF_DAY_SECONDS) { + return TEN_MINUTES; + } + return ONE_HOUR; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java new file mode 100644 index 00000000000..fb261d2c832 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +public class DetectionRule extends ToXContentToBytes implements Writeable { + public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule"); + public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name"); + public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value"); + public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective"); + public static final ParseField RULE_CONDITIONS_FIELD = new ParseField("rule_conditions"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + DETECTION_RULE_FIELD.getPreferredName(), + arr -> { + @SuppressWarnings("unchecked") + List rules = (List) arr[3]; + return new DetectionRule((String) arr[0], (String) arr[1], (Connective) arr[2], rules); + } + ); + + static { + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_FIELD_NAME_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_FIELD_VALUE_FIELD); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Connective.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), + (parser, parseFieldMatcher) -> RuleCondition.PARSER.apply(parser, parseFieldMatcher), RULE_CONDITIONS_FIELD); + } + + private final RuleAction ruleAction = RuleAction.FILTER_RESULTS; + private final String targetFieldName; + private final String targetFieldValue; + private final Connective conditionsConnective; + private final List ruleConditions; + + public DetectionRule(StreamInput in) throws IOException { + conditionsConnective = Connective.readFromStream(in); + int size = in.readVInt(); + ruleConditions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + ruleConditions.add(new RuleCondition(in)); + } + targetFieldName = in.readOptionalString(); + targetFieldValue = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + conditionsConnective.writeTo(out); + out.writeVInt(ruleConditions.size()); + for (RuleCondition condition : ruleConditions) { + condition.writeTo(out); + } + out.writeOptionalString(targetFieldName); + out.writeOptionalString(targetFieldValue); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective); + builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions); + if (targetFieldName != null) { + builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName); + } + if (targetFieldValue != null) { + builder.field(TARGET_FIELD_VALUE_FIELD.getPreferredName(), targetFieldValue); + } + builder.endObject(); + return builder; + } + + public DetectionRule(String targetFieldName, String targetFieldValue, Connective conditionsConnective, + List ruleConditions) { + if (targetFieldValue != null && targetFieldName == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue); + throw new IllegalArgumentException(msg); + } + if (ruleConditions == null || ruleConditions.isEmpty()) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION); + throw new IllegalArgumentException(msg); + } + for (RuleCondition condition : ruleConditions) { + if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, + DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName()); + throw new IllegalArgumentException(msg); + } + } + + this.targetFieldName = targetFieldName; + this.targetFieldValue = targetFieldValue; + this.conditionsConnective = conditionsConnective != null ? conditionsConnective : Connective.OR; + this.ruleConditions = Collections.unmodifiableList(ruleConditions); + } + + public RuleAction getRuleAction() { + return ruleAction; + } + + public String getTargetFieldName() { + return targetFieldName; + } + + public String getTargetFieldValue() { + return targetFieldValue; + } + + public Connective getConditionsConnective() { + return conditionsConnective; + } + + public List getRuleConditions() { + return ruleConditions; + } + + public Set extractReferencedFilters() { + return ruleConditions.stream().map(RuleCondition::getValueFilter).filter(Objects::nonNull).collect(Collectors.toSet()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof DetectionRule == false) { + return false; + } + + DetectionRule other = (DetectionRule) obj; + return Objects.equals(ruleAction, other.ruleAction) && Objects.equals(targetFieldName, other.targetFieldName) + && Objects.equals(targetFieldValue, other.targetFieldValue) + && Objects.equals(conditionsConnective, other.conditionsConnective) && Objects.equals(ruleConditions, other.ruleConditions); + } + + @Override + public int hashCode() { + return Objects.hash(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java new file mode 100644 index 00000000000..c6f3f108771 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java @@ -0,0 +1,770 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + + +/** + * Defines the fields to be used in the analysis. + * fieldname must be set and only one of byFieldName + * and overFieldName should be set. + */ +public class Detector extends ToXContentToBytes implements Writeable { + + public enum ExcludeFrequent implements Writeable { + ALL, + NONE, + BY, + OVER; + + /** + * Case-insensitive from string method. + * Works with either JSON, json, etc. + * + * @param value String representation + * @return The data format + */ + public static ExcludeFrequent forString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static ExcludeFrequent readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown ExcludeFrequent ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + public static final ParseField DETECTOR_DESCRIPTION_FIELD = new ParseField("detector_description"); + public static final ParseField FUNCTION_FIELD = new ParseField("function"); + public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name"); + public static final ParseField BY_FIELD_NAME_FIELD = new ParseField("by_field_name"); + public static final ParseField OVER_FIELD_NAME_FIELD = new ParseField("over_field_name"); + public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name"); + public static final ParseField USE_NULL_FIELD = new ParseField("use_null"); + public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent"); + public static final ParseField DETECTOR_RULES_FIELD = new ParseField("detector_rules"); + + public static final ObjectParser PARSER = new ObjectParser<>("detector", Builder::new); + + static { + PARSER.declareString(Builder::setDetectorDescription, DETECTOR_DESCRIPTION_FIELD); + PARSER.declareString(Builder::setFunction, FUNCTION_FIELD); + PARSER.declareString(Builder::setFieldName, FIELD_NAME_FIELD); + PARSER.declareString(Builder::setByFieldName, BY_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setOverFieldName, OVER_FIELD_NAME_FIELD); + PARSER.declareString(Builder::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD); + PARSER.declareBoolean(Builder::setUseNull, USE_NULL_FIELD); + PARSER.declareField(Builder::setExcludeFrequent, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return ExcludeFrequent.forString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING); + PARSER.declareObjectArray(Builder::setDetectorRules, DetectionRule.PARSER, DETECTOR_RULES_FIELD); + } + + public static final String COUNT = "count"; + public static final String HIGH_COUNT = "high_count"; + public static final String LOW_COUNT = "low_count"; + public static final String NON_ZERO_COUNT = "non_zero_count"; + public static final String LOW_NON_ZERO_COUNT = "low_non_zero_count"; + public static final String HIGH_NON_ZERO_COUNT = "high_non_zero_count"; + public static final String NZC = "nzc"; + public static final String LOW_NZC = "low_nzc"; + public static final String HIGH_NZC = "high_nzc"; + public static final String DISTINCT_COUNT = "distinct_count"; + public static final String LOW_DISTINCT_COUNT = "low_distinct_count"; + public static final String HIGH_DISTINCT_COUNT = "high_distinct_count"; + public static final String DC = "dc"; + public static final String LOW_DC = "low_dc"; + public static final String HIGH_DC = "high_dc"; + public static final String RARE = "rare"; + public static final String FREQ_RARE = "freq_rare"; + public static final String INFO_CONTENT = "info_content"; + public static final String LOW_INFO_CONTENT = "low_info_content"; + public static final String HIGH_INFO_CONTENT = "high_info_content"; + public static final String METRIC = "metric"; + public static final String MEAN = "mean"; + public static final String MEDIAN = "median"; + public static final String HIGH_MEAN = "high_mean"; + public static final String LOW_MEAN = "low_mean"; + public static final String AVG = "avg"; + public static final String HIGH_AVG = "high_avg"; + public static final String LOW_AVG = "low_avg"; + public static final String MIN = "min"; + public static final String MAX = "max"; + public static final String SUM = "sum"; + public static final String LOW_SUM = "low_sum"; + public static final String HIGH_SUM = "high_sum"; + public static final String NON_NULL_SUM = "non_null_sum"; + public static final String LOW_NON_NULL_SUM = "low_non_null_sum"; + public static final String HIGH_NON_NULL_SUM = "high_non_null_sum"; + /** + * Population variance is called varp to match Splunk + */ + public static final String POPULATION_VARIANCE = "varp"; + public static final String LOW_POPULATION_VARIANCE = "low_varp"; + public static final String HIGH_POPULATION_VARIANCE = "high_varp"; + public static final String TIME_OF_DAY = "time_of_day"; + public static final String TIME_OF_WEEK = "time_of_week"; + public static final String LAT_LONG = "lat_long"; + + + /** + * The set of valid function names. + */ + public static final Set ANALYSIS_FUNCTIONS = + new HashSet<>(Arrays.asList( + // The convention here is that synonyms (only) go on the same line + COUNT, + HIGH_COUNT, + LOW_COUNT, + NON_ZERO_COUNT, NZC, + LOW_NON_ZERO_COUNT, LOW_NZC, + HIGH_NON_ZERO_COUNT, HIGH_NZC, + DISTINCT_COUNT, DC, + LOW_DISTINCT_COUNT, LOW_DC, + HIGH_DISTINCT_COUNT, HIGH_DC, + RARE, + FREQ_RARE, + INFO_CONTENT, + LOW_INFO_CONTENT, + HIGH_INFO_CONTENT, + METRIC, + MEAN, AVG, + HIGH_MEAN, HIGH_AVG, + LOW_MEAN, LOW_AVG, + MEDIAN, + MIN, + MAX, + SUM, + LOW_SUM, + HIGH_SUM, + NON_NULL_SUM, + LOW_NON_NULL_SUM, + HIGH_NON_NULL_SUM, + POPULATION_VARIANCE, + LOW_POPULATION_VARIANCE, + HIGH_POPULATION_VARIANCE, + TIME_OF_DAY, + TIME_OF_WEEK, + LAT_LONG + )); + + /** + * The set of functions that do not require a field, by field or over field + */ + public static final Set COUNT_WITHOUT_FIELD_FUNCTIONS = + new HashSet<>(Arrays.asList( + COUNT, + HIGH_COUNT, + LOW_COUNT, + NON_ZERO_COUNT, NZC, + LOW_NON_ZERO_COUNT, LOW_NZC, + HIGH_NON_ZERO_COUNT, HIGH_NZC, + TIME_OF_DAY, + TIME_OF_WEEK + )); + + /** + * The set of functions that require a fieldname + */ + public static final Set FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + DISTINCT_COUNT, DC, + LOW_DISTINCT_COUNT, LOW_DC, + HIGH_DISTINCT_COUNT, HIGH_DC, + INFO_CONTENT, + LOW_INFO_CONTENT, + HIGH_INFO_CONTENT, + METRIC, + MEAN, AVG, + HIGH_MEAN, HIGH_AVG, + LOW_MEAN, LOW_AVG, + MEDIAN, + MIN, + MAX, + SUM, + LOW_SUM, + HIGH_SUM, + NON_NULL_SUM, + LOW_NON_NULL_SUM, + HIGH_NON_NULL_SUM, + POPULATION_VARIANCE, + LOW_POPULATION_VARIANCE, + HIGH_POPULATION_VARIANCE, + LAT_LONG + )); + + /** + * The set of functions that require a by fieldname + */ + public static final Set BY_FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + RARE, + FREQ_RARE + )); + + /** + * The set of functions that require a over fieldname + */ + public static final Set OVER_FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + FREQ_RARE + )); + + /** + * The set of functions that cannot have a by fieldname + */ + public static final Set NO_BY_FIELD_NAME_FUNCTIONS = + new HashSet<>(); + + /** + * The set of functions that cannot have an over fieldname + */ + public static final Set NO_OVER_FIELD_NAME_FUNCTIONS = + new HashSet<>(Arrays.asList( + NON_ZERO_COUNT, NZC, + LOW_NON_ZERO_COUNT, LOW_NZC, + HIGH_NON_ZERO_COUNT, HIGH_NZC + )); + + /** + * The set of functions that must not be used with overlapping buckets + */ + public static final Set NO_OVERLAPPING_BUCKETS_FUNCTIONS = + new HashSet<>(Arrays.asList( + RARE, + FREQ_RARE + )); + + /** + * The set of functions that should not be used with overlapping buckets + * as they gain no benefit but have overhead + */ + public static final Set OVERLAPPING_BUCKETS_FUNCTIONS_NOT_NEEDED = + new HashSet<>(Arrays.asList( + MIN, + MAX, + TIME_OF_DAY, + TIME_OF_WEEK + )); + + /** + * field names cannot contain any of these characters + * ", \ + */ + public static final Character[] PROHIBITED_FIELDNAME_CHARACTERS = {'"', '\\'}; + public static final String PROHIBITED = String.join(",", + Arrays.stream(PROHIBITED_FIELDNAME_CHARACTERS).map( + c -> Character.toString(c)).collect(Collectors.toList())); + + + private final String detectorDescription; + private final String function; + private final String fieldName; + private final String byFieldName; + private final String overFieldName; + private final String partitionFieldName; + private final boolean useNull; + private final ExcludeFrequent excludeFrequent; + private final List detectorRules; + + public Detector(StreamInput in) throws IOException { + detectorDescription = in.readString(); + function = in.readString(); + fieldName = in.readOptionalString(); + byFieldName = in.readOptionalString(); + overFieldName = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + useNull = in.readBoolean(); + excludeFrequent = in.readBoolean() ? ExcludeFrequent.readFromStream(in) : null; + detectorRules = in.readList(DetectionRule::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(detectorDescription); + out.writeString(function); + out.writeOptionalString(fieldName); + out.writeOptionalString(byFieldName); + out.writeOptionalString(overFieldName); + out.writeOptionalString(partitionFieldName); + out.writeBoolean(useNull); + if (excludeFrequent != null) { + out.writeBoolean(true); + excludeFrequent.writeTo(out); + } else { + out.writeBoolean(false); + } + out.writeList(detectorRules); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(DETECTOR_DESCRIPTION_FIELD.getPreferredName(), detectorDescription); + builder.field(FUNCTION_FIELD.getPreferredName(), function); + if (fieldName != null) { + builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME_FIELD.getPreferredName(), byFieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME_FIELD.getPreferredName(), overFieldName); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName); + } + if (useNull) { + builder.field(USE_NULL_FIELD.getPreferredName(), useNull); + } + if (excludeFrequent != null) { + builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent); + } + builder.field(DETECTOR_RULES_FIELD.getPreferredName(), detectorRules); + builder.endObject(); + return builder; + } + + private Detector(String detectorDescription, String function, String fieldName, String byFieldName, String overFieldName, + String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List detectorRules) { + this.function = function; + this.fieldName = fieldName; + this.byFieldName = byFieldName; + this.overFieldName = overFieldName; + this.partitionFieldName = partitionFieldName; + this.useNull = useNull; + this.excludeFrequent = excludeFrequent; + // REMOVE THIS LINE WHEN REMOVING JACKSON_DATABIND: + detectorRules = detectorRules != null ? detectorRules : Collections.emptyList(); + this.detectorRules = Collections.unmodifiableList(detectorRules); + this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this); + } + + public String getDetectorDescription() { + return detectorDescription; + } + + /** + * The analysis function used e.g. count, rare, min etc. There is no + * validation to check this value is one a predefined set + * + * @return The function or null if not set + */ + public String getFunction() { + return function; + } + + /** + * The Analysis field + * + * @return The field to analyse + */ + public String getFieldName() { + return fieldName; + } + + /** + * The 'by' field or null if not set. + * + * @return The 'by' field + */ + public String getByFieldName() { + return byFieldName; + } + + /** + * The 'over' field or null if not set. + * + * @return The 'over' field + */ + public String getOverFieldName() { + return overFieldName; + } + + /** + * Segments the analysis along another field to have completely + * independent baselines for each instance of partitionfield + * + * @return The Partition Field + */ + public String getPartitionFieldName() { + return partitionFieldName; + } + + /** + * Where there isn't a value for the 'by' or 'over' field should a new + * series be used as the 'null' series. + * + * @return true if the 'null' series should be created + */ + public boolean isUseNull() { + return useNull; + } + + /** + * Excludes frequently-occuring metrics from the analysis; + * can apply to 'by' field, 'over' field, or both + * + * @return the value that the user set + */ + public ExcludeFrequent getExcludeFrequent() { + return excludeFrequent; + } + + public List getDetectorRules() { + return detectorRules; + } + + /** + * Returns a list with the byFieldName, overFieldName and partitionFieldName that are not null + * + * @return a list with the byFieldName, overFieldName and partitionFieldName that are not null + */ + public List extractAnalysisFields() { + List analysisFields = Arrays.asList(getByFieldName(), + getOverFieldName(), getPartitionFieldName()); + return analysisFields.stream().filter(item -> item != null).collect(Collectors.toList()); + } + + public Set extractReferencedFilters() { + return detectorRules == null ? Collections.emptySet() + : detectorRules.stream().map(DetectionRule::extractReferencedFilters) + .flatMap(Set::stream).collect(Collectors.toSet()); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Detector == false) { + return false; + } + + Detector that = (Detector) other; + + return Objects.equals(this.detectorDescription, that.detectorDescription) && + Objects.equals(this.function, that.function) && + Objects.equals(this.fieldName, that.fieldName) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.useNull, that.useNull) && + Objects.equals(this.excludeFrequent, that.excludeFrequent) && + Objects.equals(this.detectorRules, that.detectorRules); + } + + @Override + public int hashCode() { + return Objects.hash(detectorDescription, function, fieldName, byFieldName, + overFieldName, partitionFieldName, useNull, excludeFrequent, + detectorRules); + } + + public static class Builder { + + /** + * Functions that do not support rules: + *
    + *
  • lat_long - because it is a multivariate feature + *
  • metric - because having the same conditions on min,max,mean is + * error-prone + *
+ */ + static final Set FUNCTIONS_WITHOUT_RULE_SUPPORT = new HashSet<>(Arrays.asList(Detector.LAT_LONG, Detector.METRIC)); + + private String detectorDescription; + private String function; + private String fieldName; + private String byFieldName; + private String overFieldName; + private String partitionFieldName; + private boolean useNull = false; + private ExcludeFrequent excludeFrequent; + private List detectorRules = Collections.emptyList(); + + public Builder() { + } + + public Builder(Detector detector) { + detectorDescription = detector.detectorDescription; + function = detector.function; + fieldName = detector.fieldName; + byFieldName = detector.byFieldName; + overFieldName = detector.overFieldName; + partitionFieldName = detector.partitionFieldName; + useNull = detector.useNull; + excludeFrequent = detector.excludeFrequent; + detectorRules = new ArrayList<>(detector.detectorRules.size()); + for (DetectionRule rule : detector.getDetectorRules()) { + detectorRules.add(rule); + } + } + + public Builder(String function, String fieldName) { + this.function = function; + this.fieldName = fieldName; + } + + public void setDetectorDescription(String detectorDescription) { + this.detectorDescription = detectorDescription; + } + + public void setFunction(String function) { + this.function = function; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public void setByFieldName(String byFieldName) { + this.byFieldName = byFieldName; + } + + public void setOverFieldName(String overFieldName) { + this.overFieldName = overFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + } + + public void setUseNull(boolean useNull) { + this.useNull = useNull; + } + + public void setExcludeFrequent(ExcludeFrequent excludeFrequent) { + this.excludeFrequent = excludeFrequent; + } + + public void setDetectorRules(List detectorRules) { + this.detectorRules = detectorRules; + } + + public List getDetectorRules() { + return detectorRules; + } + + public Detector build() { + return build(false); + } + + public Detector build(boolean isSummarised) { + boolean emptyField = Strings.isEmpty(fieldName); + boolean emptyByField = Strings.isEmpty(byFieldName); + boolean emptyOverField = Strings.isEmpty(overFieldName); + if (Detector.ANALYSIS_FUNCTIONS.contains(function) == false) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_UNKNOWN_FUNCTION, function)); + } + + if (emptyField && emptyByField && emptyOverField) { + if (!Detector.COUNT_WITHOUT_FIELD_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_NO_ANALYSIS_FIELD_NOT_COUNT)); + } + } + + if (isSummarised && Detector.METRIC.equals(function)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_INCOMPATIBLE_PRESUMMARIZED, Detector.METRIC)); + } + + // check functions have required fields + + if (emptyField && Detector.FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_REQUIRES_FIELDNAME, function)); + } + + if (!emptyField && (Detector.FIELD_NAME_FUNCTIONS.contains(function) == false)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FIELDNAME_INCOMPATIBLE_FUNCTION, function)); + } + + if (emptyByField && Detector.BY_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_REQUIRES_BYFIELD, function)); + } + + if (!emptyByField && Detector.NO_BY_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_BYFIELD_INCOMPATIBLE_FUNCTION, function)); + } + + if (emptyOverField && Detector.OVER_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FUNCTION_REQUIRES_OVERFIELD, function)); + } + + if (!emptyOverField && Detector.NO_OVER_FIELD_NAME_FUNCTIONS.contains(function)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_OVERFIELD_INCOMPATIBLE_FUNCTION, function)); + } + + // field names cannot contain certain characters + String[] fields = { fieldName, byFieldName, overFieldName, partitionFieldName }; + for (String field : fields) { + verifyFieldName(field); + } + + String function = this.function == null ? Detector.METRIC : this.function; + if (detectorRules.isEmpty() == false) { + if (FUNCTIONS_WITHOUT_RULE_SUPPORT.contains(function)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function); + throw new IllegalArgumentException(msg); + } + for (DetectionRule rule : detectorRules) { + checkScoping(rule); + } + } + + return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, + useNull, excludeFrequent, detectorRules); + } + + public List extractAnalysisFields() { + List analysisFields = Arrays.asList(byFieldName, + overFieldName, partitionFieldName); + return analysisFields.stream().filter(item -> item != null).collect(Collectors.toList()); + } + + /** + * Check that the characters used in a field name will not cause problems. + * + * @param field + * The field name to be validated + * @return true + */ + public static boolean verifyFieldName(String field) throws ElasticsearchParseException { + if (field != null && containsInvalidChar(field)) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_INVALID_FIELDNAME_CHARS, field, Detector.PROHIBITED)); + + } + return true; + } + + private static boolean containsInvalidChar(String field) { + for (Character ch : Detector.PROHIBITED_FIELDNAME_CHARACTERS) { + if (field.indexOf(ch) >= 0) { + return true; + } + } + return field.chars().anyMatch(ch -> Character.isISOControl(ch)); + } + + private void checkScoping(DetectionRule rule) throws ElasticsearchParseException { + String targetFieldName = rule.getTargetFieldName(); + checkTargetFieldNameIsValid(extractAnalysisFields(), targetFieldName); + List validOptions = getValidFieldNameOptions(rule); + for (RuleCondition condition : rule.getRuleConditions()) { + if (!validOptions.contains(condition.getFieldName())) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, validOptions, + condition.getFieldName()); + throw new IllegalArgumentException(msg); + } + } + } + + private void checkTargetFieldNameIsValid(List analysisFields, String targetFieldName) + throws ElasticsearchParseException { + if (targetFieldName != null && !analysisFields.contains(targetFieldName)) { + String msg = + Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME, analysisFields, targetFieldName); + throw new IllegalArgumentException(msg); + } + } + + private List getValidFieldNameOptions(DetectionRule rule) { + List result = new ArrayList<>(); + if (overFieldName != null) { + result.add(byFieldName == null ? overFieldName : byFieldName); + } else if (byFieldName != null) { + result.add(byFieldName); + } + + if (rule.getTargetFieldName() != null) { + ScopingLevel targetLevel = ScopingLevel.from(this, rule.getTargetFieldName()); + result = result.stream().filter(field -> targetLevel.isHigherThan(ScopingLevel.from(this, field))) + .collect(Collectors.toList()); + } + + if (isEmptyFieldNameAllowed(rule)) { + result.add(null); + } + return result; + } + + private boolean isEmptyFieldNameAllowed(DetectionRule rule) { + List analysisFields = extractAnalysisFields(); + return analysisFields.isEmpty() || (rule.getTargetFieldName() != null && analysisFields.size() == 1); + } + + enum ScopingLevel { + PARTITION(3), + OVER(2), + BY(1); + + int level; + + ScopingLevel(int level) { + this.level = level; + } + + boolean isHigherThan(ScopingLevel other) { + return level > other.level; + } + + static ScopingLevel from(Detector.Builder detector, String fieldName) { + if (fieldName.equals(detector.partitionFieldName)) { + return ScopingLevel.PARTITION; + } + if (fieldName.equals(detector.overFieldName)) { + return ScopingLevel.OVER; + } + if (fieldName.equals(detector.byFieldName)) { + return ScopingLevel.BY; + } + throw new IllegalArgumentException( + "fieldName '" + fieldName + "' does not match an analysis field"); + } + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/IgnoreDowntime.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/IgnoreDowntime.java new file mode 100644 index 00000000000..2106ccc8a04 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/IgnoreDowntime.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum IgnoreDowntime implements Writeable { + + NEVER, ONCE, ALWAYS; + + /** + *

+ * Parses a string and returns the corresponding enum value. + *

+ *

+ * The method differs from {@link #valueOf(String)} by being + * able to handle leading/trailing whitespace and being case + * insensitive. + *

+ *

+ * If there is no match {@link IllegalArgumentException} is thrown. + *

+ * + * @param value A String that should match one of the enum values + * @return the matching enum value + */ + public static IgnoreDowntime fromString(String value) { + return valueOf(value.trim().toUpperCase(Locale.ROOT)); + } + + public static IgnoreDowntime fromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown public enum IgnoreDowntime {\n ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java new file mode 100644 index 00000000000..c581c355d3b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Job.java @@ -0,0 +1,672 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.utils.MlStrings; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +/** + * This class represents a configured and created Job. The creation time is set + * to the time the object was constructed, state is set to + * {@link JobState#OPENING} and the finished time and last data time fields are + * {@code null} until the job has seen some data or it is finished respectively. + * If the job was created to read data from a list of files FileUrls will be a + * non-empty list else the expects data to be streamed to it. + */ +public class Job extends AbstractDiffable implements Writeable, ToXContent { + + public static final String TYPE = "job"; + + /* + * Field names used in serialization + */ + public static final ParseField ID = new ParseField("job_id"); + public static final ParseField ANALYSIS_CONFIG = new ParseField("analysis_config"); + public static final ParseField ANALYSIS_LIMITS = new ParseField("analysis_limits"); + public static final ParseField CREATE_TIME = new ParseField("create_time"); + public static final ParseField CUSTOM_SETTINGS = new ParseField("custom_settings"); + public static final ParseField DATA_DESCRIPTION = new ParseField("data_description"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField FINISHED_TIME = new ParseField("finished_time"); + public static final ParseField IGNORE_DOWNTIME = new ParseField("ignore_downtime"); + public static final ParseField LAST_DATA_TIME = new ParseField("last_data_time"); + public static final ParseField MODEL_DEBUG_CONFIG = new ParseField("model_debug_config"); + public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days"); + public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval"); + public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days"); + public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days"); + public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id"); + public static final ParseField INDEX_NAME = new ParseField("index_name"); + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("jobs"); + + public static final String ALL = "_all"; + + public static final ObjectParser PARSER = new ObjectParser<>("job_details", Builder::new); + + public static final int MAX_JOB_ID_LENGTH = 64; + public static final long MIN_BACKGROUND_PERSIST_INTERVAL = 3600; + + static { + PARSER.declareString(Builder::setId, ID); + PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION); + PARSER.declareField(Builder::setCreateTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + CREATE_TIME.getPreferredName() + "]"); + }, CREATE_TIME, ValueType.VALUE); + PARSER.declareField(Builder::setFinishedTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + FINISHED_TIME.getPreferredName() + "]"); + }, FINISHED_TIME, ValueType.VALUE); + PARSER.declareField(Builder::setLastDataTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LAST_DATA_TIME.getPreferredName() + "]"); + }, LAST_DATA_TIME, ValueType.VALUE); + PARSER.declareObject(Builder::setAnalysisConfig, AnalysisConfig.PARSER, ANALYSIS_CONFIG); + PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.PARSER, ANALYSIS_LIMITS); + PARSER.declareObject(Builder::setDataDescription, DataDescription.PARSER, DATA_DESCRIPTION); + PARSER.declareObject(Builder::setModelDebugConfig, ModelDebugConfig.PARSER, MODEL_DEBUG_CONFIG); + PARSER.declareField(Builder::setIgnoreDowntime, (p, c) -> IgnoreDowntime.fromString(p.text()), IGNORE_DOWNTIME, ValueType.STRING); + PARSER.declareLong(Builder::setRenormalizationWindowDays, RENORMALIZATION_WINDOW_DAYS); + PARSER.declareLong(Builder::setBackgroundPersistInterval, BACKGROUND_PERSIST_INTERVAL); + PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS); + PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS); + PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), CUSTOM_SETTINGS, ValueType.OBJECT); + PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID); + PARSER.declareString(Builder::setIndexName, INDEX_NAME); + } + + private final String jobId; + private final String description; + // NORELEASE: Use Jodatime instead + private final Date createTime; + private final Date finishedTime; + private final Date lastDataTime; + private final AnalysisConfig analysisConfig; + private final AnalysisLimits analysisLimits; + private final DataDescription dataDescription; + private final ModelDebugConfig modelDebugConfig; + private final IgnoreDowntime ignoreDowntime; + private final Long renormalizationWindowDays; + private final Long backgroundPersistInterval; + private final Long modelSnapshotRetentionDays; + private final Long resultsRetentionDays; + private final Map customSettings; + private final String modelSnapshotId; + private final String indexName; + + public Job(String jobId, String description, Date createTime, Date finishedTime, Date lastDataTime, + AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription, + ModelDebugConfig modelDebugConfig, IgnoreDowntime ignoreDowntime, + Long renormalizationWindowDays, Long backgroundPersistInterval, Long modelSnapshotRetentionDays, Long resultsRetentionDays, + Map customSettings, String modelSnapshotId, String indexName) { + + if (analysisConfig == null) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_MISSING_ANALYSISCONFIG)); + } + + checkValueNotLessThan(0, "renormalizationWindowDays", renormalizationWindowDays); + checkValueNotLessThan(MIN_BACKGROUND_PERSIST_INTERVAL, "backgroundPersistInterval", backgroundPersistInterval); + checkValueNotLessThan(0, "modelSnapshotRetentionDays", modelSnapshotRetentionDays); + checkValueNotLessThan(0, "resultsRetentionDays", resultsRetentionDays); + + if (!MlStrings.isValidId(jobId)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_ID, ID.getPreferredName(), jobId)); + } + if (jobId.length() > MAX_JOB_ID_LENGTH) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_ID_TOO_LONG, MAX_JOB_ID_LENGTH)); + } + + if (Strings.isNullOrEmpty(indexName)) { + indexName = jobId; + } else if (!MlStrings.isValidId(indexName)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.INVALID_ID, INDEX_NAME.getPreferredName())); + } + + this.jobId = jobId; + this.description = description; + this.createTime = createTime; + this.finishedTime = finishedTime; + this.lastDataTime = lastDataTime; + this.analysisConfig = analysisConfig; + this.analysisLimits = analysisLimits; + this.dataDescription = dataDescription; + this.modelDebugConfig = modelDebugConfig; + this.ignoreDowntime = ignoreDowntime; + this.renormalizationWindowDays = renormalizationWindowDays; + this.backgroundPersistInterval = backgroundPersistInterval; + this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.resultsRetentionDays = resultsRetentionDays; + this.customSettings = customSettings; + this.modelSnapshotId = modelSnapshotId; + this.indexName = indexName; + } + + public Job(StreamInput in) throws IOException { + jobId = in.readString(); + description = in.readOptionalString(); + createTime = new Date(in.readVLong()); + finishedTime = in.readBoolean() ? new Date(in.readVLong()) : null; + lastDataTime = in.readBoolean() ? new Date(in.readVLong()) : null; + analysisConfig = new AnalysisConfig(in); + analysisLimits = in.readOptionalWriteable(AnalysisLimits::new); + dataDescription = in.readOptionalWriteable(DataDescription::new); + modelDebugConfig = in.readOptionalWriteable(ModelDebugConfig::new); + ignoreDowntime = in.readOptionalWriteable(IgnoreDowntime::fromStream); + renormalizationWindowDays = in.readOptionalLong(); + backgroundPersistInterval = in.readOptionalLong(); + modelSnapshotRetentionDays = in.readOptionalLong(); + resultsRetentionDays = in.readOptionalLong(); + customSettings = in.readMap(); + modelSnapshotId = in.readOptionalString(); + indexName = in.readString(); + } + + /** + * Return the Job Id. + * + * @return The job Id string + */ + public String getId() { + return jobId; + } + + /** + * The name of the index storing the job's results and state. + * This defaults to {@link #getId()} if a specific index name is not set. + * @return The job's index name + */ + public String getIndexName() { + return indexName; + } + + /** + * The job description + * + * @return job description + */ + public String getDescription() { + return description; + } + + /** + * The Job creation time. This name is preferred when serialising to the + * REST API. + * + * @return The date the job was created + */ + public Date getCreateTime() { + return createTime; + } + + /** + * The Job creation time. This name is preferred when serialising to the + * data store. + * + * @return The date the job was created + */ + public Date getAtTimestamp() { + return createTime; + } + + /** + * The time the job was finished or null if not finished. + * + * @return The date the job was last retired or null + */ + public Date getFinishedTime() { + return finishedTime; + } + + /** + * The last time data was uploaded to the job or null if no + * data has been seen. + * + * @return The date at which the last data was processed + */ + public Date getLastDataTime() { + return lastDataTime; + } + + /** + * The analysis configuration object + * + * @return The AnalysisConfig + */ + public AnalysisConfig getAnalysisConfig() { + return analysisConfig; + } + + /** + * The analysis options object + * + * @return The AnalysisLimits + */ + public AnalysisLimits getAnalysisLimits() { + return analysisLimits; + } + + public IgnoreDowntime getIgnoreDowntime() { + return ignoreDowntime; + } + + public ModelDebugConfig getModelDebugConfig() { + return modelDebugConfig; + } + + /** + * If not set the input data is assumed to be csv with a '_time' field in + * epoch format. + * + * @return A DataDescription or null + * @see DataDescription + */ + public DataDescription getDataDescription() { + return dataDescription; + } + + /** + * The duration of the renormalization window in days + * + * @return renormalization window in days + */ + public Long getRenormalizationWindowDays() { + return renormalizationWindowDays; + } + + /** + * The background persistence interval in seconds + * + * @return background persistence interval in seconds + */ + public Long getBackgroundPersistInterval() { + return backgroundPersistInterval; + } + + public Long getModelSnapshotRetentionDays() { + return modelSnapshotRetentionDays; + } + + public Long getResultsRetentionDays() { + return resultsRetentionDays; + } + + public Map getCustomSettings() { + return customSettings; + } + + public String getModelSnapshotId() { + return modelSnapshotId; + } + + /** + * Get a list of all input data fields mentioned in the job configuration, + * namely analysis fields and the time field. + * + * @return the list of fields - never null + */ + public List allFields() { + Set allFields = new TreeSet<>(); + + // analysis fields + if (analysisConfig != null) { + allFields.addAll(analysisConfig.analysisFields()); + } + + // time field + if (dataDescription != null) { + String timeField = dataDescription.getTimeField(); + if (timeField != null) { + allFields.add(timeField); + } + } + + // remove empty strings + allFields.remove(""); + + return new ArrayList<>(allFields); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(description); + out.writeVLong(createTime.getTime()); + if (finishedTime != null) { + out.writeBoolean(true); + out.writeVLong(finishedTime.getTime()); + } else { + out.writeBoolean(false); + } + if (lastDataTime != null) { + out.writeBoolean(true); + out.writeVLong(lastDataTime.getTime()); + } else { + out.writeBoolean(false); + } + analysisConfig.writeTo(out); + out.writeOptionalWriteable(analysisLimits); + out.writeOptionalWriteable(dataDescription); + out.writeOptionalWriteable(modelDebugConfig); + out.writeOptionalWriteable(ignoreDowntime); + out.writeOptionalLong(renormalizationWindowDays); + out.writeOptionalLong(backgroundPersistInterval); + out.writeOptionalLong(modelSnapshotRetentionDays); + out.writeOptionalLong(resultsRetentionDays); + out.writeMap(customSettings); + out.writeOptionalString(modelSnapshotId); + out.writeString(indexName); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + final String humanReadableSuffix = "_string"; + + builder.field(ID.getPreferredName(), jobId); + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + builder.dateField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + humanReadableSuffix, createTime.getTime()); + if (finishedTime != null) { + builder.dateField(FINISHED_TIME.getPreferredName(), FINISHED_TIME.getPreferredName() + humanReadableSuffix, + finishedTime.getTime()); + } + if (lastDataTime != null) { + builder.dateField(LAST_DATA_TIME.getPreferredName(), LAST_DATA_TIME.getPreferredName() + humanReadableSuffix, + lastDataTime.getTime()); + } + builder.field(ANALYSIS_CONFIG.getPreferredName(), analysisConfig, params); + if (analysisLimits != null) { + builder.field(ANALYSIS_LIMITS.getPreferredName(), analysisLimits, params); + } + if (dataDescription != null) { + builder.field(DATA_DESCRIPTION.getPreferredName(), dataDescription, params); + } + if (modelDebugConfig != null) { + builder.field(MODEL_DEBUG_CONFIG.getPreferredName(), modelDebugConfig, params); + } + if (ignoreDowntime != null) { + builder.field(IGNORE_DOWNTIME.getPreferredName(), ignoreDowntime); + } + if (renormalizationWindowDays != null) { + builder.field(RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays); + } + if (backgroundPersistInterval != null) { + builder.field(BACKGROUND_PERSIST_INTERVAL.getPreferredName(), backgroundPersistInterval); + } + if (modelSnapshotRetentionDays != null) { + builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); + } + if (resultsRetentionDays != null) { + builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); + } + if (customSettings != null) { + builder.field(CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + if (modelSnapshotId != null) { + builder.field(MODEL_SNAPSHOT_ID.getPreferredName(), modelSnapshotId); + } + builder.field(INDEX_NAME.getPreferredName(), indexName); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Job == false) { + return false; + } + + Job that = (Job) other; + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.description, that.description) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.finishedTime, that.finishedTime) + && Objects.equals(this.lastDataTime, that.lastDataTime) + && Objects.equals(this.analysisConfig, that.analysisConfig) + && Objects.equals(this.analysisLimits, that.analysisLimits) && Objects.equals(this.dataDescription, that.dataDescription) + && Objects.equals(this.modelDebugConfig, that.modelDebugConfig) + && Objects.equals(this.ignoreDowntime, that.ignoreDowntime) + && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) + && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) + && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) + && Objects.equals(this.customSettings, that.customSettings) + && Objects.equals(this.modelSnapshotId, that.modelSnapshotId) + && Objects.equals(this.indexName, that.indexName); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, description, createTime, finishedTime, lastDataTime, analysisConfig, + analysisLimits, dataDescription, modelDebugConfig, renormalizationWindowDays, + backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, ignoreDowntime, customSettings, + modelSnapshotId, indexName); + } + + // Class alreadt extends from AbstractDiffable, so copied from ToXContentToBytes#toString() + @Override + public final String toString() { + return Strings.toString(this); + } + + private static void checkValueNotLessThan(long minVal, String name, Long value) { + if (value != null && value < minVal) { + throw new IllegalArgumentException(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, name, minVal, value)); + } + } + + public static class Builder { + + private String id; + private String description; + + private AnalysisConfig analysisConfig; + private AnalysisLimits analysisLimits; + private DataDescription dataDescription; + private Date createTime; + private Date finishedTime; + private Date lastDataTime; + private ModelDebugConfig modelDebugConfig; + private Long renormalizationWindowDays; + private Long backgroundPersistInterval; + private Long modelSnapshotRetentionDays; + private Long resultsRetentionDays; + private IgnoreDowntime ignoreDowntime; + private Map customSettings; + private String modelSnapshotId; + private String indexName; + + public Builder() { + } + + public Builder(String id) { + this.id = id; + } + + public Builder(Job job) { + this.id = job.getId(); + this.description = job.getDescription(); + this.analysisConfig = job.getAnalysisConfig(); + this.dataDescription = job.getDataDescription(); + this.createTime = job.getCreateTime(); + this.finishedTime = job.getFinishedTime(); + this.lastDataTime = job.getLastDataTime(); + this.modelDebugConfig = job.getModelDebugConfig(); + this.renormalizationWindowDays = job.getRenormalizationWindowDays(); + this.backgroundPersistInterval = job.getBackgroundPersistInterval(); + this.resultsRetentionDays = job.getResultsRetentionDays(); + this.ignoreDowntime = job.getIgnoreDowntime(); + this.customSettings = job.getCustomSettings(); + this.modelSnapshotId = job.getModelSnapshotId(); + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setCustomSettings(Map customSettings) { + this.customSettings = customSettings; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setAnalysisConfig(AnalysisConfig.Builder configBuilder) { + analysisConfig = configBuilder.build(); + } + + public void setAnalysisLimits(AnalysisLimits analysisLimits) { + if (this.analysisLimits != null) { + long oldMemoryLimit = this.analysisLimits.getModelMemoryLimit(); + long newMemoryLimit = analysisLimits.getModelMemoryLimit(); + if (newMemoryLimit < oldMemoryLimit) { + throw new IllegalArgumentException( + Messages.getMessage(Messages.JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED, + oldMemoryLimit, newMemoryLimit)); + } + } + this.analysisLimits = analysisLimits; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public void setFinishedTime(Date finishedTime) { + this.finishedTime = finishedTime; + } + + /** + * Set the wall clock time of the last data upload + * @param lastDataTime Wall clock time + */ + public void setLastDataTime(Date lastDataTime) { + this.lastDataTime = lastDataTime; + } + + public void setDataDescription(DataDescription.Builder description) { + dataDescription = description.build(); + } + + public void setModelDebugConfig(ModelDebugConfig modelDebugConfig) { + this.modelDebugConfig = modelDebugConfig; + } + + public void setBackgroundPersistInterval(Long backgroundPersistInterval) { + this.backgroundPersistInterval = backgroundPersistInterval; + } + + public void setRenormalizationWindowDays(Long renormalizationWindowDays) { + this.renormalizationWindowDays = renormalizationWindowDays; + } + + public void setModelSnapshotRetentionDays(Long modelSnapshotRetentionDays) { + this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + } + + public void setResultsRetentionDays(Long resultsRetentionDays) { + this.resultsRetentionDays = resultsRetentionDays; + } + + public void setIgnoreDowntime(IgnoreDowntime ignoreDowntime) { + this.ignoreDowntime = ignoreDowntime; + } + + public void setModelSnapshotId(String modelSnapshotId) { + this.modelSnapshotId = modelSnapshotId; + } + + public void setIndexName(String indexName) { + this.indexName = indexName; + } + + public Job build() { + return build(false, null); + } + + public Job build(boolean fromApi, String urlJobId) { + + Date createTime; + Date finishedTime; + Date lastDataTime; + String modelSnapshotId; + if (fromApi) { + if (id == null) { + id = urlJobId; + } else if (!id.equals(urlJobId)) { + throw new IllegalArgumentException(Messages.getMessage(Messages.INCONSISTENT_ID, ID.getPreferredName(), id, urlJobId)); + } + createTime = this.createTime == null ? new Date() : this.createTime; + finishedTime = null; + lastDataTime = null; + modelSnapshotId = null; + } else { + createTime = this.createTime; + finishedTime = this.finishedTime; + lastDataTime = this.lastDataTime; + modelSnapshotId = this.modelSnapshotId; + } + + return new Job( + id, description, createTime, finishedTime, lastDataTime, analysisConfig, analysisLimits, + dataDescription, modelDebugConfig, ignoreDowntime, renormalizationWindowDays, + backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, + indexName + ); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/JobState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/JobState.java new file mode 100644 index 00000000000..afaa03292eb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/JobState.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; + +/** + * Jobs whether running or complete are in one of these states. + * When a job is created it is initialised in to the state closed + * i.e. it is not running. + */ +public enum JobState implements Writeable { + + CLOSING, CLOSED, OPENING, OPENED, FAILED, DELETING; + + public static JobState fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static JobState fromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown public enum JobState {\n ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + /** + * @return {@code true} if state matches any of the given {@code candidates} + */ + public boolean isAnyOf(JobState... candidates) { + return Arrays.stream(candidates).anyMatch(candidate -> this == candidate); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java new file mode 100644 index 00000000000..b022bfdf9e8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java @@ -0,0 +1,380 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.inject.internal.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class JobUpdate implements Writeable, ToXContent { + public static final ParseField DETECTORS = new ParseField("detectors"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("job_update", a -> new JobUpdate((String) a[0], (List) a[1], + (ModelDebugConfig) a[2], (AnalysisLimits) a[3], (Long) a[4], (Long) a[5], (Long) a[6], (Long) a[7], + (List) a[8], (Map) a[9])); + + + static { + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), DetectorUpdate.PARSER, DETECTORS); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelDebugConfig.PARSER, Job.MODEL_DEBUG_CONFIG); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), AnalysisLimits.PARSER, Job.ANALYSIS_LIMITS); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), Job.BACKGROUND_PERSIST_INTERVAL); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), Job.RENORMALIZATION_WINDOW_DAYS); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), Job.RESULTS_RETENTION_DAYS); + PARSER.declareLong(ConstructingObjectParser.optionalConstructorArg(), Job.MODEL_SNAPSHOT_RETENTION_DAYS); + PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), AnalysisConfig.CATEGORIZATION_FILTERS); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), Job.CUSTOM_SETTINGS, + ObjectParser.ValueType.OBJECT); + } + + private final String description; + private final List detectorUpdates; + private final ModelDebugConfig modelDebugConfig; + private final AnalysisLimits analysisLimits; + private final Long renormalizationWindowDays; + private final Long backgroundPersistInterval; + private final Long modelSnapshotRetentionDays; + private final Long resultsRetentionDays; + private final List categorizationFilters; + private final Map customSettings; + + public JobUpdate(@Nullable String description, @Nullable List detectorUpdates, + @Nullable ModelDebugConfig modelDebugConfig, @Nullable AnalysisLimits analysisLimits, + @Nullable Long backgroundPersistInterval, @Nullable Long renormalizationWindowDays, + @Nullable Long resultsRetentionDays, @Nullable Long modelSnapshotRetentionDays, + @Nullable List categorisationFilters, @Nullable Map customSettings) { + this.description = description; + this.detectorUpdates = detectorUpdates; + this.modelDebugConfig = modelDebugConfig; + this.analysisLimits = analysisLimits; + this.renormalizationWindowDays = renormalizationWindowDays; + this.backgroundPersistInterval = backgroundPersistInterval; + this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; + this.resultsRetentionDays = resultsRetentionDays; + this.categorizationFilters = categorisationFilters; + this.customSettings = customSettings; + } + + public JobUpdate(StreamInput in) throws IOException { + description = in.readOptionalString(); + if (in.readBoolean()) { + detectorUpdates = in.readList(DetectorUpdate::new); + } else { + detectorUpdates = null; + } + modelDebugConfig = in.readOptionalWriteable(ModelDebugConfig::new); + analysisLimits = in.readOptionalWriteable(AnalysisLimits::new); + renormalizationWindowDays = in.readOptionalLong(); + backgroundPersistInterval = in.readOptionalLong(); + modelSnapshotRetentionDays = in.readOptionalLong(); + resultsRetentionDays = in.readOptionalLong(); + if (in.readBoolean()) { + categorizationFilters = in.readList(StreamInput::readString); + } else { + categorizationFilters = null; + } + customSettings = in.readMap(); + } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(description); + out.writeBoolean(detectorUpdates != null); + if (detectorUpdates != null) { + out.writeList(detectorUpdates); + } + out.writeOptionalWriteable(modelDebugConfig); + out.writeOptionalWriteable(analysisLimits); + out.writeOptionalLong(renormalizationWindowDays); + out.writeOptionalLong(backgroundPersistInterval); + out.writeOptionalLong(modelSnapshotRetentionDays); + out.writeOptionalLong(resultsRetentionDays); + out.writeBoolean(categorizationFilters != null); + if (categorizationFilters != null) { + out.writeStringList(categorizationFilters); + } + out.writeMap(customSettings); + } + + public String getDescription() { + return description; + } + + public List getDetectorUpdates() { + return detectorUpdates; + } + + public ModelDebugConfig getModelDebugConfig() { + return modelDebugConfig; + } + + public AnalysisLimits getAnalysisLimits() { + return analysisLimits; + } + + public Long getRenormalizationWindowDays() { + return renormalizationWindowDays; + } + + public Long getBackgroundPersistInterval() { + return backgroundPersistInterval; + } + + public Long getModelSnapshotRetentionDays() { + return modelSnapshotRetentionDays; + } + + public Long getResultsRetentionDays() { + return resultsRetentionDays; + } + + public List getCategorizationFilters() { + return categorizationFilters; + } + + public Map getCustomSettings() { + return customSettings; + } + + public boolean isAutodetectProcessUpdate() { + return modelDebugConfig != null || detectorUpdates != null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (description != null) { + builder.field(Job.DESCRIPTION.getPreferredName(), description); + } + if (detectorUpdates != null) { + builder.field(DETECTORS.getPreferredName(), detectorUpdates); + } + if (modelDebugConfig != null) { + builder.field(Job.MODEL_DEBUG_CONFIG.getPreferredName(), modelDebugConfig); + } + if (analysisLimits != null) { + builder.field(Job.ANALYSIS_LIMITS.getPreferredName(), analysisLimits); + } + if (renormalizationWindowDays != null) { + builder.field(Job.RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays); + } + if (backgroundPersistInterval != null) { + builder.field(Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName(), backgroundPersistInterval); + } + if (modelSnapshotRetentionDays != null) { + builder.field(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays); + } + if (resultsRetentionDays != null) { + builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays); + } + if (categorizationFilters != null) { + builder.field(AnalysisConfig.CATEGORIZATION_FILTERS.getPreferredName(), categorizationFilters); + } + if (customSettings != null) { + builder.field(Job.CUSTOM_SETTINGS.getPreferredName(), customSettings); + } + builder.endObject(); + return builder; + } + + /** + * Updates {@code source} with the new values in this object returning a new {@link Job}. + * + * @param source Source job to be updated + * @return A new job equivalent to {@code source} updated. + */ + public Job mergeWithJob(Job source) { + Job.Builder builder = new Job.Builder(source); + if (description != null) { + builder.setDescription(description); + } + if (detectorUpdates != null && detectorUpdates.isEmpty() == false) { + AnalysisConfig ac = source.getAnalysisConfig(); + int numDetectors = ac.getDetectors().size(); + for (DetectorUpdate dd : detectorUpdates) { + if (dd.getIndex() >= numDetectors) { + throw new IllegalArgumentException("Detector index is >= the number of detectors"); + } + + Detector.Builder detectorbuilder = new Detector.Builder(ac.getDetectors().get(dd.getIndex())); + if (dd.getDescription() != null) { + detectorbuilder.setDetectorDescription(dd.getDescription()); + } + if (dd.getRules() != null) { + detectorbuilder.setDetectorRules(dd.getRules()); + } + ac.getDetectors().set(dd.getIndex(), detectorbuilder.build()); + } + + AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(ac); + builder.setAnalysisConfig(acBuilder); + } + if (modelDebugConfig != null) { + builder.setModelDebugConfig(modelDebugConfig); + } + if (analysisLimits != null) { + builder.setAnalysisLimits(analysisLimits); + } + if (renormalizationWindowDays != null) { + builder.setRenormalizationWindowDays(renormalizationWindowDays); + } + if (backgroundPersistInterval != null) { + builder.setBackgroundPersistInterval(backgroundPersistInterval); + } + if (modelSnapshotRetentionDays != null) { + builder.setModelSnapshotRetentionDays(modelSnapshotRetentionDays); + } + if (resultsRetentionDays != null) { + builder.setResultsRetentionDays(resultsRetentionDays); + } + if (categorizationFilters != null) { + AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(source.getAnalysisConfig()); + analysisConfigBuilder.setCategorizationFilters(categorizationFilters); + builder.setAnalysisConfig(analysisConfigBuilder); + } + if (customSettings != null) { + builder.setCustomSettings(customSettings); + } + + return builder.build(); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof JobUpdate == false) { + return false; + } + + JobUpdate that = (JobUpdate) other; + + return Objects.equals(this.description, that.description) + && Objects.equals(this.detectorUpdates, that.detectorUpdates) + && Objects.equals(this.modelDebugConfig, that.modelDebugConfig) + && Objects.equals(this.analysisLimits, that.analysisLimits) + && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) + && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) + && Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays) + && Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays) + && Objects.equals(this.categorizationFilters, that.categorizationFilters) + && Objects.equals(this.customSettings, that.customSettings); + } + + @Override + public int hashCode() { + return Objects.hash(description, detectorUpdates, modelDebugConfig, analysisLimits, renormalizationWindowDays, + backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings); + } + + public static class DetectorUpdate implements Writeable, ToXContent { + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("detector_update", a -> new DetectorUpdate((int) a[0], (String) a[1], + (List) a[2])); + + public static final ParseField INDEX = new ParseField("index"); + public static final ParseField RULES = new ParseField("rules"); + + static { + PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), INDEX); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), DetectionRule.PARSER, RULES); + } + + private int index; + private String description; + private List rules; + + public DetectorUpdate(int index, String description, List rules) { + this.index = index; + this.description = description; + this.rules = rules; + } + + public DetectorUpdate(StreamInput in) throws IOException { + index = in.readInt(); + description = in.readOptionalString(); + if (in.readBoolean()) { + rules = in.readList(DetectionRule::new); + } else { + rules = null; + } + } + + public int getIndex() { + return index; + } + + public String getDescription() { + return description; + } + + public List getRules() { + return rules; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(index); + out.writeOptionalString(description); + out.writeBoolean(rules != null); + if (rules != null) { + out.writeList(rules); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.field(INDEX.getPreferredName(), index); + if (description != null) { + builder.field(Job.DESCRIPTION.getPreferredName(), description); + } + if (rules != null) { + builder.field(RULES.getPreferredName(), rules); + } + builder.endObject(); + + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(index, description, rules); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof DetectorUpdate == false) { + return false; + } + + DetectorUpdate that = (DetectorUpdate) other; + return this.index == that.index && Objects.equals(this.description, that.description) + && Objects.equals(this.rules, that.rules); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java new file mode 100644 index 00000000000..fca5bda18b8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +public class MlFilter extends ToXContentToBytes implements Writeable { + public static final ParseField TYPE = new ParseField("filter"); + public static final ParseField ID = new ParseField("id"); + public static final ParseField ITEMS = new ParseField("items"); + + // For QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("filters"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new MlFilter((String) a[0], (List) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ID); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), ITEMS); + } + + private final String id; + private final List items; + + public MlFilter(String id, List items) { + this.id = Objects.requireNonNull(id, ID.getPreferredName() + " must not be null"); + this.items = Objects.requireNonNull(items, ITEMS.getPreferredName() + " must not be null"); + } + + public MlFilter(StreamInput in) throws IOException { + id = in.readString(); + items = Arrays.asList(in.readStringArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeStringArray(items.toArray(new String[items.size()])); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID.getPreferredName(), id); + builder.field(ITEMS.getPreferredName(), items); + builder.endObject(); + return builder; + } + + public String getId() { + return id; + } + + public List getItems() { + return new ArrayList<>(items); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + + if (!(obj instanceof MlFilter)) { + return false; + } + + MlFilter other = (MlFilter) obj; + return id.equals(other.id) && items.equals(other.items); + } + + @Override + public int hashCode() { + return Objects.hash(id, items); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/ModelDebugConfig.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/ModelDebugConfig.java new file mode 100644 index 00000000000..c14b87f459e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/ModelDebugConfig.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.Locale; +import java.util.Objects; + +public class ModelDebugConfig extends ToXContentToBytes implements Writeable { + + private static final double MAX_PERCENTILE = 100.0; + + private static final ParseField TYPE_FIELD = new ParseField("model_debug_config"); + public static final ParseField BOUNDS_PERCENTILE_FIELD = new ParseField("bounds_percentile"); + public static final ParseField TERMS_FIELD = new ParseField("terms"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE_FIELD.getPreferredName(), a -> new ModelDebugConfig((Double) a[0], (String) a[1])); + + static { + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), BOUNDS_PERCENTILE_FIELD); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD); + } + + private final double boundsPercentile; + private final String terms; + + public ModelDebugConfig(double boundsPercentile, String terms) { + if (boundsPercentile < 0.0 || boundsPercentile > MAX_PERCENTILE) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE); + throw new IllegalArgumentException(msg); + } + this.boundsPercentile = boundsPercentile; + this.terms = terms; + } + + public ModelDebugConfig(StreamInput in) throws IOException { + boundsPercentile = in.readDouble(); + terms = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(boundsPercentile); + out.writeOptionalString(terms); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(BOUNDS_PERCENTILE_FIELD.getPreferredName(), boundsPercentile); + if (terms != null) { + builder.field(TERMS_FIELD.getPreferredName(), terms); + } + builder.endObject(); + return builder; + } + + public double getBoundsPercentile() { + return this.boundsPercentile; + } + + public String getTerms() { + return this.terms; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof ModelDebugConfig == false) { + return false; + } + + ModelDebugConfig that = (ModelDebugConfig) other; + return Objects.equals(this.boundsPercentile, that.boundsPercentile) && Objects.equals(this.terms, that.terms); + } + + @Override + public int hashCode() { + return Objects.hash(boundsPercentile, terms); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Operator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Operator.java new file mode 100644 index 00000000000..f6b4d72f2f9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/Operator.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Enum representing logical comparisons on doubles + */ +public enum Operator implements Writeable { + EQ { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) == 0; + } + }, + GT { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) > 0; + } + }, + GTE { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) >= 0; + } + }, + LT { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) < 0; + } + }, + LTE { + @Override + public boolean test(double lhs, double rhs) { + return Double.compare(lhs, rhs) <= 0; + } + }, + MATCH { + @Override + public boolean match(Pattern pattern, String field) { + Matcher match = pattern.matcher(field); + return match.matches(); + } + + @Override + public boolean expectsANumericArgument() { + return false; + } + }; + + public static final ParseField OPERATOR_FIELD = new ParseField("operator"); + + public boolean test(double lhs, double rhs) { + return false; + } + + public boolean match(Pattern pattern, String field) { + return false; + } + + public boolean expectsANumericArgument() { + return true; + } + + public static Operator fromString(String name) { + return valueOf(name.trim().toUpperCase(Locale.ROOT)); + } + + public static Operator readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Operator ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java new file mode 100644 index 00000000000..073f150611a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import java.util.Locale; + +public enum RuleAction { + FILTER_RESULTS; + + /** + * Case-insensitive from string method. + * + * @param value String representation + * @return The rule action + */ + public static RuleAction fromString(String value) { + return RuleAction.valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java new file mode 100644 index 00000000000..a0c16d57f5e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java @@ -0,0 +1,245 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.EnumSet; +import java.util.Objects; + +public class RuleCondition extends ToXContentToBytes implements Writeable { + public static final ParseField CONDITION_TYPE_FIELD = new ParseField("condition_type"); + public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition"); + public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name"); + public static final ParseField FIELD_VALUE_FIELD = new ParseField("field_value"); + public static final ParseField VALUE_FILTER_FIELD = new ParseField("value_filter"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), + a -> new RuleCondition((RuleConditionType) a[0], (String) a[1], (String) a[2], (Condition) a[3], (String) a[4])); + + static { + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return RuleConditionType.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, CONDITION_TYPE_FIELD, ValueType.STRING); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_NAME_FIELD); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_VALUE_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Condition.PARSER, Condition.CONDITION_FIELD); + PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), VALUE_FILTER_FIELD); + } + + private final RuleConditionType conditionType; + private final String fieldName; + private final String fieldValue; + private final Condition condition; + private final String valueFilter; + + public RuleCondition(StreamInput in) throws IOException { + conditionType = RuleConditionType.readFromStream(in); + condition = in.readOptionalWriteable(Condition::new); + fieldName = in.readOptionalString(); + fieldValue = in.readOptionalString(); + valueFilter = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + conditionType.writeTo(out); + out.writeOptionalWriteable(condition); + out.writeOptionalString(fieldName); + out.writeOptionalString(fieldValue); + out.writeOptionalString(valueFilter); + } + + public RuleCondition(RuleConditionType conditionType, String fieldName, String fieldValue, Condition condition, String valueFilter) { + this.conditionType = conditionType; + this.fieldName = fieldName; + this.fieldValue = fieldValue; + this.condition = condition; + this.valueFilter = valueFilter; + + verifyFieldsBoundToType(this); + verifyFieldValueRequiresFieldName(this); + } + + public RuleCondition(RuleCondition ruleCondition) { + this.conditionType = ruleCondition.conditionType; + this.fieldName = ruleCondition.fieldName; + this.fieldValue = ruleCondition.fieldValue; + this.condition = ruleCondition.condition; + this.valueFilter = ruleCondition.valueFilter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(CONDITION_TYPE_FIELD.getPreferredName(), conditionType); + if (condition != null) { + builder.field(Condition.CONDITION_FIELD.getPreferredName(), condition); + } + if (fieldName != null) { + builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName); + } + if (fieldValue != null) { + builder.field(FIELD_VALUE_FIELD.getPreferredName(), fieldValue); + } + if (valueFilter != null) { + builder.field(VALUE_FILTER_FIELD.getPreferredName(), valueFilter); + } + builder.endObject(); + return builder; + } + + public RuleConditionType getConditionType() { + return conditionType; + } + + /** + * The field name for which the rule applies. Can be null, meaning rule + * applies to all results. + */ + public String getFieldName() { + return fieldName; + } + + /** + * The value of the field name for which the rule applies. When set, the + * rule applies only to the results that have the fieldName/fieldValue pair. + * When null, the rule applies to all values for of the specified field + * name. Only applicable when fieldName is not null. + */ + public String getFieldValue() { + return fieldValue; + } + + public Condition getCondition() { + return condition; + } + + /** + * The unique identifier of a filter. Required when the rule type is + * categorical. Should be null for all other types. + */ + public String getValueFilter() { + return valueFilter; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof RuleCondition == false) { + return false; + } + + RuleCondition other = (RuleCondition) obj; + return Objects.equals(conditionType, other.conditionType) && Objects.equals(fieldName, other.fieldName) + && Objects.equals(fieldValue, other.fieldValue) && Objects.equals(condition, other.condition) + && Objects.equals(valueFilter, other.valueFilter); + } + + @Override + public int hashCode() { + return Objects.hash(conditionType, fieldName, fieldValue, condition, valueFilter); + } + + public static RuleCondition createCategorical(String fieldName, String valueFilter) { + return new RuleCondition(RuleConditionType.CATEGORICAL, fieldName, null, null, valueFilter); + } + + private static void verifyFieldsBoundToType(RuleCondition ruleCondition) throws ElasticsearchParseException { + switch (ruleCondition.getConditionType()) { + case CATEGORICAL: + verifyCategorical(ruleCondition); + break; + case NUMERICAL_ACTUAL: + case NUMERICAL_TYPICAL: + case NUMERICAL_DIFF_ABS: + verifyNumerical(ruleCondition); + break; + default: + throw new IllegalStateException(); + } + } + + private static void verifyCategorical(RuleCondition ruleCondition) throws ElasticsearchParseException { + checkCategoricalHasNoField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); + checkCategoricalHasNoField(RuleCondition.FIELD_VALUE_FIELD.getPreferredName(), ruleCondition.getFieldValue()); + checkCategoricalHasField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter()); + } + + private static void checkCategoricalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void checkCategoricalHasField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_MISSING_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void verifyNumerical(RuleCondition ruleCondition) throws ElasticsearchParseException { + checkNumericalHasNoField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter()); + checkNumericalHasField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); + if (ruleCondition.getFieldName() != null && ruleCondition.getFieldValue() == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE); + throw new IllegalArgumentException(msg); + } + checkNumericalConditionOparatorsAreValid(ruleCondition); + } + + private static void checkNumericalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue != null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void checkNumericalHasField(String fieldName, Object fieldValue) throws ElasticsearchParseException { + if (fieldValue == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_MISSING_OPTION, fieldName); + throw new IllegalArgumentException(msg); + } + } + + private static void verifyFieldValueRequiresFieldName(RuleCondition ruleCondition) throws ElasticsearchParseException { + if (ruleCondition.getFieldValue() != null && ruleCondition.getFieldName() == null) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_MISSING_FIELD_NAME, + ruleCondition.getFieldValue()); + throw new IllegalArgumentException(msg); + } + } + + static EnumSet VALID_CONDITION_OPERATORS = EnumSet.of(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE); + + private static void checkNumericalConditionOparatorsAreValid(RuleCondition ruleCondition) throws ElasticsearchParseException { + Operator operator = ruleCondition.getCondition().getOperator(); + if (!VALID_CONDITION_OPERATORS.contains(operator)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPERATOR, operator); + throw new IllegalArgumentException(msg); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleConditionType.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleConditionType.java new file mode 100644 index 00000000000..b13418d9188 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleConditionType.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + + +import java.io.IOException; +import java.util.Locale; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +public enum RuleConditionType implements Writeable { + CATEGORICAL, + NUMERICAL_ACTUAL, + NUMERICAL_TYPICAL, + NUMERICAL_DIFF_ABS; + + /** + * Case-insensitive from string method. + * + * @param value + * String representation + * @return The condition type + */ + public static RuleConditionType fromString(String value) { + return RuleConditionType.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static RuleConditionType readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown RuleConditionType ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java new file mode 100644 index 00000000000..e1a2d428ad9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java @@ -0,0 +1,242 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.messages; + +import java.text.MessageFormat; +import java.util.Locale; +import java.util.ResourceBundle; + +/** + * Defines the keys for all the message strings + */ +public final class Messages { + /** + * The base name of the bundle without the .properties extension + * or locale + */ + private static final String BUNDLE_NAME = "org.elasticsearch.xpack.ml.job.messages.ml_messages"; + public static final String AUTODETECT_FLUSH_UNEXPTECTED_DEATH = "autodetect.flush.failed.unexpected.death"; + + public static final String CPU_LIMIT_JOB = "cpu.limit.jobs"; + + public static final String DATASTORE_ERROR_DELETING = "datastore.error.deleting"; + public static final String DATASTORE_ERROR_DELETING_MISSING_INDEX = "datastore.error.deleting.missing.index"; + public static final String DATASTORE_ERROR_EXECUTING_SCRIPT = "datastore.error.executing.script"; + + public static final String INVALID_ID = "invalid.id"; + public static final String INCONSISTENT_ID = "inconsistent.id"; + + public static final String LICENSE_LIMIT_DETECTORS = "license.limit.detectors"; + public static final String LICENSE_LIMIT_JOBS = "license.limit.jobs"; + public static final String LICENSE_LIMIT_DETECTORS_REACTIVATE = "license.limit.detectors.reactivate"; + public static final String LICENSE_LIMIT_JOBS_REACTIVATE = "license.limit.jobs.reactivate"; + public static final String LICENSE_LIMIT_PARTITIONS = "license.limit.partitions"; + + public static final String JOB_AUDIT_CREATED = "job.audit.created"; + public static final String JOB_AUDIT_DELETED = "job.audit.deleted"; + public static final String JOB_AUDIT_PAUSED = "job.audit.paused"; + public static final String JOB_AUDIT_RESUMED = "job.audit.resumed"; + public static final String JOB_AUDIT_UPDATED = "job.audit.updated"; + public static final String JOB_AUDIT_REVERTED = "job.audit.reverted"; + public static final String JOB_AUDIT_OLD_RESULTS_DELETED = "job.audit.old.results.deleted"; + public static final String JOB_AUDIT_SNAPSHOT_DELETED = "job.audit.snapshot.deleted"; + public static final String JOB_AUDIT_DATAFEED_STARTED_FROM_TO = "job.audit.datafeed.started.from.to"; + public static final String JOB_AUDIT_DATAFEED_CONTINUED_REALTIME = "job.audit.datafeed.continued.realtime"; + public static final String JOB_AUDIT_DATAFEED_STARTED_REALTIME = "job.audit.datafeed.started.realtime"; + public static final String JOB_AUDIT_DATAFEED_LOOKBACK_COMPLETED = "job.audit.datafeed.lookback.completed"; + public static final String JOB_AUDIT_DATAFEED_STOPPED = "job.audit.datafeed.stopped"; + public static final String JOB_AUDIT_DATAFEED_NO_DATA = "job.audit.datafeed.no.data"; + public static final String JOB_AUDIR_DATAFEED_DATA_SEEN_AGAIN = "job.audit.datafeed.data.seen.again"; + public static final String JOB_AUDIT_DATAFEED_DATA_ANALYSIS_ERROR = "job.audit.datafeed.data.analysis.error"; + public static final String JOB_AUDIT_DATAFEED_DATA_EXTRACTION_ERROR = "job.audit.datafeed.data.extraction.error"; + public static final String JOB_AUDIT_DATAFEED_RECOVERED = "job.audit.datafeed.recovered"; + + public static final String SYSTEM_AUDIT_STARTED = "system.audit.started"; + public static final String SYSTEM_AUDIT_SHUTDOWN = "system.audit.shutdown"; + + public static final String JOB_CANNOT_DELETE_WHILE_RUNNING = "job.cannot.delete.while.running"; + public static final String JOB_CANNOT_PAUSE = "job.cannot.pause"; + public static final String JOB_CANNOT_RESUME = "job.cannot.resume"; + + public static final String JOB_CONFIG_BYFIELD_INCOMPATIBLE_FUNCTION = "job.config.byField.incompatible.function"; + public static final String JOB_CONFIG_BYFIELD_NEEDS_ANOTHER = "job.config.byField.needs.another"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_REQUIRE_CATEGORIZATION_FIELD_NAME = "job.config.categorization.filters." + + "require.categorization.field.name"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES = "job.config.categorization.filters.contains" + + ".duplicates"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY = "job.config.categorization.filter.contains.empty"; + public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_INVALID_REGEX = "job.config.categorization.filter.contains." + + "invalid.regex"; + public static final String JOB_CONFIG_CONDITION_INVALID_OPERATOR = "job.config.condition.invalid.operator"; + public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_NULL = "job.config.condition.invalid.value.null"; + public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER = "job.config.condition.invalid.value.numeric"; + public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX = "job.config.condition.invalid.value.regex"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION = "job.config.detectionrule.condition." + + "categorical.invalid.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_MISSING_OPTION = "job.config.detectionrule.condition." + + "categorical.missing.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME = "job.config.detectionrule.condition.invalid." + + "fieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_MISSING_FIELD_NAME = "job.config.detectionrule.condition.missing." + + "fieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPERATOR = "job.config.detectionrule.condition." + + "numerical.invalid.operator"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPTION = "job.config.detectionrule.condition." + + "numerical.invalid.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_MISSING_OPTION = "job.config.detectionrule.condition." + + "numerical.missing.option"; + public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE = "job.config." + + "detectionrule.condition.numerical.with.fieldname.requires.fieldvalue"; + public static final String JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME = "job.config.detectionrule.invalid.targetfieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME = "job.config.detectionrule.missing.targetfieldname"; + public static final String JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION = "job.config.detectionrule.not.supported.by.function"; + public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION = "job.config.detectionrule.requires.at." + + "least.one.condition"; + public static final String JOB_CONFIG_FIELDNAME_INCOMPATIBLE_FUNCTION = "job.config.fieldname.incompatible.function"; + public static final String JOB_CONFIG_FUNCTION_REQUIRES_BYFIELD = "job.config.function.requires.byfield"; + public static final String JOB_CONFIG_FUNCTION_REQUIRES_FIELDNAME = "job.config.function.requires.fieldname"; + public static final String JOB_CONFIG_FUNCTION_REQUIRES_OVERFIELD = "job.config.function.requires.overfield"; + public static final String JOB_CONFIG_ID_TOO_LONG = "job.config.id.too.long"; + public static final String JOB_CONFIG_ID_ALREADY_TAKEN = "job.config.id.already.taken"; + public static final String JOB_CONFIG_INVALID_FIELDNAME_CHARS = "job.config.invalid.fieldname.chars"; + public static final String JOB_CONFIG_INVALID_TIMEFORMAT = "job.config.invalid.timeformat"; + public static final String JOB_CONFIG_FUNCTION_INCOMPATIBLE_PRESUMMARIZED = "job.config.function.incompatible.presummarized"; + public static final String JOB_CONFIG_MISSING_ANALYSISCONFIG = "job.config.missing.analysisconfig"; + public static final String JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE = "job.config.model.debug.config.invalid.bounds." + + "percentile"; + public static final String JOB_CONFIG_FIELD_VALUE_TOO_LOW = "job.config.field.value.too.low"; + public static final String JOB_CONFIG_NO_ANALYSIS_FIELD = "job.config.no.analysis.field"; + public static final String JOB_CONFIG_NO_ANALYSIS_FIELD_NOT_COUNT = "job.config.no.analysis.field.not.count"; + public static final String JOB_CONFIG_NO_DETECTORS = "job.config.no.detectors"; + public static final String JOB_CONFIG_OVERFIELD_INCOMPATIBLE_FUNCTION = "job.config.overField.incompatible.function"; + public static final String JOB_CONFIG_OVERLAPPING_BUCKETS_INCOMPATIBLE_FUNCTION = "job.config.overlapping.buckets.incompatible." + + "function"; + public static final String JOB_CONFIG_OVERFIELD_NEEDS_ANOTHER = "job.config.overField.needs.another"; + public static final String JOB_CONFIG_MULTIPLE_BUCKETSPANS_REQUIRE_BUCKETSPAN = "job.config.multiple.bucketspans.require.bucket_span"; + public static final String JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE = "job.config.multiple.bucketspans.must.be.multiple"; + public static final String JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD = "job.config.per.partition.normalization." + + "requires.partition.field"; + public static final String JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS = "job.config.per.partition.normalization." + + "cannot.use.influencers"; + + + public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_PARSE_ERROR = "job.config.update.analysis.limits.parse.error"; + public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_CANNOT_BE_NULL = "job.config.update.analysis.limits.cannot.be.null"; + public static final String JOB_CONFIG_UPDATE_ANALYSIS_LIMITS_MODEL_MEMORY_LIMIT_CANNOT_BE_DECREASED = "job.config.update.analysis." + + "limits.model.memory.limit.cannot.be.decreased"; + public static final String JOB_CONFIG_UPDATE_CATEGORIZATION_FILTERS_INVALID = "job.config.update.categorization.filters.invalid"; + public static final String JOB_CONFIG_UPDATE_CUSTOM_SETTINGS_INVALID = "job.config.update.custom.settings.invalid"; + public static final String JOB_CONFIG_UPDATE_DESCRIPTION_INVALID = "job.config.update.description.invalid"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_INVALID = "job.config.update.detectors.invalid"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_INVALID_DETECTOR_INDEX = "job.config.update.detectors.invalid.detector.index"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_DETECTOR_INDEX_SHOULD_BE_INTEGER = "job.config.update.detectors.detector.index." + + "should.be.integer"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_MISSING_PARAMS = "job.config.update.detectors.missing.params"; + public static final String JOB_CONFIG_UPDATE_DETECTORS_DESCRIPTION_SHOULD_BE_STRING = "job.config.update.detectors.description.should" + + ".be.string"; + public static final String JOB_CONFIG_UPDATE_DETECTOR_RULES_PARSE_ERROR = "job.config.update.detectors.rules.parse.error"; + public static final String JOB_CONFIG_UPDATE_FAILED = "job.config.update.failed"; + public static final String JOB_CONFIG_UPDATE_INVALID_KEY = "job.config.update.invalid.key"; + public static final String JOB_CONFIG_UPDATE_IGNORE_DOWNTIME_PARSE_ERROR = "job.config.update.ignore.downtime.parse.error"; + public static final String JOB_CONFIG_UPDATE_JOB_IS_NOT_CLOSED = "job.config.update.job.is.not.closed"; + public static final String JOB_CONFIG_UPDATE_MODEL_DEBUG_CONFIG_PARSE_ERROR = "job.config.update.model.debug.config.parse.error"; + public static final String JOB_CONFIG_UPDATE_REQUIRES_NON_EMPTY_OBJECT = "job.config.update.requires.non.empty.object"; + public static final String JOB_CONFIG_UPDATE_PARSE_ERROR = "job.config.update.parse.error"; + public static final String JOB_CONFIG_UPDATE_BACKGROUND_PERSIST_INTERVAL_INVALID = "job.config.update.background.persist.interval." + + "invalid"; + public static final String JOB_CONFIG_UPDATE_RENORMALIZATION_WINDOW_DAYS_INVALID = "job.config.update.renormalization.window.days." + + "invalid"; + public static final String JOB_CONFIG_UPDATE_MODEL_SNAPSHOT_RETENTION_DAYS_INVALID = "job.config.update.model.snapshot.retention.days." + + "invalid"; + public static final String JOB_CONFIG_UPDATE_RESULTS_RETENTION_DAYS_INVALID = "job.config.update.results.retention.days.invalid"; + public static final String JOB_CONFIG_UPDATE_DATAFEED_CONFIG_PARSE_ERROR = "job.config.update.datafeed.config.parse.error"; + public static final String JOB_CONFIG_UPDATE_DATAFEED_CONFIG_CANNOT_BE_NULL = "job.config.update.datafeed.config.cannot.be.null"; + + public static final String JOB_CONFIG_UNKNOWN_FUNCTION = "job.config.unknown.function"; + + public static final String JOB_INDEX_ALREADY_EXISTS = "job.index.already.exists"; + + public static final String JOB_DATA_CONCURRENT_USE_CLOSE = "job.data.concurrent.use.close"; + public static final String JOB_DATA_CONCURRENT_USE_FLUSH = "job.data.concurrent.use.flush"; + public static final String JOB_DATA_CONCURRENT_USE_UPDATE = "job.data.concurrent.use.update"; + public static final String JOB_DATA_CONCURRENT_USE_UPLOAD = "job.data.concurrent.use.upload"; + + public static final String DATAFEED_CONFIG_INVALID_OPTION_VALUE = "datafeed.config.invalid.option.value"; + public static final String DATAFEED_CONFIG_CANNOT_USE_SCRIPT_FIELDS_WITH_AGGS = "datafeed.config.cannot.use.script.fields.with.aggs"; + + public static final String DATAFEED_DOES_NOT_SUPPORT_JOB_WITH_LATENCY = "datafeed.does.not.support.job.with.latency"; + public static final String DATAFEED_AGGREGATIONS_REQUIRES_JOB_WITH_SUMMARY_COUNT_FIELD = + "datafeed.aggregations.requires.job.with.summary.count.field"; + + public static final String DATAFEED_CANNOT_START = "datafeed.cannot.start"; + public static final String DATAFEED_CANNOT_STOP_IN_CURRENT_STATE = "datafeed.cannot.stop.in.current.state"; + public static final String DATAFEED_CANNOT_UPDATE_IN_CURRENT_STATE = "datafeed.cannot.update.in.current.state"; + public static final String DATAFEED_CANNOT_DELETE_IN_CURRENT_STATE = "datafeed.cannot.delete.in.current.state"; + public static final String DATAFEED_FAILED_TO_STOP = "datafeed.failed.to.stop"; + public static final String DATAFEED_NOT_FOUND = "datafeed.not.found"; + + public static final String JOB_MISSING_QUANTILES = "job.missing.quantiles"; + public static final String JOB_UNKNOWN_ID = "job.unknown.id"; + + public static final String JSON_JOB_CONFIG_MAPPING = "json.job.config.mapping.error"; + public static final String JSON_JOB_CONFIG_PARSE = "json.job.config.parse.error"; + + public static final String JSON_DETECTOR_CONFIG_MAPPING = "json.detector.config.mapping.error"; + public static final String JSON_DETECTOR_CONFIG_PARSE = "json.detector.config.parse.error"; + + public static final String REST_ACTION_NOT_ALLOWED_FOR_DATAFEED_JOB = "rest.action.not.allowed.for.datafeed.job"; + + public static final String REST_INVALID_DATETIME_PARAMS = "rest.invalid.datetime.params"; + public static final String REST_INVALID_FLUSH_PARAMS_MISSING = "rest.invalid.flush.params.missing.argument"; + public static final String REST_INVALID_FLUSH_PARAMS_UNEXPECTED = "rest.invalid.flush.params.unexpected"; + public static final String REST_INVALID_RESET_PARAMS = "rest.invalid.reset.params"; + public static final String REST_INVALID_FROM = "rest.invalid.from"; + public static final String REST_INVALID_SIZE = "rest.invalid.size"; + public static final String REST_INVALID_FROM_SIZE_SUM = "rest.invalid.from.size.sum"; + public static final String REST_START_AFTER_END = "rest.start.after.end"; + public static final String REST_RESET_BUCKET_NO_LATENCY = "rest.reset.bucket.no.latency"; + public static final String REST_JOB_NOT_CLOSED_REVERT = "rest.job.not.closed.revert"; + public static final String REST_NO_SUCH_MODEL_SNAPSHOT = "rest.no.such.model.snapshot"; + public static final String REST_DESCRIPTION_ALREADY_USED = "rest.description.already.used"; + public static final String REST_CANNOT_DELETE_HIGHEST_PRIORITY = "rest.cannot.delete.highest.priority"; + + public static final String PROCESS_ACTION_SLEEPING_JOB = "process.action.sleeping.job"; + public static final String PROCESS_ACTION_CLOSED_JOB = "process.action.closed.job"; + public static final String PROCESS_ACTION_CLOSING_JOB = "process.action.closing.job"; + public static final String PROCESS_ACTION_DELETING_JOB = "process.action.deleting.job"; + public static final String PROCESS_ACTION_FLUSHING_JOB = "process.action.flushing.job"; + public static final String PROCESS_ACTION_PAUSING_JOB = "process.action.pausing.job"; + public static final String PROCESS_ACTION_RESUMING_JOB = "process.action.resuming.job"; + public static final String PROCESS_ACTION_REVERTING_JOB = "process.action.reverting.job"; + public static final String PROCESS_ACTION_UPDATING_JOB = "process.action.updating.job"; + public static final String PROCESS_ACTION_WRITING_JOB = "process.action.writing.job"; + + private Messages() { + } + + public static ResourceBundle load() { + return ResourceBundle.getBundle(Messages.BUNDLE_NAME, Locale.getDefault()); + } + + /** + * Look up the message string from the resource bundle. + * + * @param key Must be one of the statics defined in this file] + */ + public static String getMessage(String key) { + return load().getString(key); + } + + /** + * Look up the message string from the resource bundle and format with + * the supplied arguments + * @param key the key for the message + * @param args MessageFormat arguments. See {@linkplain MessageFormat#format(Object)}] + */ + public static String getMessage(String key, Object...args) { + return new MessageFormat(load().getString(key), Locale.ROOT).format(args); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/Allocation.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/Allocation.java new file mode 100644 index 00000000000..2afab663868 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/Allocation.java @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.metadata; + +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; + +import java.io.IOException; +import java.util.Objects; + +public class Allocation extends AbstractDiffable implements ToXContent { + + private static final ParseField NODE_ID_FIELD = new ParseField("node_id"); + private static final ParseField JOB_ID_FIELD = new ParseField("job_id"); + private static final ParseField IGNORE_DOWNTIME_FIELD = new ParseField("ignore_downtime"); + public static final ParseField STATE = new ParseField("state"); + public static final ParseField STATE_REASON = new ParseField("state_reason"); + + static final ObjectParser PARSER = new ObjectParser<>("allocation", Builder::new); + + static { + PARSER.declareString(Builder::setNodeId, NODE_ID_FIELD); + PARSER.declareString(Builder::setJobId, JOB_ID_FIELD); + PARSER.declareBoolean(Builder::setIgnoreDowntime, IGNORE_DOWNTIME_FIELD); + PARSER.declareField(Builder::setState, (p, c) -> JobState.fromString(p.text()), STATE, ObjectParser.ValueType.STRING); + PARSER.declareString(Builder::setStateReason, STATE_REASON); + } + + private final String nodeId; + private final String jobId; + private final boolean ignoreDowntime; + private final JobState state; + private final String stateReason; + + public Allocation(String nodeId, String jobId, boolean ignoreDowntime, JobState state, String stateReason) { + this.nodeId = nodeId; + this.jobId = jobId; + this.ignoreDowntime = ignoreDowntime; + this.state = state; + this.stateReason = stateReason; + } + + public Allocation(StreamInput in) throws IOException { + this.nodeId = in.readOptionalString(); + this.jobId = in.readString(); + this.ignoreDowntime = in.readBoolean(); + this.state = JobState.fromStream(in); + this.stateReason = in.readOptionalString(); + } + + public String getNodeId() { + return nodeId; + } + + public String getJobId() { + return jobId; + } + + /** + * @return Whether to ignore downtime at startup. + * + * When the job state is set to STARTED, to ignoreDowntime will be set to false. + */ + public boolean isIgnoreDowntime() { + return ignoreDowntime; + } + + public JobState getState() { + return state; + } + + public String getStateReason() { + return stateReason; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(nodeId); + out.writeString(jobId); + out.writeBoolean(ignoreDowntime); + state.writeTo(out); + out.writeOptionalString(stateReason); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (nodeId != null) { + builder.field(NODE_ID_FIELD.getPreferredName(), nodeId); + } + builder.field(JOB_ID_FIELD.getPreferredName(), jobId); + builder.field(IGNORE_DOWNTIME_FIELD.getPreferredName(), ignoreDowntime); + builder.field(STATE.getPreferredName(), state); + if (stateReason != null) { + builder.field(STATE_REASON.getPreferredName(), stateReason); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Allocation that = (Allocation) o; + return Objects.equals(nodeId, that.nodeId) && + Objects.equals(jobId, that.jobId) && + Objects.equals(ignoreDowntime, that.ignoreDowntime) && + Objects.equals(state, that.state) && + Objects.equals(stateReason, that.stateReason); + } + + @Override + public int hashCode() { + return Objects.hash(nodeId, jobId, ignoreDowntime, state, stateReason); + } + + // Class already extends from AbstractDiffable, so copied from ToXContentToBytes#toString() + @Override + public final String toString() { + return Strings.toString(this); + } + + public static class Builder { + + private String nodeId; + private String jobId; + private boolean ignoreDowntime; + private JobState state; + private String stateReason; + + public Builder() { + } + + public Builder(Job job) { + this.jobId = job.getId(); + } + + public Builder(Allocation allocation) { + this.nodeId = allocation.nodeId; + this.jobId = allocation.jobId; + this.ignoreDowntime = allocation.ignoreDowntime; + this.state = allocation.state; + this.stateReason = allocation.stateReason; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public void setIgnoreDowntime(boolean ignoreDownTime) { + this.ignoreDowntime = ignoreDownTime; + } + + @SuppressWarnings("incomplete-switch") + public void setState(JobState newState) { + if (this.state != null) { + switch (newState) { + case CLOSING: + if (this.state != JobState.OPENED) { + throw new IllegalArgumentException("[" + jobId + "] expected state [" + JobState.OPENED + + "], but got [" + state +"]"); + } + break; + case OPENING: + if (this.state.isAnyOf(JobState.CLOSED, JobState.FAILED) == false) { + throw new IllegalArgumentException("[" + jobId + "] expected state [" + JobState.CLOSED + + "] or [" + JobState.FAILED + "], but got [" + state +"]"); + } + break; + case OPENED: + ignoreDowntime = false; + break; + } + } + + this.state = newState; + } + + public void setStateReason(String stateReason) { + this.stateReason = stateReason; + } + + public Allocation build() { + return new Allocation(nodeId, jobId, ignoreDowntime, state, stateReason); + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/MlInitializationService.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/MlInitializationService.java new file mode 100644 index 00000000000..f57b6d9a359 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/MlInitializationService.java @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.metadata; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.notifications.Auditor; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class MlInitializationService extends AbstractComponent implements ClusterStateListener { + + private final ThreadPool threadPool; + private final ClusterService clusterService; + private final JobProvider jobProvider; + + private final AtomicBoolean installMlMetadataCheck = new AtomicBoolean(false); + private final AtomicBoolean createMlAuditIndexCheck = new AtomicBoolean(false); + private final AtomicBoolean createMlMetaIndexCheck = new AtomicBoolean(false); + private final AtomicBoolean createStateIndexCheck = new AtomicBoolean(false); + + public MlInitializationService(Settings settings, ThreadPool threadPool, ClusterService clusterService, + JobProvider jobProvider) { + super(settings); + this.threadPool = threadPool; + this.clusterService = clusterService; + this.jobProvider = jobProvider; + clusterService.addListener(this); + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeMaster()) { + MetaData metaData = event.state().metaData(); + if (metaData.custom(MlMetadata.TYPE) == null) { + if (installMlMetadataCheck.compareAndSet(false, true)) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { + clusterService.submitStateUpdateTask("install-ml-metadata", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + ClusterState.Builder builder = new ClusterState.Builder(currentState); + MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData()); + metadataBuilder.putCustom(MlMetadata.TYPE, MlMetadata.EMPTY_METADATA); + builder.metaData(metadataBuilder.build()); + return builder.build(); + } + + @Override + public void onFailure(String source, Exception e) { + logger.error("unable to install ml metadata upon startup", e); + } + }); + }); + } + } else { + installMlMetadataCheck.set(false); + } + if (metaData.hasIndex(Auditor.NOTIFICATIONS_INDEX) == false) { + if (createMlAuditIndexCheck.compareAndSet(false, true)) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { + jobProvider.createNotificationMessageIndex((result, error) -> { + if (result) { + logger.info("successfully created {} index", Auditor.NOTIFICATIONS_INDEX); + } else { + if (error instanceof ResourceAlreadyExistsException) { + logger.debug("not able to create {} index as it already exists", Auditor.NOTIFICATIONS_INDEX); + } else { + logger.error( + new ParameterizedMessage("not able to create {} index", Auditor.NOTIFICATIONS_INDEX), error); + } + } + createMlAuditIndexCheck.set(false); + }); + }); + } + } + if (metaData.hasIndex(JobProvider.ML_META_INDEX) == false) { + if (createMlMetaIndexCheck.compareAndSet(false, true)) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { + jobProvider.createMetaIndex((result, error) -> { + if (result) { + logger.info("successfully created {} index", JobProvider.ML_META_INDEX); + } else { + if (error instanceof ResourceAlreadyExistsException) { + logger.debug("not able to create {} index as it already exists", JobProvider.ML_META_INDEX); + } else { + logger.error(new ParameterizedMessage("not able to create {} index", JobProvider.ML_META_INDEX), error); + } + } + createMlMetaIndexCheck.set(false); + }); + }); + } + } + String stateIndexName = AnomalyDetectorsIndex.jobStateIndexName(); + if (metaData.hasIndex(stateIndexName) == false) { + if (createStateIndexCheck.compareAndSet(false, true)) { + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { + jobProvider.createJobStateIndex((result, error) -> { + if (result) { + logger.info("successfully created {} index", stateIndexName); + } else { + if (error instanceof ResourceAlreadyExistsException) { + logger.debug("not able to create {} index as it already exists", stateIndexName); + } else { + logger.error("not able to create " + stateIndexName + " index", error); + } + } + createStateIndexCheck.set(false); + }); + }); + } + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/MlMetadata.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/MlMetadata.java new file mode 100644 index 00000000000..aabdba67cb7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/metadata/MlMetadata.java @@ -0,0 +1,441 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.metadata; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.cluster.Diff; +import org.elasticsearch.cluster.DiffableUtils; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedJobValidator; +import org.elasticsearch.xpack.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress.PersistentTaskInProgress; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.EnumSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.function.Predicate; + +public class MlMetadata implements MetaData.Custom { + + private static final ParseField JOBS_FIELD = new ParseField("jobs"); + private static final ParseField ALLOCATIONS_FIELD = new ParseField("allocations"); + private static final ParseField DATAFEEDS_FIELD = new ParseField("datafeeds"); + + public static final String TYPE = "ml"; + public static final MlMetadata EMPTY_METADATA = new MlMetadata(Collections.emptySortedMap(), + Collections.emptySortedMap(), Collections.emptySortedMap()); + + public static final ObjectParser ML_METADATA_PARSER = new ObjectParser<>("ml_metadata", + Builder::new); + + static { + ML_METADATA_PARSER.declareObjectArray(Builder::putJobs, (p, c) -> Job.PARSER.apply(p, c).build(), JOBS_FIELD); + ML_METADATA_PARSER.declareObjectArray(Builder::putAllocations, Allocation.PARSER, ALLOCATIONS_FIELD); + ML_METADATA_PARSER.declareObjectArray(Builder::putDatafeeds, (p, c) -> DatafeedConfig.PARSER.apply(p, c).build(), DATAFEEDS_FIELD); + } + + private final SortedMap jobs; + private final SortedMap allocations; + private final SortedMap datafeeds; + + private MlMetadata(SortedMap jobs, SortedMap allocations, + SortedMap datafeeds) { + this.jobs = Collections.unmodifiableSortedMap(jobs); + this.allocations = Collections.unmodifiableSortedMap(allocations); + this.datafeeds = Collections.unmodifiableSortedMap(datafeeds); + } + + public Map getJobs() { + return jobs; + } + + public SortedMap getAllocations() { + return allocations; + } + + public SortedMap getDatafeeds() { + return datafeeds; + } + + public DatafeedConfig getDatafeed(String datafeedId) { + return datafeeds.get(datafeedId); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + @Override + public EnumSet context() { + // NORELEASE: Also include SNAPSHOT, but then we need to split the allocations from here and add them + // as ClusterState.Custom metadata, because only the job definitions should be stored in snapshots. + return MetaData.API_AND_GATEWAY; + } + + @Override + public Diff diff(MetaData.Custom previousState) { + return new MlMetadataDiff((MlMetadata) previousState, this); + } + + public MlMetadata(StreamInput in) throws IOException { + int size = in.readVInt(); + TreeMap jobs = new TreeMap<>(); + for (int i = 0; i < size; i++) { + jobs.put(in.readString(), new Job(in)); + } + this.jobs = jobs; + size = in.readVInt(); + TreeMap allocations = new TreeMap<>(); + for (int i = 0; i < size; i++) { + allocations.put(in.readString(), new Allocation(in)); + } + this.allocations = allocations; + size = in.readVInt(); + TreeMap datafeeds = new TreeMap<>(); + for (int i = 0; i < size; i++) { + datafeeds.put(in.readString(), new DatafeedConfig(in)); + } + this.datafeeds = datafeeds; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeMap(jobs, out); + writeMap(allocations, out); + writeMap(datafeeds, out); + } + + private static void writeMap(Map map, StreamOutput out) throws IOException { + out.writeVInt(map.size()); + for (Map.Entry entry : map.entrySet()) { + out.writeString(entry.getKey()); + entry.getValue().writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + mapValuesToXContent(JOBS_FIELD, jobs, builder, params); + mapValuesToXContent(ALLOCATIONS_FIELD, allocations, builder, params); + mapValuesToXContent(DATAFEEDS_FIELD, datafeeds, builder, params); + return builder; + } + + private static void mapValuesToXContent(ParseField field, Map map, XContentBuilder builder, + Params params) throws IOException { + builder.startArray(field.getPreferredName()); + for (Map.Entry entry : map.entrySet()) { + entry.getValue().toXContent(builder, params); + } + builder.endArray(); + } + + public static class MlMetadataDiff implements NamedDiff { + + final Diff> jobs; + final Diff> allocations; + final Diff> datafeeds; + + MlMetadataDiff(MlMetadata before, MlMetadata after) { + this.jobs = DiffableUtils.diff(before.jobs, after.jobs, DiffableUtils.getStringKeySerializer()); + this.allocations = DiffableUtils.diff(before.allocations, after.allocations, DiffableUtils.getStringKeySerializer()); + this.datafeeds = DiffableUtils.diff(before.datafeeds, after.datafeeds, DiffableUtils.getStringKeySerializer()); + } + + public MlMetadataDiff(StreamInput in) throws IOException { + this.jobs = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Job::new, + MlMetadataDiff::readJobDiffFrom); + this.allocations = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), Allocation::new, + MlMetadataDiff::readAllocationDiffFrom); + this.datafeeds = DiffableUtils.readJdkMapDiff(in, DiffableUtils.getStringKeySerializer(), DatafeedConfig::new, + MlMetadataDiff::readSchedulerDiffFrom); + } + + @Override + public MetaData.Custom apply(MetaData.Custom part) { + TreeMap newJobs = new TreeMap<>(jobs.apply(((MlMetadata) part).jobs)); + TreeMap newAllocations = new TreeMap<>(allocations.apply(((MlMetadata) part).allocations)); + TreeMap newDatafeeds = new TreeMap<>(datafeeds.apply(((MlMetadata) part).datafeeds)); + return new MlMetadata(newJobs, newAllocations, newDatafeeds); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + jobs.writeTo(out); + allocations.writeTo(out); + datafeeds.writeTo(out); + } + + @Override + public String getWriteableName() { + return TYPE; + } + + static Diff readJobDiffFrom(StreamInput in) throws IOException { + return AbstractDiffable.readDiffFrom(Job::new, in); + } + + static Diff readAllocationDiffFrom(StreamInput in) throws IOException { + return AbstractDiffable.readDiffFrom(Allocation::new, in); + } + + static Diff readSchedulerDiffFrom(StreamInput in) throws IOException { + return AbstractDiffable.readDiffFrom(DatafeedConfig::new, in); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + MlMetadata that = (MlMetadata) o; + return Objects.equals(jobs, that.jobs) && + Objects.equals(allocations, that.allocations) && + Objects.equals(datafeeds, that.datafeeds); + } + + @Override + public final String toString() { + return Strings.toString(this); + } + + @Override + public int hashCode() { + return Objects.hash(jobs, allocations, datafeeds); + } + + public static class Builder { + + private TreeMap jobs; + private TreeMap allocations; + private TreeMap datafeeds; + + public Builder() { + this.jobs = new TreeMap<>(); + this.allocations = new TreeMap<>(); + this.datafeeds = new TreeMap<>(); + } + + public Builder(MlMetadata previous) { + jobs = new TreeMap<>(previous.jobs); + allocations = new TreeMap<>(previous.allocations); + datafeeds = new TreeMap<>(previous.datafeeds); + } + + public Builder putJob(Job job, boolean overwrite) { + if (jobs.containsKey(job.getId()) && overwrite == false) { + throw ExceptionsHelper.jobAlreadyExists(job.getId()); + } + this.jobs.put(job.getId(), job); + + Allocation allocation = allocations.get(job.getId()); + if (allocation == null) { + Allocation.Builder builder = new Allocation.Builder(job); + builder.setState(JobState.CLOSED); + allocations.put(job.getId(), builder.build()); + } + return this; + } + + public Builder deleteJob(String jobId) { + + Job job = jobs.remove(jobId); + if (job == null) { + throw new ResourceNotFoundException("job [" + jobId + "] does not exist"); + } + + Optional datafeed = getDatafeedByJobId(jobId); + if (datafeed.isPresent()) { + throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] while datafeed [" + + datafeed.get().getId() + "] refers to it"); + } + + Allocation previousAllocation = this.allocations.remove(jobId); + if (previousAllocation != null) { + if (!previousAllocation.getState().equals(JobState.DELETING)) { + throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] because it is in [" + + previousAllocation.getState() + "] state. Must be in [" + JobState.DELETING + "] state."); + } + } else { + throw new ResourceNotFoundException("No Cluster State found for job [" + jobId + "]"); + } + + return this; + } + + public Builder putDatafeed(DatafeedConfig datafeedConfig) { + if (datafeeds.containsKey(datafeedConfig.getId())) { + throw new ResourceAlreadyExistsException("A datafeed with id [" + datafeedConfig.getId() + "] already exists"); + } + String jobId = datafeedConfig.getJobId(); + Job job = jobs.get(jobId); + if (job == null) { + throw ExceptionsHelper.missingJobException(jobId); + } + Optional existingDatafeed = getDatafeedByJobId(jobId); + if (existingDatafeed.isPresent()) { + throw ExceptionsHelper.conflictStatusException("A datafeed [" + existingDatafeed.get().getId() + + "] already exists for job [" + jobId + "]"); + } + DatafeedJobValidator.validate(datafeedConfig, job); + + datafeeds.put(datafeedConfig.getId(), datafeedConfig); + return this; + } + + public Builder removeDatafeed(String datafeedId, PersistentTasksInProgress persistentTasksInProgress) { + DatafeedConfig datafeed = datafeeds.get(datafeedId); + if (datafeed == null) { + throw ExceptionsHelper.missingDatafeedException(datafeedId); + } + if (persistentTasksInProgress != null) { + Predicate> predicate = t -> { + StartDatafeedAction.Request storedRequest = (StartDatafeedAction.Request) t.getRequest(); + return storedRequest.getDatafeedId().equals(datafeedId); + }; + if (persistentTasksInProgress.tasksExist(StartDatafeedAction.NAME, predicate)) { + String msg = Messages.getMessage(Messages.DATAFEED_CANNOT_DELETE_IN_CURRENT_STATE, datafeedId, + DatafeedState.STARTED); + throw ExceptionsHelper.conflictStatusException(msg); + } + } + datafeeds.remove(datafeedId); + return this; + } + + private Optional getDatafeedByJobId(String jobId) { + return datafeeds.values().stream().filter(s -> s.getJobId().equals(jobId)).findFirst(); + } + + // only for parsing + private Builder putAllocations(Collection allocations) { + for (Allocation.Builder allocationBuilder : allocations) { + Allocation allocation = allocationBuilder.build(); + this.allocations.put(allocation.getJobId(), allocation); + } + return this; + } + + private Builder putJobs(Collection jobs) { + for (Job job : jobs) { + putJob(job, true); + } + return this; + } + + private Builder putDatafeeds(Collection datafeeds) { + for (DatafeedConfig datafeed : datafeeds) { + this.datafeeds.put(datafeed.getId(), datafeed); + } + return this; + } + + public MlMetadata build() { + return new MlMetadata(jobs, allocations, datafeeds); + } + + public Builder assignToNode(String jobId, String nodeId) { + Allocation allocation = allocations.get(jobId); + if (allocation == null) { + throw new IllegalStateException("[" + jobId + "] no allocation to assign to node [" + nodeId + "]"); + } + Allocation.Builder builder = new Allocation.Builder(allocation); + builder.setNodeId(nodeId); + allocations.put(jobId, builder.build()); + return this; + } + + public Builder updateState(String jobId, JobState jobState, @Nullable String reason) { + if (jobs.containsKey(jobId) == false) { + throw ExceptionsHelper.missingJobException(jobId); + } + + Allocation previous = allocations.get(jobId); + if (previous == null) { + throw new IllegalStateException("[" + jobId + "] no allocation exist to update the state to [" + jobState + "]"); + } + + // Cannot update the state to DELETING if there are datafeeds attached + if (jobState.equals(JobState.DELETING)) { + Optional datafeed = getDatafeedByJobId(jobId); + if (datafeed.isPresent()) { + throw ExceptionsHelper.conflictStatusException("Cannot delete job [" + jobId + "] while datafeed [" + + datafeed.get().getId() + "] refers to it"); + } + } + + if (previous.getState().equals(JobState.DELETING)) { + // If we're already Deleting there's nothing to do + if (jobState.equals(JobState.DELETING)) { + return this; + } + + // Once a job goes into Deleting, it cannot be changed + throw new ElasticsearchStatusException("Cannot change state of job [" + jobId + "] to [" + jobState + "] because " + + "it is currently in [" + JobState.DELETING + "] state.", RestStatus.CONFLICT); + } + Allocation.Builder builder = new Allocation.Builder(previous); + builder.setState(jobState); + if (reason != null) { + builder.setStateReason(reason); + } + if (previous.getState() != jobState && jobState == JobState.CLOSED) { + Job.Builder job = new Job.Builder(this.jobs.get(jobId)); + job.setFinishedTime(new Date()); + this.jobs.put(job.getId(), job.build()); + } + allocations.put(jobId, builder.build()); + return this; + } + + public Builder setIgnoreDowntime(String jobId) { + if (jobs.containsKey(jobId) == false) { + throw ExceptionsHelper.missingJobException(jobId); + } + + Allocation allocation = allocations.get(jobId); + if (allocation == null) { + throw new IllegalStateException("[" + jobId + "] no allocation to ignore downtime"); + } + Allocation.Builder builder = new Allocation.Builder(allocation); + builder.setIgnoreDowntime(true); + allocations.put(jobId, builder.build()); + return this; + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/AnomalyDetectorsIndex.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/AnomalyDetectorsIndex.java new file mode 100644 index 00000000000..37fc8cf552a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/AnomalyDetectorsIndex.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +/** + * Methods for handling index naming related functions + */ +public final class AnomalyDetectorsIndex { + private static final String RESULTS_INDEX_PREFIX = ".ml-anomalies-"; + private static final String STATE_INDEX_NAME = ".ml-state"; + + private AnomalyDetectorsIndex() { + } + + /** + * The name of the default index where the job's results are stored + * @param jobId Job Id + * @return The index name + */ + public static String jobResultsIndexName(String jobId) { + return RESULTS_INDEX_PREFIX + jobId; + } + + /** + * The name of the default index where a job's state is stored + * @return The index name + */ + public static String jobStateIndexName() { + return STATE_INDEX_NAME; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedBucketsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedBucketsIterator.java new file mode 100644 index 00000000000..c05056e8220 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedBucketsIterator.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.ml.job.results.Bucket; + +import java.io.IOException; + +class BatchedBucketsIterator extends BatchedResultsIterator { + + BatchedBucketsIterator(Client client, String jobId) { + super(client, jobId, Bucket.RESULT_TYPE_VALUE); + } + + @Override + protected ResultWithIndex map(SearchHit hit) { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse bucket", e); + } + Bucket bucket = Bucket.PARSER.apply(parser, null); + return new ResultWithIndex<>(hit.getIndex(), bucket); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java new file mode 100644 index 00000000000..63d91b9ef18 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIterator.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.xpack.ml.job.results.Bucket; + +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.NoSuchElementException; +import java.util.Objects; + +/** + * An iterator useful to fetch a big number of documents of type T + * and iterate through them in batches. + */ +public abstract class BatchedDocumentsIterator { + private static final Logger LOGGER = Loggers.getLogger(BatchedDocumentsIterator.class); + + private static final String CONTEXT_ALIVE_DURATION = "5m"; + private static final int BATCH_SIZE = 10000; + + private final Client client; + private final String index; + private final ResultsFilterBuilder filterBuilder; + private volatile long count; + private volatile long totalHits; + private volatile String scrollId; + private volatile boolean isScrollInitialised; + + public BatchedDocumentsIterator(Client client, String index) { + this(client, index, new ResultsFilterBuilder()); + } + + protected BatchedDocumentsIterator(Client client, String index, QueryBuilder queryBuilder) { + this(client, index, new ResultsFilterBuilder(queryBuilder)); + } + + private BatchedDocumentsIterator(Client client, String index, ResultsFilterBuilder resultsFilterBuilder) { + this.client = Objects.requireNonNull(client); + this.index = Objects.requireNonNull(index); + totalHits = 0; + count = 0; + filterBuilder = Objects.requireNonNull(resultsFilterBuilder); + isScrollInitialised = false; + } + + /** + * Query documents whose timestamp is within the given time range + * + * @param startEpochMs the start time as epoch milliseconds (inclusive) + * @param endEpochMs the end time as epoch milliseconds (exclusive) + * @return the iterator itself + */ + public BatchedDocumentsIterator timeRange(long startEpochMs, long endEpochMs) { + filterBuilder.timeRange(Bucket.TIMESTAMP.getPreferredName(), startEpochMs, endEpochMs); + return this; + } + + /** + * Include interim documents + * + * @param interimFieldName Name of the include interim field + */ + public BatchedDocumentsIterator includeInterim(String interimFieldName) { + filterBuilder.interim(interimFieldName, true); + return this; + } + + /** + * Returns {@code true} if the iteration has more elements. + * (In other words, returns {@code true} if {@link #next} would + * return an element rather than throwing an exception.) + * + * @return {@code true} if the iteration has more elements + */ + public boolean hasNext() { + return !isScrollInitialised || count != totalHits; + } + + /** + * The first time next() is called, the search will be performed and the first + * batch will be returned. Any subsequent call will return the following batches. + *

+ * Note that in some implementations it is possible that when there are no + * results at all, the first time this method is called an empty {@code Deque} is returned. + * + * @return a {@code Deque} with the next batch of documents + * @throws NoSuchElementException if the iteration has no more elements + */ + public Deque next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + SearchResponse searchResponse; + if (scrollId == null) { + searchResponse = initScroll(); + } else { + SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollId).scroll(CONTEXT_ALIVE_DURATION); + searchResponse = client.searchScroll(searchScrollRequest).actionGet(); + } + scrollId = searchResponse.getScrollId(); + return mapHits(searchResponse); + } + + private SearchResponse initScroll() { + LOGGER.trace("ES API CALL: search all of type {} from index {}", getType(), index); + + isScrollInitialised = true; + + SearchRequest searchRequest = new SearchRequest(index); + searchRequest.types(getType()); + searchRequest.scroll(CONTEXT_ALIVE_DURATION); + searchRequest.source(new SearchSourceBuilder() + .size(BATCH_SIZE) + .query(filterBuilder.build()) + .sort(SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC))); + + SearchResponse searchResponse = client.search(searchRequest).actionGet(); + totalHits = searchResponse.getHits().getTotalHits(); + scrollId = searchResponse.getScrollId(); + return searchResponse; + } + + private Deque mapHits(SearchResponse searchResponse) { + Deque results = new ArrayDeque<>(); + + SearchHit[] hits = searchResponse.getHits().getHits(); + for (SearchHit hit : hits) { + T mapped = map(hit); + if (mapped != null) { + results.add(mapped); + } + } + count += hits.length; + + if (!hasNext() && scrollId != null) { + client.prepareClearScroll().setScrollIds(Arrays.asList(scrollId)).get(); + } + return results; + } + + protected abstract String getType(); + + /** + * Maps the search hit to the document type + * @param hit + * the search hit + * @return The mapped document or {@code null} if the mapping failed + */ + protected abstract T map(SearchHit hit); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java new file mode 100644 index 00000000000..2d908786cff --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.ml.job.results.Influencer; + +import java.io.IOException; + +class BatchedInfluencersIterator extends BatchedResultsIterator { + BatchedInfluencersIterator(Client client, String jobId) { + super(client, jobId, Influencer.RESULT_TYPE_VALUE); + } + + @Override + protected ResultWithIndex map(SearchHit hit) { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parser influencer", e); + } + + Influencer influencer = Influencer.PARSER.apply(parser, null); + return new ResultWithIndex<>(hit.getIndex(), influencer); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java new file mode 100644 index 00000000000..71d65802787 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; + +import java.io.IOException; + +class BatchedRecordsIterator extends BatchedResultsIterator { + + BatchedRecordsIterator(Client client, String jobId) { + super(client, jobId, AnomalyRecord.RESULT_TYPE_VALUE); + } + + @Override + protected ResultWithIndex map(SearchHit hit) { + BytesReference source = hit.getSourceRef(); + XContentParser parser; + try { + parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse record", e); + } + AnomalyRecord record = AnomalyRecord.PARSER.apply(parser, null); + return new ResultWithIndex<>(hit.getIndex(), record); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedResultsIterator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedResultsIterator.java new file mode 100644 index 00000000000..eac6a2e1a79 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedResultsIterator.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.xpack.ml.job.results.Result; + +public abstract class BatchedResultsIterator + extends BatchedDocumentsIterator> { + + public BatchedResultsIterator(Client client, String jobId, String resultType) { + super(client, AnomalyDetectorsIndex.jobResultsIndexName(jobId), + new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), resultType)); + } + + @Override + protected String getType() { + return Result.TYPE.getPreferredName(); + } + + public static class ResultWithIndex { + public final String indexName; + public final T result; + + public ResultWithIndex(String indexName, T result) { + this.indexName = indexName; + this.result = result; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilder.java new file mode 100644 index 00000000000..f6fcdf3726e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilder.java @@ -0,0 +1,230 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.xpack.ml.job.results.Bucket; + +import java.util.Objects; + +/** + * One time query builder for buckets. + *

    + *
  • From- Skip the first N Buckets. This parameter is for paging if not + * required set to 0. Default = 0
    • + *
  • Size- Take only this number of Buckets. Default = + * {@value DEFAULT_SIZE}
    • + *
  • Expand- Include anomaly records. Default= false
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= + * this value. Default = 0.0
    • + *
  • normalizedProbabilityThreshold- Return only buckets with a + * maxNormalizedProbability >= this value. Default = 0.0
    • + *
  • start- The start bucket time. A bucket with this timestamp will be + * included in the results. If 0 all buckets up to endEpochMs are + * returned. Default = -1
    • + *
  • end- The end bucket timestamp buckets up to but NOT including this + * timestamp are returned. If 0 all buckets from startEpochMs are + * returned. Default = -1
    • + *
  • partitionValue Set the bucket's max normalized probability to this + * partition field value's max normalized probability. Default = null
    • + *
+ */ +public final class BucketsQueryBuilder { + public static final int DEFAULT_SIZE = 100; + + private BucketsQuery bucketsQuery = new BucketsQuery(); + + public BucketsQueryBuilder from(int from) { + bucketsQuery.from = from; + return this; + } + + public BucketsQueryBuilder size(int size) { + bucketsQuery.size = size; + return this; + } + + public BucketsQueryBuilder expand(boolean expand) { + bucketsQuery.expand = expand; + return this; + } + + public BucketsQueryBuilder includeInterim(boolean include) { + bucketsQuery.includeInterim = include; + return this; + } + + public BucketsQueryBuilder anomalyScoreThreshold(Double anomalyScoreFilter) { + if (anomalyScoreFilter != null) { + bucketsQuery.anomalyScoreFilter = anomalyScoreFilter; + } + return this; + } + + public BucketsQueryBuilder normalizedProbabilityThreshold(Double normalizedProbability) { + if (normalizedProbability != null) { + bucketsQuery.normalizedProbability = normalizedProbability; + } + return this; + } + + /** + * @param partitionValue Not set if null or empty + */ + public BucketsQueryBuilder partitionValue(String partitionValue) { + if (!Strings.isNullOrEmpty(partitionValue)) { + bucketsQuery.partitionValue = partitionValue; + } + return this; + } + + public BucketsQueryBuilder sortField(String sortField) { + bucketsQuery.sortField = sortField; + return this; + } + + public BucketsQueryBuilder sortDescending(boolean sortDescending) { + bucketsQuery.sortDescending = sortDescending; + return this; + } + + /** + * If startTime <= 0 the parameter is not set + */ + public BucketsQueryBuilder start(String startTime) { + bucketsQuery.start = startTime; + return this; + } + + /** + * If endTime <= 0 the parameter is not set + */ + public BucketsQueryBuilder end(String endTime) { + bucketsQuery.end = endTime; + return this; + } + + public BucketsQueryBuilder timestamp(String timestamp) { + bucketsQuery.timestamp = timestamp; + bucketsQuery.size = 1; + return this; + } + + public BucketsQueryBuilder.BucketsQuery build() { + if (bucketsQuery.timestamp != null && (bucketsQuery.start != null || bucketsQuery.end != null)) { + throw new IllegalStateException("Either specify timestamp or start/end"); + } + + return bucketsQuery; + } + + public void clear() { + bucketsQuery = new BucketsQueryBuilder.BucketsQuery(); + } + + + public class BucketsQuery { + private int from = 0; + private int size = DEFAULT_SIZE; + private boolean expand = false; + private boolean includeInterim = false; + private double anomalyScoreFilter = 0.0d; + private double normalizedProbability = 0.0d; + private String start; + private String end; + private String timestamp; + private String partitionValue = null; + private String sortField = Bucket.TIMESTAMP.getPreferredName(); + private boolean sortDescending = false; + + public int getFrom() { + return from; + } + + public int getSize() { + return size; + } + + public boolean isExpand() { + return expand; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public double getNormalizedProbability() { + return normalizedProbability; + } + + public String getStart() { + return start; + } + + public String getEnd() { + return end; + } + + public String getTimestamp() { + return timestamp; + } + + /** + * @return Null if not set + */ + public String getPartitionValue() { + return partitionValue; + } + + public String getSortField() { + return sortField; + } + + public boolean isSortDescending() { + return sortDescending; + } + + @Override + public int hashCode() { + return Objects.hash(from, size, expand, includeInterim, anomalyScoreFilter, normalizedProbability, start, end, + timestamp, partitionValue, sortField, sortDescending); + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + BucketsQuery other = (BucketsQuery) obj; + return Objects.equals(from, other.from) && + Objects.equals(size, other.size) && + Objects.equals(expand, other.expand) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(timestamp, other.timestamp) && + Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && + Objects.equals(normalizedProbability, other.normalizedProbability) && + Objects.equals(partitionValue, other.partitionValue) && + Objects.equals(sortField, other.sortField) && + this.sortDescending == other.sortDescending; + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java new file mode 100644 index 00000000000..d62faa22929 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java @@ -0,0 +1,671 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; +import org.elasticsearch.xpack.ml.job.results.ReservedFieldNames; +import org.elasticsearch.xpack.ml.notifications.AuditActivity; +import org.elasticsearch.xpack.ml.notifications.AuditMessage; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.results.AnomalyCause; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Influence; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.ml.job.results.PerPartitionMaxProbabilities; +import org.elasticsearch.xpack.ml.job.results.Result; + +import java.io.IOException; +import java.util.Collection; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * Static methods to create Elasticsearch mappings for the autodetect + * persisted objects/documents + *

+ * ElasticSearch automatically recognises array types so they are + * not explicitly mapped as such. For arrays of objects the type + * must be set to nested so the arrays are searched properly + * see https://www.elastic.co/guide/en/elasticsearch/guide/current/nested-objects.html + *

+ * It is expected that indexes to which these mappings are applied have their + * default analyzer set to "keyword", which does not tokenise fields. The + * index-wide default analyzer cannot be set via these mappings, so needs to be + * set in the index settings during index creation. For the results mapping the + * _all field is disabled and a custom all field is used in its place. The index + * settings must have {@code "index.query.default_field": "all_field_values" } set + * for the queries to use the custom all field. The custom all field has its + * analyzer set to "whitespace" by these mappings, so that it gets tokenised + * using whitespace. + */ +public class ElasticsearchMappings { + /** + * String constants used in mappings + */ + static final String ENABLED = "enabled"; + static final String ANALYZER = "analyzer"; + static final String WHITESPACE = "whitespace"; + static final String NESTED = "nested"; + static final String COPY_TO = "copy_to"; + static final String PROPERTIES = "properties"; + static final String TYPE = "type"; + static final String DYNAMIC = "dynamic"; + + /** + * Name of the custom 'all' field for results + */ + public static final String ALL_FIELD_VALUES = "all_field_values"; + + /** + * Name of the Elasticsearch field by which documents are sorted by default + */ + static final String ES_DOC = "_doc"; + + /** + * Elasticsearch data types + */ + static final String BOOLEAN = "boolean"; + static final String DATE = "date"; + static final String DOUBLE = "double"; + static final String INTEGER = "integer"; + static final String KEYWORD = "keyword"; + static final String LONG = "long"; + static final String TEXT = "text"; + + private ElasticsearchMappings() { + } + + /** + * Create the Elasticsearch mapping for results objects + * {@link Bucket}s, {@link AnomalyRecord}s, {@link Influencer} and + * {@link BucketInfluencer} + * + * The mapping has a custom all field containing the *_FIELD_VALUE fields + * e.g. BY_FIELD_VALUE, OVER_FIELD_VALUE, etc. The custom all field {@link #ALL_FIELD_VALUES} + * must be set in the index settings. A custom all field is preferred over the usual + * '_all' field as most fields do not belong in '_all', disabling '_all' and + * using a custom all field simplifies the mapping. + * + * These fields are copied to the custom all field + *

    + *
  • by_field_value
    • + *
  • partition_field_value
    • + *
  • over_field_value
    • + *
  • AnomalyCause.correlated_by_field_value
    • + *
  • AnomalyCause.by_field_value
    • + *
  • AnomalyCause.partition_field_value
    • + *
  • AnomalyCause.over_field_value
    • + *
  • AnomalyRecord.Influencers.influencer_field_values
    • + *
  • Influencer.influencer_field_value
    • + *
+ * + * @param termFieldNames All the term fields (by, over, partition) and influencers + * included in the mapping + * + * @return The mapping + * @throws IOException On write error + */ + public static XContentBuilder resultsMapping(Collection termFieldNames) throws IOException { + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject(Result.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(ALL_FIELD_VALUES) + .field(TYPE, TEXT) + .field(ANALYZER, WHITESPACE) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(Bucket.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(Bucket.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(Bucket.RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.EVENT_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.BUCKET_SPAN.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.PROCESSING_TIME_MS.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.PARTITION_SCORES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + + .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(BucketInfluencer.SEQUENCE_NUM.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(BucketInfluencer.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .endObject() + .endObject() + + // per-partition max probabilities mapping + .startObject(PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() + + // Model Debug Output + .startObject(ModelDebugOutput.DEBUG_FEATURE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelDebugOutput.DEBUG_LOWER.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(ModelDebugOutput.DEBUG_UPPER.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(ModelDebugOutput.DEBUG_MEDIAN.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject(); + + addAnomalyRecordFieldsToMapping(builder); + addInfluencerFieldsToMapping(builder); + addModelSizeStatsFieldsToMapping(builder); + + for (String fieldName : termFieldNames) { + if (ReservedFieldNames.isValidFieldName(fieldName)) { + builder.startObject(fieldName).field(TYPE, KEYWORD).endObject(); + } + } + + // End result properties + builder.endObject(); + // End result + builder.endObject(); + // End mapping + builder.endObject(); + + return builder; + } + + /** + * AnomalyRecord fields to be added under the 'properties' section of the mapping + * @param builder Add properties to this builder + * @return builder + * @throws IOException On write error + */ + private static XContentBuilder addAnomalyRecordFieldsToMapping(XContentBuilder builder) + throws IOException { + builder.startObject(AnomalyRecord.DETECTOR_INDEX.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(AnomalyRecord.SEQUENCE_NUM.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(AnomalyRecord.ACTUAL.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.TYPICAL.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.FUNCTION.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.FUNCTION_DESCRIPTION.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.BY_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(AnomalyRecord.FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(AnomalyRecord.OVER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.OVER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.CAUSES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyCause.ACTUAL.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyCause.TYPICAL.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyCause.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyCause.FUNCTION.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.FUNCTION_DESCRIPTION.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.BY_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(AnomalyCause.CORRELATED_BY_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(AnomalyCause.FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .startObject(AnomalyCause.OVER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyCause.OVER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .endObject() + .endObject() + .startObject(AnomalyRecord.INFLUENCERS.getPreferredName()) + /* Array of influences */ + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(Influence.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Influence.INFLUENCER_FIELD_VALUES.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject() + .endObject() + .endObject(); + + return builder; + } + + private static XContentBuilder addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException { + builder.startObject(Influencer.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Influencer.INFLUENCER_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject(); + + return builder; + } + + /** + * {@link DataCounts} mapping. + * The type is disabled so {@link DataCounts} aren't searchable and + * the '_all' field is disabled + * + * @return The builder + * @throws IOException On builder write error + */ + public static XContentBuilder dataCountsMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(DataCounts.TYPE.getPreferredName()) + .field(ENABLED, false) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_BYTES.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.EARLIEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * {@link CategorizerState} mapping. + * The type is disabled so {@link CategorizerState} is not searchable and + * the '_all' field is disabled + * + * @return The builder + * @throws IOException On builder write error + */ + public static XContentBuilder categorizerStateMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(CategorizerState.TYPE) + .field(ENABLED, false) + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain Quantiles}. + * The type is disabled as is the '_all' field as the document isn't meant to be searched. + *

+ * The quantile state string is not searchable (enabled = false) as it could be + * very large. + */ + public static XContentBuilder quantilesMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(Quantiles.TYPE.getPreferredName()) + .field(ENABLED, false) + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain CategoryDefinition}. + * The '_all' field is disabled as the document isn't meant to be searched. + * + * @return The builder + * @throws IOException On builder error + */ + public static XContentBuilder categoryDefinitionMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(CategoryDefinition.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(CategoryDefinition.CATEGORY_ID.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(CategoryDefinition.TERMS.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(CategoryDefinition.REGEX.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(CategoryDefinition.EXAMPLES.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain ModelState}. + * The model state could potentially be huge (over a gigabyte in size) + * so all analysis by Elasticsearch is disabled. The only way to + * retrieve the model state is by knowing the ID of a particular + * document or by searching for all documents of this type. + */ + public static XContentBuilder modelStateMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(ModelState.TYPE.getPreferredName()) + .field(ENABLED, false) + .endObject() + .endObject(); + } + + /** + * Create the Elasticsearch mapping for {@linkplain ModelSnapshot}. + * The '_all' field is disabled but the type is searchable + */ + public static XContentBuilder modelSnapshotMapping() throws IOException { + XContentBuilder builder = jsonBuilder() + .startObject() + .startObject(ModelSnapshot.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSnapshot.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(ModelSnapshot.RESTORE_PRIORITY.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSnapshot.SNAPSHOT_ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName()) + .field(TYPE, DATE) + .endObject(); + + addModelSizeStatsFieldsToMapping(builder); + + builder.endObject() + .endObject() + .startObject(Quantiles.TYPE.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + + return builder; + } + + /** + * {@link ModelSizeStats} fields to be added under the 'properties' section of the mapping + * @param builder Add properties to this builder + * @return builder + * @throws IOException On write error + */ + private static XContentBuilder addModelSizeStatsFieldsToMapping(XContentBuilder builder) throws IOException { + builder.startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) + .field(TYPE, DATE) + .endObject(); + + return builder; + } + + public static XContentBuilder auditMessageMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(AuditMessage.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AuditMessage.LEVEL.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AuditMessage.MESSAGE.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(AuditMessage.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } + + public static XContentBuilder auditActivityMapping() throws IOException { + return jsonBuilder() + .startObject() + .startObject(AuditActivity.TYPE.getPreferredName()) + .startObject(PROPERTIES) + .startObject(AuditActivity.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .endObject() + .endObject() + .endObject(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java new file mode 100644 index 00000000000..5e7f68c04ae --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.xpack.ml.job.results.Influencer; + +import java.util.Objects; + +/** + * One time query builder for influencers. + *

    + *
  • From- Skip the first N Influencers. This parameter is for paging if not + * required set to 0. Default = 0
    • + *
  • Size- Take only this number of Influencers. Default = + * {@value DEFAULT_SIZE}
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • anomalyScoreThreshold- Return only influencers with an anomalyScore >= + * this value. Default = 0.0
    • + *
  • start- The start influencer time. An influencer with this timestamp will be + * included in the results. If 0 all influencers up to end are + * returned. Default = -1
    • + *
  • end- The end influencer timestamp. Influencers up to but NOT including this + * timestamp are returned. If 0 all influencers from start are + * returned. Default = -1
    • + *
  • partitionValue Set the bucket's max normalized probability to this + * partition field value's max normalized probability. Default = null
    • + *
+ */ +public final class InfluencersQueryBuilder { + public static final int DEFAULT_SIZE = 100; + + private InfluencersQuery influencersQuery = new InfluencersQuery(); + + public InfluencersQueryBuilder from(int from) { + influencersQuery.from = from; + return this; + } + + public InfluencersQueryBuilder size(int size) { + influencersQuery.size = size; + return this; + } + + public InfluencersQueryBuilder includeInterim(boolean include) { + influencersQuery.includeInterim = include; + return this; + } + + public InfluencersQueryBuilder anomalyScoreThreshold(Double anomalyScoreFilter) { + influencersQuery.anomalyScoreFilter = anomalyScoreFilter; + return this; + } + + public InfluencersQueryBuilder sortField(String sortField) { + influencersQuery.sortField = sortField; + return this; + } + + public InfluencersQueryBuilder sortDescending(boolean sortDescending) { + influencersQuery.sortDescending = sortDescending; + return this; + } + + /** + * If startTime >= 0 the parameter is not set + */ + public InfluencersQueryBuilder start(String startTime) { + influencersQuery.start = startTime; + return this; + } + + /** + * If endTime >= 0 the parameter is not set + */ + public InfluencersQueryBuilder end(String endTime) { + influencersQuery.end = endTime; + return this; + } + + public InfluencersQueryBuilder.InfluencersQuery build() { + return influencersQuery; + } + + public void clear() { + influencersQuery = new InfluencersQueryBuilder.InfluencersQuery(); + } + + + public class InfluencersQuery { + private int from = 0; + private int size = DEFAULT_SIZE; + private boolean includeInterim = false; + private double anomalyScoreFilter = 0.0d; + private String start; + private String end; + private String sortField = Influencer.ANOMALY_SCORE.getPreferredName(); + private boolean sortDescending = false; + + public int getFrom() { + return from; + } + + public int getSize() { + return size; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public double getAnomalyScoreFilter() { + return anomalyScoreFilter; + } + + public String getStart() { + return start; + } + + public String getEnd() { + return end; + } + + public String getSortField() { + return sortField; + } + + public boolean isSortDescending() { + return sortDescending; + } + + @Override + public int hashCode() { + return Objects.hash(from, size, includeInterim, anomalyScoreFilter, start, end, sortField, sortDescending); + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + + InfluencersQuery other = (InfluencersQuery) obj; + return Objects.equals(from, other.from) && + Objects.equals(size, other.size) && + Objects.equals(includeInterim, other.includeInterim) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && + Objects.equals(sortField, other.sortField) && + this.sortDescending == other.sortDescending; + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java new file mode 100644 index 00000000000..7f772afadb0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * Update a job's dataCounts + * i.e. the number of processed records, fields etc. + */ +public class JobDataCountsPersister extends AbstractComponent { + + private final Client client; + + public JobDataCountsPersister(Settings settings, Client client) { + super(settings); + this.client = client; + } + + private XContentBuilder serialiseCounts(DataCounts counts) throws IOException { + XContentBuilder builder = jsonBuilder(); + return counts.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + + /** + * Update the job's data counts stats and figures. + * + * @param jobId Job to update + * @param counts The counts + * @param listener Action response listener + */ + public void persistDataCounts(String jobId, DataCounts counts, ActionListener listener) { + try { + XContentBuilder content = serialiseCounts(counts); + client.prepareIndex(AnomalyDetectorsIndex.jobResultsIndexName(jobId), DataCounts.TYPE.getPreferredName(), + DataCounts.documentId(jobId)) + .setSource(content).execute(new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + listener.onResponse(true); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + } catch (IOException ioe) { + logger.warn((Supplier)() -> new ParameterizedMessage("[{}] Error serialising DataCounts stats", jobId), ioe); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java new file mode 100644 index 00000000000..22d9a9fd1a6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteAction; +import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.query.ConstantScoreQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.Result; + +import java.util.Date; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +public class JobDataDeleter { + + private static final Logger LOGGER = Loggers.getLogger(JobDataDeleter.class); + + private static final int SCROLL_SIZE = 1000; + private static final String SCROLL_CONTEXT_DURATION = "5m"; + + private final Client client; + private final String jobId; + private final BulkRequestBuilder bulkRequestBuilder; + private long deletedResultCount; + private long deletedModelSnapshotCount; + private long deletedModelStateCount; + private boolean quiet; + + public JobDataDeleter(Client client, String jobId) { + this(client, jobId, false); + } + + public JobDataDeleter(Client client, String jobId, boolean quiet) { + this.client = Objects.requireNonNull(client); + this.jobId = Objects.requireNonNull(jobId); + bulkRequestBuilder = client.prepareBulk(); + deletedResultCount = 0; + deletedModelSnapshotCount = 0; + deletedModelStateCount = 0; + this.quiet = quiet; + } + + /** + * Asynchronously delete all result types (Buckets, Records, Influencers) from {@code cutOffTime} + * + * @param cutoffEpochMs Results at and after this time will be deleted + * @param listener Response listener + */ + public void deleteResultsFromTime(long cutoffEpochMs, ActionListener listener) { + String index = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + + RangeQueryBuilder timeRange = QueryBuilders.rangeQuery(Bucket.TIMESTAMP.getPreferredName()); + timeRange.gte(cutoffEpochMs); + timeRange.lt(new Date().getTime()); + + RepeatingSearchScrollListener scrollSearchListener = new RepeatingSearchScrollListener(index, listener); + + client.prepareSearch(index) + .setTypes(Result.TYPE.getPreferredName()) + .setFetchSource(false) + .setQuery(timeRange) + .setScroll(SCROLL_CONTEXT_DURATION) + .setSize(SCROLL_SIZE) + .execute(scrollSearchListener); + } + + private void addDeleteRequestForSearchHits(SearchHits hits, String index) { + for (SearchHit hit : hits.getHits()) { + LOGGER.trace("Search hit for result: {}", hit.getId()); + addDeleteRequest(hit, index); + } + deletedResultCount = hits.getTotalHits(); + } + + private void addDeleteRequest(SearchHit hit, String index) { + DeleteRequestBuilder deleteRequest = DeleteAction.INSTANCE.newRequestBuilder(client) + .setIndex(index) + .setType(hit.getType()) + .setId(hit.getId()); + bulkRequestBuilder.add(deleteRequest); + } + + /** + * Delete a {@code ModelSnapshot} + * + * @param modelSnapshot the model snapshot to delete + */ + public void deleteModelSnapshot(ModelSnapshot modelSnapshot) { + String snapshotId = modelSnapshot.getSnapshotId(); + int docCount = modelSnapshot.getSnapshotDocCount(); + String stateIndexName = AnomalyDetectorsIndex.jobStateIndexName(); + // Deduce the document IDs of the state documents from the information + // in the snapshot document - we cannot query the state itself as it's + // too big and has no mappings + for (int i = 0; i < docCount; ++i) { + String stateId = snapshotId + '_' + i; + bulkRequestBuilder.add(client.prepareDelete(stateIndexName, ModelState.TYPE.getPreferredName(), stateId)); + ++deletedModelStateCount; + } + + bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsIndexName(modelSnapshot.getJobId()), + ModelSnapshot.TYPE.getPreferredName(), snapshotId)); + ++deletedModelSnapshotCount; + } + + /** + * Delete all results marked as interim + */ + public void deleteInterimResults() { + String index = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + + QueryBuilder qb = QueryBuilders.termQuery(Bucket.IS_INTERIM.getPreferredName(), true); + + SearchResponse searchResponse = client.prepareSearch(index) + .setTypes(Result.TYPE.getPreferredName()) + .setQuery(new ConstantScoreQueryBuilder(qb)) + .setFetchSource(false) + .setScroll(SCROLL_CONTEXT_DURATION) + .setSize(SCROLL_SIZE) + .get(); + + String scrollId = searchResponse.getScrollId(); + long totalHits = searchResponse.getHits().getTotalHits(); + long totalDeletedCount = 0; + while (totalDeletedCount < totalHits) { + for (SearchHit hit : searchResponse.getHits()) { + LOGGER.trace("Search hit for result: {}", hit.getId()); + ++totalDeletedCount; + addDeleteRequest(hit, index); + ++deletedResultCount; + } + + searchResponse = client.prepareSearchScroll(scrollId).setScroll(SCROLL_CONTEXT_DURATION).get(); + } + } + + /** + * Commit the deletions without enforcing the removal of data from disk + */ + public void commit(ActionListener listener) { + if (bulkRequestBuilder.numberOfActions() == 0) { + listener.onResponse(new BulkResponse(new BulkItemResponse[0], 0L)); + return; + } + + Level logLevel = quiet ? Level.DEBUG : Level.INFO; + LOGGER.log(logLevel, "Requesting deletion of {} results, {} model snapshots and {} model state documents", + deletedResultCount, deletedModelSnapshotCount, deletedModelStateCount); + + try { + bulkRequestBuilder.execute(listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Blocking version of {@linkplain #commit(ActionListener)} + */ + public void commit() { + if (bulkRequestBuilder.numberOfActions() == 0) { + return; + } + + Level logLevel = quiet ? Level.DEBUG : Level.INFO; + LOGGER.log(logLevel, "Requesting deletion of {} results, {} model snapshots and {} model state documents", + deletedResultCount, deletedModelSnapshotCount, deletedModelStateCount); + + BulkResponse response = bulkRequestBuilder.get(); + if (response.hasFailures()) { + LOGGER.debug("Bulk request has failures. {}", response.buildFailureMessage()); + } + } + + /** + * Repeats a scroll search adding the hits to the bulk delete request + */ + private class RepeatingSearchScrollListener implements ActionListener { + + private final AtomicLong totalDeletedCount; + private final String index; + private final ActionListener scrollFinishedListener; + + RepeatingSearchScrollListener(String index, ActionListener scrollFinishedListener) { + totalDeletedCount = new AtomicLong(0L); + this.index = index; + this.scrollFinishedListener = scrollFinishedListener; + } + + @Override + public void onResponse(SearchResponse searchResponse) { + addDeleteRequestForSearchHits(searchResponse.getHits(), index); + + totalDeletedCount.addAndGet(searchResponse.getHits().getHits().length); + if (totalDeletedCount.get() < searchResponse.getHits().getTotalHits()) { + client.prepareSearchScroll(searchResponse.getScrollId()).setScroll(SCROLL_CONTEXT_DURATION).execute(this); + } + else { + scrollFinishedListener.onResponse(true); + } + } + + @Override + public void onFailure(Exception e) { + scrollFinishedListener.onFailure(e); + } + }; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java new file mode 100644 index 00000000000..ed857c252aa --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -0,0 +1,1095 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.get.MultiGetItemResponse; +import org.elasticsearch.action.get.MultiGetRequest; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.UidFieldMapper; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.ml.action.DeleteJobAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder.BucketsQuery; +import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.ml.job.results.PerPartitionMaxProbabilities; +import org.elasticsearch.xpack.ml.job.results.Result; +import org.elasticsearch.xpack.ml.notifications.AuditActivity; +import org.elasticsearch.xpack.ml.notifications.AuditMessage; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Supplier; + +public class JobProvider { + private static final Logger LOGGER = Loggers.getLogger(JobProvider.class); + + /** + * Where to store the ml info in Elasticsearch - must match what's + * expected by kibana/engineAPI/app/directives/mlLogUsage.js + */ + public static final String ML_META_INDEX = ".ml-meta"; + + private static final String ASYNC = "async"; + + private static final List SECONDARY_SORT = Arrays.asList( + AnomalyRecord.ANOMALY_SCORE.getPreferredName(), + AnomalyRecord.OVER_FIELD_VALUE.getPreferredName(), + AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), + AnomalyRecord.BY_FIELD_VALUE.getPreferredName(), + AnomalyRecord.FIELD_NAME.getPreferredName(), + AnomalyRecord.FUNCTION.getPreferredName() + ); + + private static final int RECORDS_SIZE_PARAM = 500; + + + private final Client client; + private final int numberOfReplicas; + + public JobProvider(Client client, int numberOfReplicas) { + this.client = Objects.requireNonNull(client); + this.numberOfReplicas = numberOfReplicas; + } + + /** + * Create the Audit index with the audit message document mapping. + */ + public void createNotificationMessageIndex(BiConsumer listener) { + try { + LOGGER.info("Creating the internal '{}' index", Auditor.NOTIFICATIONS_INDEX); + XContentBuilder auditMessageMapping = ElasticsearchMappings.auditMessageMapping(); + XContentBuilder auditActivityMapping = ElasticsearchMappings.auditActivityMapping(); + + CreateIndexRequest createIndexRequest = new CreateIndexRequest(Auditor.NOTIFICATIONS_INDEX); + createIndexRequest.settings(mlNotificationIndexSettings()); + createIndexRequest.mapping(AuditMessage.TYPE.getPreferredName(), auditMessageMapping); + createIndexRequest.mapping(AuditActivity.TYPE.getPreferredName(), auditActivityMapping); + + client.admin().indices().create(createIndexRequest, + ActionListener.wrap(r -> listener.accept(true, null), e -> listener.accept(false, e))); + } catch (IOException e) { + LOGGER.warn("Error creating mappings for the audit message index", e); + } + } + + /** + * Create the meta index with the filter list document mapping. + */ + public void createMetaIndex(BiConsumer listener) { + LOGGER.info("Creating the internal '{}' index", ML_META_INDEX); + + CreateIndexRequest createIndexRequest = new CreateIndexRequest(ML_META_INDEX); + createIndexRequest.settings(mlNotificationIndexSettings()); + + client.admin().indices().create(createIndexRequest, + ActionListener.wrap(r -> listener.accept(true, null), e -> listener.accept(false, e))); + } + + /** + * Build the Elasticsearch index settings that we want to apply to results + * indexes. It's better to do this in code rather than in elasticsearch.yml + * because then the settings can be applied regardless of whether we're + * using our own Elasticsearch to store results or a customer's pre-existing + * Elasticsearch. + * + * @return An Elasticsearch builder initialised with the desired settings + * for Ml indexes. + */ + Settings.Builder mlResultsIndexSettings() { + return Settings.builder() + // Our indexes are small and one shard puts the + // least possible burden on Elasticsearch + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + // Sacrifice durability for performance: in the event of power + // failure we can lose the last 5 seconds of changes, but it's + // much faster + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), ASYNC) + // We need to allow fields not mentioned in the mappings to + // pick up default mappings and be used in queries + .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), true) + // set the default all search field + .put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), ElasticsearchMappings.ALL_FIELD_VALUES); + } + + /** + * Build the Elasticsearch index settings that we want to apply to the state + * index. It's better to do this in code rather than in elasticsearch.yml + * because then the settings can be applied regardless of whether we're + * using our own Elasticsearch to store results or a customer's pre-existing + * Elasticsearch. + * + * @return An Elasticsearch builder initialised with the desired settings + * for Ml indexes. + */ + Settings.Builder mlStateIndexSettings() { + // TODO review these settings + return Settings.builder() + // Our indexes are small and one shard puts the + // least possible burden on Elasticsearch + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + // Sacrifice durability for performance: in the event of power + // failure we can lose the last 5 seconds of changes, but it's + // much faster + .put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), ASYNC); + } + + /** + * Settings for the notification messages index + * + * @return An Elasticsearch builder initialised with the desired settings + * for Ml indexes. + */ + Settings.Builder mlNotificationIndexSettings() { + return Settings.builder() + // Our indexes are small and one shard puts the + // least possible burden on Elasticsearch + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, numberOfReplicas) + // We need to allow fields not mentioned in the mappings to + // pick up default mappings and be used in queries + .put(MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey(), true); + } + + /** + * Create the Elasticsearch index and the mappings + */ + public void createJobResultIndex(Job job, ActionListener listener) { + Collection termFields = (job.getAnalysisConfig() != null) ? job.getAnalysisConfig().termFields() : Collections.emptyList(); + try { + XContentBuilder resultsMapping = ElasticsearchMappings.resultsMapping(termFields); + XContentBuilder categoryDefinitionMapping = ElasticsearchMappings.categoryDefinitionMapping(); + XContentBuilder dataCountsMapping = ElasticsearchMappings.dataCountsMapping(); + XContentBuilder modelSnapshotMapping = ElasticsearchMappings.modelSnapshotMapping(); + + String jobId = job.getId(); + boolean createIndexAlias = !job.getIndexName().equals(job.getId()); + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(job.getIndexName()); + + LOGGER.trace("ES API CALL: create index {}", indexName); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); + createIndexRequest.settings(mlResultsIndexSettings()); + createIndexRequest.mapping(Result.TYPE.getPreferredName(), resultsMapping); + createIndexRequest.mapping(CategoryDefinition.TYPE.getPreferredName(), categoryDefinitionMapping); + createIndexRequest.mapping(DataCounts.TYPE.getPreferredName(), dataCountsMapping); + createIndexRequest.mapping(ModelSnapshot.TYPE.getPreferredName(), modelSnapshotMapping); + + if (createIndexAlias) { + final ActionListener responseListener = listener; + listener = ActionListener.wrap(aBoolean -> { + client.admin().indices().prepareAliases() + .addAlias(indexName, AnomalyDetectorsIndex.jobResultsIndexName(jobId)) + .execute(ActionListener.wrap(r -> responseListener.onResponse(true), responseListener::onFailure)); + }, + listener::onFailure); + } + + final ActionListener createdListener = listener; + client.admin().indices().create(createIndexRequest, + ActionListener.wrap(r -> createdListener.onResponse(true), createdListener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void createJobStateIndex(BiConsumer listener) { + try { + XContentBuilder categorizerStateMapping = ElasticsearchMappings.categorizerStateMapping(); + XContentBuilder quantilesMapping = ElasticsearchMappings.quantilesMapping(); + XContentBuilder modelStateMapping = ElasticsearchMappings.modelStateMapping(); + + LOGGER.trace("ES API CALL: create state index {}", AnomalyDetectorsIndex.jobStateIndexName()); + CreateIndexRequest createIndexRequest = new CreateIndexRequest(AnomalyDetectorsIndex.jobStateIndexName()); + createIndexRequest.settings(mlStateIndexSettings()); + createIndexRequest.mapping(CategorizerState.TYPE, categorizerStateMapping); + createIndexRequest.mapping(Quantiles.TYPE.getPreferredName(), quantilesMapping); + createIndexRequest.mapping(ModelState.TYPE.getPreferredName(), modelStateMapping); + + client.admin().indices().create(createIndexRequest, + ActionListener.wrap(r -> listener.accept(true, null), e -> listener.accept(false, e))); + } catch (Exception e) { + LOGGER.error("Error creating the " + AnomalyDetectorsIndex.jobStateIndexName() + " index", e); + } + } + + + /** + * Delete all the job related documents from the database. + */ + // TODO: should live together with createJobRelatedIndices (in case it moves)? + public void deleteJobRelatedIndices(String jobId, ActionListener listener) { + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + LOGGER.trace("ES API CALL: delete index {}", indexName); + + try { + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); + client.admin().indices().delete(deleteIndexRequest, + ActionListener.wrap(r -> listener.onResponse(new DeleteJobAction.Response(r.isAcknowledged())), listener::onFailure)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Get the job's data counts + * + * @param jobId The job id + */ + public void dataCounts(String jobId, Consumer handler, Consumer errorHandler) { + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + get(jobId, indexName, DataCounts.TYPE.getPreferredName(), DataCounts.documentId(jobId), handler, errorHandler, + DataCounts.PARSER, () -> new DataCounts(jobId)); + } + + private void get(String jobId, String indexName, String type, String id, Consumer handler, Consumer errorHandler, + BiFunction objectParser, Supplier notFoundSupplier) { + GetRequest getRequest = new GetRequest(indexName, type, id); + client.get(getRequest, ActionListener.wrap( + response -> { + if (response.isExists() == false) { + handler.accept(notFoundSupplier.get()); + } else { + BytesReference source = response.getSourceAsBytesRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + handler.accept(objectParser.apply(parser, null)); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse " + type, e); + } + } + }, + e -> { + if (e instanceof IndexNotFoundException) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + } else { + errorHandler.accept(e); + } + })); + } + + private void mget(String indexName, String type, Set ids, Consumer> handler, Consumer errorHandler, + BiFunction objectParser) { + if (ids.isEmpty()) { + handler.accept(Collections.emptySet()); + return; + } + + MultiGetRequest multiGetRequest = new MultiGetRequest(); + for (String id : ids) { + multiGetRequest.add(indexName, type, id); + } + client.multiGet(multiGetRequest, ActionListener.wrap( + mresponse -> { + Set objects = new HashSet<>(); + for (MultiGetItemResponse item : mresponse) { + GetResponse response = item.getResponse(); + if (response.isExists()) { + BytesReference source = response.getSourceAsBytesRef(); + try (XContentParser parser = XContentFactory.xContent(source) + .createParser(NamedXContentRegistry.EMPTY, source)) { + objects.add(objectParser.apply(parser, null)); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse " + type, e); + } + } + } + handler.accept(objects); + }, + errorHandler) + ); + } + + private Optional getBlocking(String indexName, String type, String id, BiFunction objectParser) { + GetRequest getRequest = new GetRequest(indexName, type, id); + try { + GetResponse response = client.get(getRequest).actionGet(); + if (!response.isExists()) { + return Optional.empty(); + } + BytesReference source = response.getSourceAsBytesRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + return Optional.of(objectParser.apply(parser, null)); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse " + type, e); + } + } catch (IndexNotFoundException e) { + LOGGER.error("Missing index when getting " + type, e); + throw e; + } + } + + /** + * Search for buckets with the parameters in the {@link BucketsQueryBuilder} + */ + public void buckets(String jobId, BucketsQuery query, Consumer> handler, Consumer errorHandler) + throws ResourceNotFoundException { + ResultsFilterBuilder rfb = new ResultsFilterBuilder(); + if (query.getTimestamp() != null) { + rfb.timeRange(Bucket.TIMESTAMP.getPreferredName(), query.getTimestamp()); + } else { + rfb.timeRange(Bucket.TIMESTAMP.getPreferredName(), query.getStart(), query.getEnd()) + .score(Bucket.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreFilter()) + .score(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), query.getNormalizedProbability()) + .interim(Bucket.IS_INTERIM.getPreferredName(), query.isIncludeInterim()); + } + + SortBuilder sortBuilder = new FieldSortBuilder(query.getSortField()) + .order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC); + + QueryBuilder boolQuery = new BoolQueryBuilder() + .filter(rfb.build()) + .filter(QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), Bucket.RESULT_TYPE_VALUE)); + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.types(Result.TYPE.getPreferredName()); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.sort(sortBuilder); + searchSourceBuilder.query(boolQuery); + searchSourceBuilder.from(query.getFrom()); + searchSourceBuilder.size(query.getSize()); + searchRequest.source(searchSourceBuilder); + + MultiSearchRequest mrequest = new MultiSearchRequest(); + mrequest.add(searchRequest); + if (Strings.hasLength(query.getPartitionValue())) { + mrequest.add(createPartitionMaxNormailizedProbabilitiesRequest(jobId, query.getStart(), query.getEnd(), + query.getPartitionValue())); + } + + client.multiSearch(mrequest, ActionListener.wrap(mresponse -> { + MultiSearchResponse.Item item1 = mresponse.getResponses()[0]; + if (item1.isFailure()) { + Exception e = item1.getFailure(); + if (e instanceof IndexNotFoundException) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + } else { + errorHandler.accept(e); + } + return; + } + + SearchResponse searchResponse = item1.getResponse(); + SearchHits hits = searchResponse.getHits(); + if (query.getTimestamp() != null) { + if (hits.getTotalHits() == 0) { + throw QueryPage.emptyQueryPage(Bucket.RESULTS_FIELD); + } else if (hits.getTotalHits() > 1) { + LOGGER.error("Found more than one bucket with timestamp [{}]" + " from index {}", query.getTimestamp(), indexName); + } + } + + List results = new ArrayList<>(); + for (SearchHit hit : hits.getHits()) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + Bucket bucket = Bucket.PARSER.apply(parser, null); + if (query.isIncludeInterim() || bucket.isInterim() == false) { + results.add(bucket); + } + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse bucket", e); + } + } + + if (query.getTimestamp() != null && results.isEmpty()) { + throw QueryPage.emptyQueryPage(Bucket.RESULTS_FIELD); + } + + QueryPage buckets = new QueryPage<>(results, searchResponse.getHits().getTotalHits(), Bucket.RESULTS_FIELD); + if (Strings.hasLength(query.getPartitionValue())) { + MultiSearchResponse.Item item2 = mresponse.getResponses()[1]; + if (item2.isFailure()) { + Exception e = item2.getFailure(); + if (e instanceof IndexNotFoundException) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + } else { + errorHandler.accept(e); + } + return; + } + List partitionProbs = + handlePartitionMaxNormailizedProbabilitiesResponse(item2.getResponse()); + mergePartitionScoresIntoBucket(partitionProbs, buckets.results(), query.getPartitionValue()); + + if (query.isExpand()) { + Iterator bucketsToExpand = buckets.results().stream() + .filter(bucket -> bucket.getRecordCount() > 0).iterator(); + expandBuckets(jobId, query, buckets, bucketsToExpand, 0, handler, errorHandler); + return; + } + } else { + if (query.isExpand()) { + Iterator bucketsToExpand = buckets.results().stream() + .filter(bucket -> bucket.getRecordCount() > 0).iterator(); + expandBuckets(jobId, query, buckets, bucketsToExpand, 0, handler, errorHandler); + return; + } + } + handler.accept(buckets); + }, errorHandler)); + } + + private void expandBuckets(String jobId, BucketsQuery query, QueryPage buckets, Iterator bucketsToExpand, + int from, Consumer> handler, Consumer errorHandler) { + if (bucketsToExpand.hasNext()) { + Consumer c = i -> { + expandBuckets(jobId, query, buckets, bucketsToExpand, from + RECORDS_SIZE_PARAM, handler, errorHandler); + }; + expandBucket(jobId, query.isIncludeInterim(), bucketsToExpand.next(), query.getPartitionValue(), from, c, errorHandler); + } else { + handler.accept(buckets); + } + } + + void mergePartitionScoresIntoBucket(List partitionProbs, List buckets, String partitionValue) { + Iterator itr = partitionProbs.iterator(); + PerPartitionMaxProbabilities partitionProb = itr.hasNext() ? itr.next() : null; + for (Bucket b : buckets) { + if (partitionProb == null) { + b.setMaxNormalizedProbability(0.0); + } else { + if (partitionProb.getTimestamp().equals(b.getTimestamp())) { + b.setMaxNormalizedProbability(partitionProb.getMaxProbabilityForPartition(partitionValue)); + partitionProb = itr.hasNext() ? itr.next() : null; + } else { + b.setMaxNormalizedProbability(0.0); + } + } + } + } + + private SearchRequest createPartitionMaxNormailizedProbabilitiesRequest(String jobId, Object epochStart, Object epochEnd, + String partitionFieldValue) { + QueryBuilder timeRangeQuery = new ResultsFilterBuilder() + .timeRange(Bucket.TIMESTAMP.getPreferredName(), epochStart, epochEnd) + .build(); + + QueryBuilder boolQuery = new BoolQueryBuilder() + .filter(timeRangeQuery) + .filter(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), PerPartitionMaxProbabilities.RESULT_TYPE_VALUE)) + .filter(new TermsQueryBuilder(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue)); + + FieldSortBuilder sb = new FieldSortBuilder(Bucket.TIMESTAMP.getPreferredName()).order(SortOrder.ASC); + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.sort(sb); + sourceBuilder.query(boolQuery); + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.source(sourceBuilder); + return searchRequest; + } + + private List handlePartitionMaxNormailizedProbabilitiesResponse(SearchResponse searchResponse) { + List results = new ArrayList<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + results.add(PerPartitionMaxProbabilities.PARSER.apply(parser, null)); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse PerPartitionMaxProbabilities", e); + } + } + return results; + } + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a large number of buckets of the given job. + * The bucket and source indexes are returned by the iterator. + * + * @param jobId the id of the job for which buckets are requested + * @return a bucket {@link BatchedDocumentsIterator} + */ + public BatchedDocumentsIterator> newBatchedBucketsIterator(String jobId) { + return new BatchedBucketsIterator(client, jobId); + } + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a large number of records in the given job + * The records and source indexes are returned by the iterator. + * + * @param jobId the id of the job for which buckets are requested + * @return a record {@link BatchedDocumentsIterator} + */ + public BatchedDocumentsIterator> + newBatchedRecordsIterator(String jobId) { + return new BatchedRecordsIterator(client, jobId); + } + + // TODO (norelease): Use scroll search instead of multiple searches with increasing from + public void expandBucket(String jobId, boolean includeInterim, Bucket bucket, String partitionFieldValue, int from, + Consumer consumer, Consumer errorHandler) { + Consumer> h = page -> { + bucket.getRecords().addAll(page.results()); + if (partitionFieldValue != null) { + bucket.setAnomalyScore(bucket.partitionAnomalyScore(partitionFieldValue)); + } + if (page.count() > from + RECORDS_SIZE_PARAM) { + expandBucket(jobId, includeInterim, bucket, partitionFieldValue, from + RECORDS_SIZE_PARAM, consumer, errorHandler); + } else { + consumer.accept(bucket.getRecords().size()); + } + }; + bucketRecords(jobId, bucket, from, RECORDS_SIZE_PARAM, includeInterim, AnomalyRecord.PROBABILITY.getPreferredName(), + false, partitionFieldValue, h, errorHandler); + } + + // keep blocking variant around for ScoresUpdater as that can remain a blocking as this is ran from dedicated ml threadpool. + // also refactoring that to be non blocking is a lot of work. + public int expandBucket(String jobId, boolean includeInterim, Bucket bucket) { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference holder = new AtomicReference<>(); + AtomicReference errorHolder = new AtomicReference<>(); + expandBucket(jobId, includeInterim, bucket, null, 0, records -> { + holder.set(records); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (errorHolder.get() != null) { + throw new RuntimeException(errorHolder.get()); + } else { + return holder.get(); + } + } + + void bucketRecords(String jobId, Bucket bucket, int from, int size, boolean includeInterim, String sortField, + boolean descending, String partitionFieldValue, Consumer> handler, + Consumer errorHandler) { + // Find the records using the time stamp rather than a parent-child + // relationship. The parent-child filter involves two queries behind + // the scenes, and Elasticsearch documentation claims it's significantly + // slower. Here we rely on the record timestamps being identical to the + // bucket timestamp. + QueryBuilder recordFilter = QueryBuilders.termQuery(Bucket.TIMESTAMP.getPreferredName(), bucket.getTimestamp().getTime()); + + ResultsFilterBuilder builder = new ResultsFilterBuilder(recordFilter) + .interim(AnomalyRecord.IS_INTERIM.getPreferredName(), includeInterim); + if (partitionFieldValue != null) { + builder.term(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + recordFilter = builder.build(); + + FieldSortBuilder sb = null; + if (sortField != null) { + sb = new FieldSortBuilder(sortField) + .missing("_last") + .order(descending ? SortOrder.DESC : SortOrder.ASC); + } + + records(jobId, from, size, recordFilter, sb, SECONDARY_SORT, descending, handler, errorHandler); + } + + /** + * Get a page of {@linkplain CategoryDefinition}s for the given jobId. + * + * @param jobId the job id + * @param from Skip the first N categories. This parameter is for paging + * @param size Take only this number of categories + */ + public void categoryDefinitions(String jobId, String categoryId, Integer from, Integer size, + Consumer> handler, + Consumer errorHandler) { + if (categoryId != null && (from != null || size != null)) { + throw new IllegalStateException("Both categoryId and pageParams are specified"); + } + + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + LOGGER.trace("ES API CALL: search all of type {} from index {} sort ascending {} from {} size {}", + CategoryDefinition.TYPE.getPreferredName(), indexName, CategoryDefinition.CATEGORY_ID.getPreferredName(), from, size); + + SearchRequest searchRequest = new SearchRequest(indexName); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + if (categoryId != null) { + String documentId = CategoryDefinition.documentId(jobId, categoryId); + String uid = Uid.createUid(CategoryDefinition.TYPE.getPreferredName(), documentId); + sourceBuilder.query(QueryBuilders.termQuery(UidFieldMapper.NAME, uid)); + searchRequest.routing(documentId); + } else if (from != null && size != null) { + searchRequest.types(CategoryDefinition.TYPE.getPreferredName()); + sourceBuilder.from(from).size(size) + .sort(new FieldSortBuilder(CategoryDefinition.CATEGORY_ID.getPreferredName()).order(SortOrder.ASC)); + } else { + throw new IllegalStateException("Both categoryId and pageParams are not specified"); + } + searchRequest.source(sourceBuilder); + client.search(searchRequest, ActionListener.wrap(searchResponse -> { + SearchHit[] hits = searchResponse.getHits().getHits(); + List results = new ArrayList<>(hits.length); + for (SearchHit hit : hits) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + CategoryDefinition categoryDefinition = CategoryDefinition.PARSER.apply(parser, null); + results.add(categoryDefinition); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse category definition", e); + } + } + QueryPage result = + new QueryPage<>(results, searchResponse.getHits().getTotalHits(), CategoryDefinition.RESULTS_FIELD); + handler.accept(result); + }, e -> { + if (e instanceof IndexNotFoundException) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + } else { + errorHandler.accept(e); + } + })); + } + + /** + * Search for anomaly records with the parameters in the + * {@link org.elasticsearch.xpack.ml.job.persistence.RecordsQueryBuilder.RecordsQuery} + */ + public void records(String jobId, RecordsQueryBuilder.RecordsQuery query, Consumer> handler, + Consumer errorHandler) { + QueryBuilder fb = new ResultsFilterBuilder() + .timeRange(Bucket.TIMESTAMP.getPreferredName(), query.getStart(), query.getEnd()) + .score(AnomalyRecord.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreThreshold()) + .score(AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName(), query.getNormalizedProbabilityThreshold()) + .interim(AnomalyRecord.IS_INTERIM.getPreferredName(), query.isIncludeInterim()) + .term(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), query.getPartitionFieldValue()).build(); + FieldSortBuilder sb = null; + if (query.getSortField() != null) { + sb = new FieldSortBuilder(query.getSortField()) + .missing("_last") + .order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC); + } + records(jobId, query.getFrom(), query.getSize(), fb, sb, SECONDARY_SORT, query.isSortDescending(), handler, errorHandler); + } + + /** + * The returned records have their id set. + */ + private void records(String jobId, int from, int size, + QueryBuilder recordFilter, FieldSortBuilder sb, List secondarySort, + boolean descending, Consumer> handler, + Consumer errorHandler) { + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + + recordFilter = new BoolQueryBuilder() + .filter(recordFilter) + .filter(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), AnomalyRecord.RESULT_TYPE_VALUE)); + + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.types(Result.TYPE.getPreferredName()); + searchRequest.source(new SearchSourceBuilder() + .from(from) + .size(size) + .query(recordFilter) + .sort(sb == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC) : sb) + .fetchSource(true) + ); + + for (String sortField : secondarySort) { + searchRequest.source().sort(sortField, descending ? SortOrder.DESC : SortOrder.ASC); + } + + LOGGER.trace("ES API CALL: search all of result type {} from index {}{}{} with filter after sort from {} size {}", + AnomalyRecord.RESULT_TYPE_VALUE, indexName, (sb != null) ? " with sort" : "", + secondarySort.isEmpty() ? "" : " with secondary sort", from, size); + client.search(searchRequest, ActionListener.wrap(searchResponse -> { + List results = new ArrayList<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + results.add(AnomalyRecord.PARSER.apply(parser, null)); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse records", e); + } + } + QueryPage queryPage = + new QueryPage<>(results, searchResponse.getHits().getTotalHits(), AnomalyRecord.RESULTS_FIELD); + handler.accept(queryPage); + }, e -> { + if (e instanceof IndexNotFoundException) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + } else { + errorHandler.accept(e); + } + })); + } + + /** + * Return a page of influencers for the given job and within the given date + * range + * + * @param jobId The job ID for which influencers are requested + * @param query the query + */ + public void influencers(String jobId, InfluencersQuery query, Consumer> handler, + Consumer errorHandler) { + QueryBuilder fb = new ResultsFilterBuilder() + .timeRange(Bucket.TIMESTAMP.getPreferredName(), query.getStart(), query.getEnd()) + .score(Bucket.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreFilter()) + .interim(Bucket.IS_INTERIM.getPreferredName(), query.isIncludeInterim()) + .build(); + + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + LOGGER.trace("ES API CALL: search all of result type {} from index {}{} with filter from {} size {}", + () -> Influencer.RESULT_TYPE_VALUE, () -> indexName, + () -> (query.getSortField() != null) ? + " with sort " + (query.isSortDescending() ? "descending" : "ascending") + " on field " + query.getSortField() : "", + query::getFrom, query::getSize); + + QueryBuilder qb = new BoolQueryBuilder() + .filter(fb) + .filter(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), Influencer.RESULT_TYPE_VALUE)); + + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.types(Result.TYPE.getPreferredName()); + FieldSortBuilder sb = query.getSortField() == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC) + : new FieldSortBuilder(query.getSortField()).order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC); + searchRequest.source(new SearchSourceBuilder().query(qb).from(query.getFrom()).size(query.getSize()).sort(sb)); + + client.search(searchRequest, ActionListener.wrap(response -> { + List influencers = new ArrayList<>(); + for (SearchHit hit : response.getHits().getHits()) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + influencers.add(Influencer.PARSER.apply(parser, null)); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse influencer", e); + } + } + QueryPage result = new QueryPage<>(influencers, response.getHits().getTotalHits(), Influencer.RESULTS_FIELD); + handler.accept(result); + }, errorHandler)); + } + + /** + * Returns a {@link BatchedDocumentsIterator} that allows querying + * and iterating over a large number of influencers of the given job + * + * @param jobId the id of the job for which influencers are requested + * @return an influencer {@link BatchedDocumentsIterator} + */ + public BatchedDocumentsIterator> + newBatchedInfluencersIterator(String jobId) { + return new BatchedInfluencersIterator(client, jobId); + } + + /** + * Get the persisted quantiles state for the job + */ + public void getQuantiles(String jobId, Consumer handler, Consumer errorHandler) { + String indexName = AnomalyDetectorsIndex.jobStateIndexName(); + String quantilesId = Quantiles.documentId(jobId); + LOGGER.trace("ES API CALL: get ID {} type {} from index {}", quantilesId, Quantiles.TYPE.getPreferredName(), indexName); + get(jobId, indexName, Quantiles.TYPE.getPreferredName(), quantilesId, handler, errorHandler, Quantiles.PARSER, () -> { + LOGGER.info("There are currently no quantiles for job " + jobId); + return null; + }); + } + + /** + * Get model snapshots for the job ordered by descending restore priority. + * + * @param jobId the job id + * @param from number of snapshots to from + * @param size number of snapshots to retrieve + */ + public void modelSnapshots(String jobId, int from, int size, Consumer> handler, + Consumer errorHandler) { + modelSnapshots(jobId, from, size, null, false, QueryBuilders.matchAllQuery(), handler, errorHandler); + } + + /** + * Get model snapshots for the job ordered by descending restore priority. + * + * @param jobId the job id + * @param from number of snapshots to from + * @param size number of snapshots to retrieve + * @param startEpochMs earliest time to include (inclusive) + * @param endEpochMs latest time to include (exclusive) + * @param sortField optional sort field name (may be null) + * @param sortDescending Sort in descending order + * @param snapshotId optional snapshot ID to match (null for all) + * @param description optional description to match (null for all) + */ + public void modelSnapshots(String jobId, + int from, + int size, + String startEpochMs, + String endEpochMs, + String sortField, + boolean sortDescending, + String snapshotId, + String description, + Consumer> handler, + Consumer errorHandler) { + boolean haveId = snapshotId != null && !snapshotId.isEmpty(); + boolean haveDescription = description != null && !description.isEmpty(); + ResultsFilterBuilder fb; + if (haveId || haveDescription) { + BoolQueryBuilder query = QueryBuilders.boolQuery(); + if (haveId) { + query.filter(QueryBuilders.termQuery(ModelSnapshot.SNAPSHOT_ID.getPreferredName(), snapshotId)); + } + if (haveDescription) { + query.filter(QueryBuilders.termQuery(ModelSnapshot.DESCRIPTION.getPreferredName(), description)); + } + + fb = new ResultsFilterBuilder(query); + } else { + fb = new ResultsFilterBuilder(); + } + + QueryBuilder qb = fb.timeRange(Bucket.TIMESTAMP.getPreferredName(), startEpochMs, endEpochMs).build(); + modelSnapshots(jobId, from, size, sortField, sortDescending, qb, handler, errorHandler); + } + + private void modelSnapshots(String jobId, + int from, + int size, + String sortField, + boolean sortDescending, + QueryBuilder qb, + Consumer> handler, + Consumer errorHandler) { + if (Strings.isEmpty(sortField)) { + sortField = ModelSnapshot.RESTORE_PRIORITY.getPreferredName(); + } + + FieldSortBuilder sb = new FieldSortBuilder(sortField) + .order(sortDescending ? SortOrder.DESC : SortOrder.ASC); + + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + LOGGER.trace("ES API CALL: search all of type {} from index {} sort ascending {} with filter after sort from {} size {}", + ModelSnapshot.TYPE, indexName, sortField, from, size); + + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.types(ModelSnapshot.TYPE.getPreferredName()); + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); + sourceBuilder.sort(sb); + sourceBuilder.query(qb); + sourceBuilder.from(from); + sourceBuilder.size(size); + searchRequest.source(sourceBuilder); + client.search(searchRequest, ActionListener.wrap(searchResponse -> { + List results = new ArrayList<>(); + for (SearchHit hit : searchResponse.getHits().getHits()) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + ModelSnapshot modelSnapshot = ModelSnapshot.PARSER.apply(parser, null); + results.add(modelSnapshot); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse modelSnapshot", e); + } + } + + QueryPage result = + new QueryPage<>(results, searchResponse.getHits().getTotalHits(), ModelSnapshot.RESULTS_FIELD); + handler.accept(result); + }, errorHandler)); + } + + /** + * Given a model snapshot, get the corresponding state and write it to the supplied + * stream. If there are multiple state documents they are separated using '\0' + * when written to the stream. + * + * @param jobId the job id + * @param modelSnapshot the model snapshot to be restored + * @param restoreStream the stream to write the state to + */ + public void restoreStateToStream(String jobId, ModelSnapshot modelSnapshot, OutputStream restoreStream) throws IOException { + String indexName = AnomalyDetectorsIndex.jobStateIndexName(); + + // First try to restore categorizer state. There are no snapshots for this, so the IDs simply + // count up until a document is not found. It's NOT an error to have no categorizer state. + int docNum = 0; + while (true) { + String docId = CategorizerState.categorizerStateDocId(jobId, ++docNum); + + LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, CategorizerState.TYPE, indexName); + + GetResponse stateResponse = client.prepareGet(indexName, CategorizerState.TYPE, docId).get(); + if (!stateResponse.isExists()) { + break; + } + writeStateToStream(stateResponse.getSourceAsBytesRef(), restoreStream); + } + + // Finally try to restore model state. This must come after categorizer state because that's + // the order the C++ process expects. + int numDocs = modelSnapshot.getSnapshotDocCount(); + for (docNum = 1; docNum <= numDocs; ++docNum) { + String docId = String.format(Locale.ROOT, "%s_%d", modelSnapshot.getSnapshotId(), docNum); + + LOGGER.trace("ES API CALL: get ID {} type {} from index {}", docId, ModelState.TYPE, indexName); + + GetResponse stateResponse = client.prepareGet(indexName, ModelState.TYPE.getPreferredName(), docId).get(); + if (!stateResponse.isExists()) { + LOGGER.error("Expected {} documents for model state for {} snapshot {} but failed to find {}", + numDocs, jobId, modelSnapshot.getSnapshotId(), docId); + break; + } + writeStateToStream(stateResponse.getSourceAsBytesRef(), restoreStream); + } + } + + private void writeStateToStream(BytesReference source, OutputStream stream) throws IOException { + // The source bytes are already UTF-8. The C++ process wants UTF-8, so we + // can avoid converting to a Java String only to convert back again. + BytesRefIterator iterator = source.iterator(); + for (BytesRef ref = iterator.next(); ref != null; ref = iterator.next()) { + // There's a complication that the source can already have trailing 0 bytes + int length = ref.bytes.length; + while (length > 0 && ref.bytes[length - 1] == 0) { + --length; + } + if (length > 0) { + stream.write(ref.bytes, 0, length); + } + } + // This is dictated by RapidJSON on the C++ side; it treats a '\0' as end-of-file + // even when it's not really end-of-file, and this is what we need because we're + // sending multiple JSON documents via the same named pipe. + stream.write(0); + } + + public QueryPage modelDebugOutput(String jobId, int from, int size) { + SearchResponse searchResponse; + try { + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + LOGGER.trace("ES API CALL: search result type {} from index {} from {}, size {}", + ModelDebugOutput.RESULT_TYPE_VALUE, indexName, from, size); + + searchResponse = client.prepareSearch(indexName) + .setTypes(Result.TYPE.getPreferredName()) + .setQuery(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), ModelDebugOutput.RESULT_TYPE_VALUE)) + .setFrom(from).setSize(size) + .get(); + } catch (IndexNotFoundException e) { + throw ExceptionsHelper.missingJobException(jobId); + } + + List results = new ArrayList<>(); + + for (SearchHit hit : searchResponse.getHits().getHits()) { + BytesReference source = hit.getSourceRef(); + try (XContentParser parser = XContentFactory.xContent(source).createParser(NamedXContentRegistry.EMPTY, source)) { + ModelDebugOutput modelDebugOutput = ModelDebugOutput.PARSER.apply(parser, null); + results.add(modelDebugOutput); + } catch (IOException e) { + throw new ElasticsearchParseException("failed to parse modelDebugOutput", e); + } + } + + return new QueryPage<>(results, searchResponse.getHits().getTotalHits(), ModelDebugOutput.RESULTS_FIELD); + } + + /** + * Get the job's model size stats. + */ + public void modelSizeStats(String jobId, Consumer handler, Consumer errorHandler) { + LOGGER.trace("ES API CALL: get result type {} ID {} for job {}", + ModelSizeStats.RESULT_TYPE_VALUE, ModelSizeStats.RESULT_TYPE_FIELD, jobId); + + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + get(jobId, indexName, Result.TYPE.getPreferredName(), ModelSizeStats.documentId(jobId), + handler, errorHandler, (parser, context) -> ModelSizeStats.PARSER.apply(parser, null).build(), + () -> { + LOGGER.trace("No memory usage details for job with id {}", jobId); + return null; + }); + } + + /** + * Retrieves the filter with the given {@code filterId} from the datastore. + * + * @param ids the id of the requested filter + */ + public void getFilters(Consumer> handler, Consumer errorHandler, Set ids) { + mget(ML_META_INDEX, MlFilter.TYPE.getPreferredName(), ids, handler, errorHandler, MlFilter.PARSER); + } + + /** + * Get an auditor for the given job + * + * @param jobId the job id + * @return the {@code Auditor} + */ + public Auditor audit(String jobId) { + return new Auditor(client, jobId); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java new file mode 100644 index 00000000000..fffc96fc677 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.process.normalizer.BucketNormalizable; +import org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; +import org.elasticsearch.xpack.ml.job.results.Result; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + + +/** + * Interface for classes that update {@linkplain Bucket Buckets} + * for a particular job with new normalized anomaly scores and + * unusual scores. + *

+ * Renormalized results must already have an ID. + *

+ * This class is NOT thread safe. + */ +public class JobRenormalizedResultsPersister extends AbstractComponent { + + private final Client client; + private BulkRequest bulkRequest; + + public JobRenormalizedResultsPersister(Settings settings, Client client) { + super(settings); + this.client = client; + bulkRequest = new BulkRequest(); + } + + public void updateBucket(BucketNormalizable normalizable) { + updateResult(normalizable.getId(), normalizable.getOriginatingIndex(), normalizable.getBucket()); + updateBucketInfluencersStandalone(normalizable.getOriginatingIndex(), normalizable.getBucket().getBucketInfluencers()); + } + + private void updateBucketInfluencersStandalone(String indexName, List bucketInfluencers) { + if (bucketInfluencers != null && bucketInfluencers.isEmpty() == false) { + for (BucketInfluencer bucketInfluencer : bucketInfluencers) { + updateResult(bucketInfluencer.getId(), indexName, bucketInfluencer); + } + } + } + + public void updateResults(List normalizables) { + for (Normalizable normalizable : normalizables) { + updateResult(normalizable.getId(), normalizable.getOriginatingIndex(), normalizable); + } + } + + public void updateResult(String id, String index, ToXContent resultDoc) { + try { + XContentBuilder content = toXContentBuilder(resultDoc); + bulkRequest.add(new IndexRequest(index, Result.TYPE.getPreferredName(), id).source(content)); + } catch (IOException e) { + logger.error("Error serialising result", e); + } + } + + private XContentBuilder toXContentBuilder(ToXContent obj) throws IOException { + XContentBuilder builder = jsonBuilder(); + obj.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + /** + * Execute the bulk action + * + * @param jobId The job Id + */ + public void executeRequest(String jobId) { + if (bulkRequest.numberOfActions() == 0) { + return; + } + logger.trace("[{}] ES API CALL: bulk request with {} actions", jobId, bulkRequest.numberOfActions()); + + BulkResponse addRecordsResponse = client.bulk(bulkRequest).actionGet(); + if (addRecordsResponse.hasFailures()) { + logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage()); + } + + bulkRequest = new BulkRequest(); + } + + BulkRequest getBulkRequest() { + return bulkRequest; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java new file mode 100644 index 00000000000..c501be6a633 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java @@ -0,0 +1,376 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.ml.job.results.PerPartitionMaxProbabilities; +import org.elasticsearch.xpack.ml.job.results.Result; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Consumer; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +/** + * Persists result types, Quantiles etc to Elasticsearch
+ *

Bucket

Bucket result. The anomaly score of the bucket may not match the summed + * score of all the records as all the records may not have been outputted for the + * bucket. Contains bucket influencers that are persisted both with the bucket + * and separately. + * Anomaly Record Each record was generated by a detector which can be identified via + * the detectorIndex field. + * Influencers + * Quantiles may contain model quantiles used in normalization and are + * stored in documents of type {@link Quantiles#TYPE}
+ * ModelSizeStats This is stored in a flat structure
+ * ModelSnapShot This is stored in a flat structure
+ * + * @see org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings + */ +public class JobResultsPersister extends AbstractComponent { + + private final Client client; + + + public JobResultsPersister(Settings settings, Client client) { + super(settings); + this.client = client; + } + + public Builder bulkPersisterBuilder(String jobId) { + return new Builder(jobId); + } + + public class Builder { + private BulkRequest bulkRequest; + private final String jobId; + private final String indexName; + + private Builder(String jobId) { + this.jobId = Objects.requireNonNull(jobId); + indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + bulkRequest = new BulkRequest(); + } + + /** + * Persist the result bucket and its bucket influencers + * Buckets are persisted with a consistent ID + * + * @param bucket The bucket to persist + * @return this + */ + public Builder persistBucket(Bucket bucket) { + // If the supplied bucket has records then create a copy with records + // removed, because we never persist nested records in buckets + Bucket bucketWithoutRecords = bucket; + if (!bucketWithoutRecords.getRecords().isEmpty()) { + bucketWithoutRecords = new Bucket(bucket); + bucketWithoutRecords.setRecords(Collections.emptyList()); + } + try { + XContentBuilder content = toXContentBuilder(bucketWithoutRecords); + logger.trace("[{}] ES API CALL: index result type {} to index {} at epoch {}", + jobId, Bucket.RESULT_TYPE_VALUE, indexName, bucketWithoutRecords.getEpoch()); + + bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), + bucketWithoutRecords.getId()).source(content)); + + persistBucketInfluencersStandalone(jobId, bucketWithoutRecords.getBucketInfluencers()); + } catch (IOException e) { + logger.error(new ParameterizedMessage("[{}] Error serialising bucket", new Object[] {jobId}), e); + } + + return this; + } + + private void persistBucketInfluencersStandalone(String jobId, List bucketInfluencers) + throws IOException { + if (bucketInfluencers != null && bucketInfluencers.isEmpty() == false) { + for (BucketInfluencer bucketInfluencer : bucketInfluencers) { + XContentBuilder content = serialiseBucketInfluencerStandalone(bucketInfluencer); + // Need consistent IDs to ensure overwriting on renormalization + String id = bucketInfluencer.getId(); + logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", + jobId, BucketInfluencer.RESULT_TYPE_VALUE, indexName, id); + bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), id).source(content)); + } + } + } + + /** + * Persist a list of anomaly records + * + * @param records the records to persist + * @return this + */ + public Builder persistRecords(List records) { + + try { + for (AnomalyRecord record : records) { + XContentBuilder content = toXContentBuilder(record); + logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", + jobId, AnomalyRecord.RESULT_TYPE_VALUE, indexName, record.getId()); + bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), record.getId()).source(content)); + } + } catch (IOException e) { + logger.error(new ParameterizedMessage("[{}] Error serialising records", new Object [] {jobId}), e); + } + + return this; + } + + /** + * Persist a list of influencers optionally using each influencer's ID or + * an auto generated ID + * + * @param influencers the influencers to persist + * @return this + */ + public Builder persistInfluencers(List influencers) { + try { + for (Influencer influencer : influencers) { + XContentBuilder content = toXContentBuilder(influencer); + logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", + jobId, Influencer.RESULT_TYPE_VALUE, indexName, influencer.getId()); + bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), influencer.getId()).source(content)); + } + } catch (IOException e) { + logger.error(new ParameterizedMessage("[{}] Error serialising influencers", new Object[] {jobId}), e); + } + + return this; + } + + /** + * Persist {@link PerPartitionMaxProbabilities} + * + * @param partitionProbabilities The probabilities to persist + * @return this + */ + public Builder persistPerPartitionMaxProbabilities(PerPartitionMaxProbabilities partitionProbabilities) { + try { + XContentBuilder builder = toXContentBuilder(partitionProbabilities); + logger.trace("[{}] ES API CALL: index result type {} to index {} at timestamp {} with ID {}", + jobId, PerPartitionMaxProbabilities.RESULT_TYPE_VALUE, indexName, partitionProbabilities.getTimestamp(), + partitionProbabilities.getId()); + bulkRequest.add( + new IndexRequest(indexName, Result.TYPE.getPreferredName(), partitionProbabilities.getId()).source(builder)); + } catch (IOException e) { + logger.error(new ParameterizedMessage("[{}] error serialising bucket per partition max normalized scores", + new Object[]{jobId}), e); + } + + return this; + } + + /** + * Execute the bulk action + */ + public void executeRequest() { + if (bulkRequest.numberOfActions() == 0) { + return; + } + logger.trace("[{}] ES API CALL: bulk request with {} actions", jobId, bulkRequest.numberOfActions()); + + BulkResponse addRecordsResponse = client.bulk(bulkRequest).actionGet(); + if (addRecordsResponse.hasFailures()) { + logger.error("[{}] Bulk index of results has errors: {}", jobId, addRecordsResponse.buildFailureMessage()); + } + } + } + + /** + * Persist the category definition + * + * @param category The category to be persisted + */ + public void persistCategoryDefinition(CategoryDefinition category) { + Persistable persistable = new Persistable(category.getJobId(), category, CategoryDefinition.TYPE.getPreferredName(), + CategoryDefinition.documentId(category.getJobId(), Long.toString(category.getCategoryId()))); + persistable.persist(AnomalyDetectorsIndex.jobResultsIndexName(category.getJobId())); + // Don't commit as we expect masses of these updates and they're not + // read again by this process + } + + /** + * Persist the quantiles + */ + public void persistQuantiles(Quantiles quantiles) { + Persistable persistable = new Persistable(quantiles.getJobId(), quantiles, Quantiles.TYPE.getPreferredName(), + Quantiles.documentId(quantiles.getJobId())); + if (persistable.persist(AnomalyDetectorsIndex.jobStateIndexName())) { + // Refresh the index when persisting quantiles so that previously + // persisted results will be available for searching. Do this using the + // indices API rather than the index API (used to write the quantiles + // above), because this will refresh all shards rather than just the + // shard that the quantiles document itself was written to. + commitStateWrites(quantiles.getJobId()); + } + } + + /** + * Persist a model snapshot description + */ + public void persistModelSnapshot(ModelSnapshot modelSnapshot) { + Persistable persistable = new Persistable(modelSnapshot.getJobId(), modelSnapshot, ModelSnapshot.TYPE.getPreferredName(), + modelSnapshot.documentId()); + persistable.persist(AnomalyDetectorsIndex.jobResultsIndexName(modelSnapshot.getJobId())); + } + + public void updateModelSnapshot(ModelSnapshot modelSnapshot, Consumer handler, Consumer errorHandler) { + String index = AnomalyDetectorsIndex.jobResultsIndexName(modelSnapshot.getJobId()); + IndexRequest indexRequest = new IndexRequest(index, ModelSnapshot.TYPE.getPreferredName(), modelSnapshot.documentId()); + try { + indexRequest.source(toXContentBuilder(modelSnapshot)); + } catch (IOException e) { + errorHandler.accept(e); + } + client.index(indexRequest, ActionListener.wrap(r -> handler.accept(true), errorHandler)); + } + + /** + * Persist the memory usage data + */ + public void persistModelSizeStats(ModelSizeStats modelSizeStats) { + String jobId = modelSizeStats.getJobId(); + logger.trace("[{}] Persisting model size stats, for size {}", jobId, modelSizeStats.getModelBytes()); + Persistable persistable = new Persistable(modelSizeStats.getJobId(), modelSizeStats, Result.TYPE.getPreferredName(), + ModelSizeStats.documentId(jobId)); + persistable.persist(AnomalyDetectorsIndex.jobResultsIndexName(jobId)); + persistable = new Persistable(modelSizeStats.getJobId(), modelSizeStats, Result.TYPE.getPreferredName(), null); + persistable.persist(AnomalyDetectorsIndex.jobResultsIndexName(jobId)); + // Don't commit as we expect masses of these updates and they're only + // for information at the API level + } + + /** + * Persist model debug output + */ + public void persistModelDebugOutput(ModelDebugOutput modelDebugOutput) { + Persistable persistable = new Persistable(modelDebugOutput.getJobId(), modelDebugOutput, Result.TYPE.getPreferredName(), null); + persistable.persist(AnomalyDetectorsIndex.jobResultsIndexName(modelDebugOutput.getJobId())); + // Don't commit as we expect masses of these updates and they're not + // read again by this process + } + + /** + * Delete any existing interim results synchronously + */ + public void deleteInterimResults(String jobId) { + JobDataDeleter deleter = new JobDataDeleter(client, jobId, true); + deleter.deleteInterimResults(); + deleter.commit(); + } + + /** + * Once all the job data has been written this function will be + * called to commit the writes to the datastore. + * + * @param jobId The job Id + * @return True if successful + */ + public boolean commitResultWrites(String jobId) { + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + // Refresh should wait for Lucene to make the data searchable + logger.trace("[{}] ES API CALL: refresh index {}", jobId, indexName); + client.admin().indices().refresh(new RefreshRequest(indexName)).actionGet(); + return true; + } + + /** + * Once the job state has been written calling this function makes it + * immediately searchable. + * + * @param jobId The job Id + * @return True if successful + * */ + public boolean commitStateWrites(String jobId) { + String indexName = AnomalyDetectorsIndex.jobStateIndexName(); + // Refresh should wait for Lucene to make the data searchable + logger.trace("[{}] ES API CALL: refresh index {}", jobId, indexName); + RefreshRequest refreshRequest = new RefreshRequest(indexName); + client.admin().indices().refresh(refreshRequest).actionGet(); + return true; + } + + XContentBuilder toXContentBuilder(ToXContent obj) throws IOException { + XContentBuilder builder = jsonBuilder(); + obj.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + private XContentBuilder serialiseBucketInfluencerStandalone(BucketInfluencer bucketInfluencer) throws IOException { + XContentBuilder builder = jsonBuilder(); + bucketInfluencer.toXContent(builder, ToXContent.EMPTY_PARAMS); + return builder; + } + + private class Persistable { + + private final String jobId; + private final ToXContent object; + private final String type; + private final String id; + + Persistable(String jobId, ToXContent object, String type, String id) { + this.jobId = jobId; + this.object = object; + this.type = type; + // TODO: (norelease): Fix the assertion tripping in internal engine for index requests without an id being retried: + this.id = id != null ? id : UUIDs.base64UUID(); + } + + boolean persist(String indexName) { + if (object == null) { + logger.warn("[{}] No {} to persist for job ", jobId, type); + return false; + } + + logCall(indexName); + + try { + IndexRequest indexRequest = new IndexRequest(indexName, type, id) + .source(toXContentBuilder(object)); + client.index(indexRequest).actionGet(); + return true; + } catch (IOException e) { + logger.error(new ParameterizedMessage("[{}] Error writing {}", new Object[]{jobId, type}), e); + return false; + } + } + + private void logCall(String indexName) { + if (id != null) { + logger.trace("[{}] ES API CALL: index type {} to index {} with ID {}", jobId, type, indexName, id); + } else { + logger.trace("[{}] ES API CALL: index type {} to index {} with auto-generated ID", jobId, type, indexName); + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java new file mode 100644 index 00000000000..f0307348286 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobStorageDeletionTask.java @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.action.bulk.byscroll.DeleteByQueryRequest; +import org.elasticsearch.action.bulk.byscroll.BulkByScrollResponse; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.xpack.ml.action.MlDeleteByQueryAction; + + +import java.util.function.Consumer; + +public class JobStorageDeletionTask extends Task { + private final Logger logger; + + public JobStorageDeletionTask(long id, String type, String action, String description, TaskId parentTask) { + super(id, type, action, description, parentTask); + this.logger = Loggers.getLogger(getClass()); + } + + public void delete(String jobId, String indexName, Client client, + CheckedConsumer finishedHandler, + Consumer failureHandler) { + + // Step 2. Regardless of if the DBQ succeeds, we delete the physical index + // ------- + CheckedConsumer dbqHandler = bulkByScrollResponse -> { + if (bulkByScrollResponse.isTimedOut()) { + logger.warn("DeleteByQuery for index [" + indexName + "] timed out. Continuing to delete index."); + } + if (!bulkByScrollResponse.getBulkFailures().isEmpty()) { + logger.warn("[" + bulkByScrollResponse.getBulkFailures().size() + + "] failures encountered while running DeleteByQuery on index [" + indexName + "]. " + + "Continuing to delete index"); + } + + DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); + client.admin().indices().delete(deleteIndexRequest, ActionListener.wrap(deleteIndexResponse -> { + logger.info("Deleting index [" + indexName + "] successful"); + + if (deleteIndexResponse.isAcknowledged()) { + logger.info("Index deletion acknowledged"); + } else { + logger.warn("Index deletion not acknowledged"); + } + finishedHandler.accept(deleteIndexResponse.isAcknowledged()); + }, missingIndexHandler(indexName, finishedHandler, failureHandler))); + }; + + // Step 1. DeleteByQuery on the index, matching all docs with the right job_id + // ------- + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.source(new SearchSourceBuilder().query(new TermQueryBuilder("job_id", jobId))); + DeleteByQueryRequest request = new DeleteByQueryRequest(searchRequest); + request.setSlices(5); + + client.execute(MlDeleteByQueryAction.INSTANCE, request, + ActionListener.wrap(dbqHandler, missingIndexHandler(indexName, finishedHandler, failureHandler))); + } + + // If the index doesn't exist, we need to catch the exception and carry onwards so that the cluster + // state is properly updated + private Consumer missingIndexHandler(String indexName, CheckedConsumer finishedHandler, + Consumer failureHandler) { + return e -> { + if (e instanceof IndexNotFoundException) { + logger.warn("Physical index [" + indexName + "] not found. Continuing to delete job."); + try { + finishedHandler.accept(false); + } catch (Exception e1) { + failureHandler.accept(e1); + } + } else { + // all other exceptions should die + failureHandler.accept(e); + } + }; + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java new file mode 100644 index 00000000000..fe01627066d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +/** + * One time query builder for records. Sets default values for the following + * parameters: + *
    + *
  • From- Skip the first N records. This parameter is for paging if not + * required set to 0. Default = 0
    • + *
  • Size- Take only this number of records. Default = + * {@value DEFAULT_SIZE}
    • + *
  • IncludeInterim- Include interim results. Default = false
    • + *
  • SortField- The field to sort results by if null no sort is + * applied. Default = null
    • + *
  • SortDescending- Sort in descending order. Default = true
    • + *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= + * this value. Default = 0.0
    • + *
  • normalizedProbabilityThreshold. Return only buckets with a + * maxNormalizedProbability >= this value. Default = 0.0
    • + *
  • start- The start bucket time. A bucket with this timestamp will be + * included in the results. If 0 all buckets up to endEpochMs are + * returned. Default = -1
    • + *
  • end- The end bucket timestamp buckets up to but NOT including this + * timestamp are returned. If 0 all buckets from startEpochMs are + * returned. Default = -1
    • + *
+ */ +public final class RecordsQueryBuilder { + + public static final int DEFAULT_SIZE = 100; + + private RecordsQuery recordsQuery = new RecordsQuery(); + + public RecordsQueryBuilder from(int from) { + recordsQuery.from = from; + return this; + } + + public RecordsQueryBuilder size(int size) { + recordsQuery.size = size; + return this; + } + + public RecordsQueryBuilder epochStart(String startTime) { + recordsQuery.start = startTime; + return this; + } + + public RecordsQueryBuilder epochEnd(String endTime) { + recordsQuery.end = endTime; + return this; + } + + public RecordsQueryBuilder includeInterim(boolean include) { + recordsQuery.includeInterim = include; + return this; + } + + public RecordsQueryBuilder sortField(String fieldname) { + recordsQuery.sortField = fieldname; + return this; + } + + public RecordsQueryBuilder sortDescending(boolean sortDescending) { + recordsQuery.sortDescending = sortDescending; + return this; + } + + public RecordsQueryBuilder anomalyScoreThreshold(double anomalyScoreFilter) { + recordsQuery.anomalyScoreFilter = anomalyScoreFilter; + return this; + } + + public RecordsQueryBuilder normalizedProbability(double normalizedProbability) { + recordsQuery.normalizedProbability = normalizedProbability; + return this; + } + + public RecordsQueryBuilder partitionFieldValue(String partitionFieldValue) { + recordsQuery.partitionFieldValue = partitionFieldValue; + return this; + } + + public RecordsQuery build() { + return recordsQuery; + } + + public void clear() { + recordsQuery = new RecordsQuery(); + } + + public class RecordsQuery { + + private int from = 0; + private int size = DEFAULT_SIZE; + private boolean includeInterim = false; + private String sortField; + private boolean sortDescending = true; + private double anomalyScoreFilter = 0.0d; + private double normalizedProbability = 0.0d; + private String partitionFieldValue; + private String start; + private String end; + + + public int getSize() { + return size; + } + + public boolean isIncludeInterim() { + return includeInterim; + } + + public String getSortField() { + return sortField; + } + + public boolean isSortDescending() { + return sortDescending; + } + + public double getAnomalyScoreThreshold() { + return anomalyScoreFilter; + } + + public double getNormalizedProbabilityThreshold() { + return normalizedProbability; + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public int getFrom() { + return from; + } + + public String getStart() { + return start; + } + + public String getEnd() { + return end; + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ResultsFilterBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ResultsFilterBuilder.java new file mode 100644 index 00000000000..c615ff7af7d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ResultsFilterBuilder.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.xpack.ml.job.results.Result; + +import java.util.ArrayList; +import java.util.List; + +/** + * This builder facilitates the creation of a {@link QueryBuilder} with common + * characteristics to both buckets and records. + */ +class ResultsFilterBuilder { + private final List queries; + + ResultsFilterBuilder() { + queries = new ArrayList<>(); + } + + ResultsFilterBuilder(QueryBuilder queryBuilder) { + this(); + queries.add(queryBuilder); + } + + ResultsFilterBuilder timeRange(String field, Object start, Object end) { + if (start != null || end != null) { + RangeQueryBuilder timeRange = QueryBuilders.rangeQuery(field); + if (start != null) { + timeRange.gte(start); + } + if (end != null) { + timeRange.lt(end); + } + addQuery(timeRange); + } + return this; + } + + ResultsFilterBuilder timeRange(String field, String timestamp) { + addQuery(QueryBuilders.matchQuery(field, timestamp)); + return this; + } + + ResultsFilterBuilder score(String fieldName, double threshold) { + if (threshold > 0.0) { + RangeQueryBuilder scoreFilter = QueryBuilders.rangeQuery(fieldName); + scoreFilter.gte(threshold); + addQuery(scoreFilter); + } + return this; + } + + public ResultsFilterBuilder interim(String fieldName, boolean includeInterim) { + if (includeInterim) { + // Including interim results does not stop final results being + // shown, so including interim results means no filtering on the + // isInterim field + return this; + } + + // Implemented as "NOT isInterim == true" so that not present and null + // are equivalent to false. This improves backwards compatibility. + // Also, note how for a boolean field, unlike numeric term queries, the + // term value is supplied as a string. + TermQueryBuilder interimFilter = QueryBuilders.termQuery(fieldName, + Boolean.TRUE.toString()); + QueryBuilder notInterimFilter = QueryBuilders.boolQuery().mustNot(interimFilter); + addQuery(notInterimFilter); + return this; + } + + ResultsFilterBuilder term(String fieldName, String fieldValue) { + if (Strings.isNullOrEmpty(fieldName) || Strings.isNullOrEmpty(fieldValue)) { + return this; + } + + TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(fieldName, fieldValue); + addQuery(termQueryBuilder); + return this; + } + + ResultsFilterBuilder resultType(String resultType) { + return term(Result.RESULT_TYPE.getPreferredName(), resultType); + } + + private void addQuery(QueryBuilder fb) { + queries.add(fb); + } + + public QueryBuilder build() { + if (queries.isEmpty()) { + return QueryBuilders.matchAllQuery(); + } + if (queries.size() == 1) { + return queries.get(0); + } + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (QueryBuilder query : queries) { + boolQueryBuilder.filter(query); + } + return boolQueryBuilder; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/CountingInputStream.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/CountingInputStream.java new file mode 100644 index 00000000000..559347d1fa3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/CountingInputStream.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * Simple wrapper around an inputstream instance that counts + * all the bytes passing through it reporting that number to + * the {@link DataCountsReporter} + *

+ * Overrides the read methods counting the number of bytes read. + */ +public class CountingInputStream extends FilterInputStream { + private DataCountsReporter dataCountsReporter; + + /** + * @param in + * input stream + * @param dataCountsReporter + * Write number of records, bytes etc. + */ + public CountingInputStream(InputStream in, DataCountsReporter dataCountsReporter) { + super(in); + this.dataCountsReporter = dataCountsReporter; + } + + /** + * Report 1 byte read + */ + @Override + public int read() throws IOException { + int read = in.read(); + dataCountsReporter.reportBytesRead(read < 0 ? 0 : 1); + + return read; + } + + @Override + public int read(byte[] b) throws IOException { + int read = in.read(b); + + dataCountsReporter.reportBytesRead(read < 0 ? 0 : read); + + return read; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int read = in.read(b, off, len); + + dataCountsReporter.reportBytesRead(read < 0 ? 0 : read); + return read; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java new file mode 100644 index 00000000000..24c8453032a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/DataCountsReporter.java @@ -0,0 +1,361 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; + +import java.io.Closeable; +import java.util.Date; +import java.util.Locale; +import java.util.function.Function; + + +/** + * Status reporter for tracking counts of the good/bad records written to the API. + * Call one of the reportXXX() methods to update the records counts. + * + * Stats are logged at specific stages + *

    + *
  1. Every 100 records for the first 1000 records
    2. + *
  2. Every 1000 records for the first 20000 records
    3. + *
  3. Every 10000 records after 20000 records
    4. + *
+ * The {@link #reportingBoundaryFunction} member points to a different + * function depending on which reporting stage is the current, the function + * changes when each of the reporting stages are passed. If the + * function returns {@code true} the usage is logged. + * + * DataCounts are persisted periodically in a datafeed task via + * {@linkplain JobDataCountsPersister}, {@link #close()} must be called to + * cancel the datafeed task. + */ +public class DataCountsReporter extends AbstractComponent implements Closeable { + /** + * The max percentage of date parse errors allowed before + * an exception is thrown. + */ + public static final Setting ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING = Setting.intSetting("max.percent.date.errors", 25, + Property.NodeScope); + + /** + * The max percentage of out of order records allowed before + * an exception is thrown. + */ + public static final Setting ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING = Setting + .intSetting("max.percent.outoforder.errors", 25, Property.NodeScope); + + private static final TimeValue PERSIST_INTERVAL = TimeValue.timeValueMillis(10_000L); + + private final String jobId; + private final JobDataCountsPersister dataCountsPersister; + + private final DataCounts totalRecordStats; + private volatile DataCounts incrementalRecordStats; + + private long analyzedFieldsPerRecord = 1; + + private long lastRecordCountQuotient = 0; + private long logEvery = 1; + private long logCount = 0; + + private final int acceptablePercentDateParseErrors; + private final int acceptablePercentOutOfOrderErrors; + + private Function reportingBoundaryFunction; + + private volatile boolean persistDataCountsOnNextRecord; + private final ThreadPool.Cancellable persistDataCountsDatafeedAction; + + public DataCountsReporter(ThreadPool threadPool, Settings settings, String jobId, DataCounts counts, + JobDataCountsPersister dataCountsPersister) { + + super(settings); + + this.jobId = jobId; + this.dataCountsPersister = dataCountsPersister; + + totalRecordStats = counts; + incrementalRecordStats = new DataCounts(jobId); + + acceptablePercentDateParseErrors = ACCEPTABLE_PERCENTAGE_DATE_PARSE_ERRORS_SETTING.get(settings); + acceptablePercentOutOfOrderErrors = ACCEPTABLE_PERCENTAGE_OUT_OF_ORDER_ERRORS_SETTING.get(settings); + + reportingBoundaryFunction = this::reportEvery100Records; + + persistDataCountsDatafeedAction = threadPool.scheduleWithFixedDelay(() -> persistDataCountsOnNextRecord = true, + PERSIST_INTERVAL, ThreadPool.Names.GENERIC); + } + + /** + * Increment the number of records written by 1 and increment + * the total number of fields read. + * + * @param inputFieldCount Number of fields in the record. + * Note this is not the number of processed fields (by field etc) + * but the actual number of fields in the record + * @param recordTimeMs The time of the latest record written + * in milliseconds from the epoch. + */ + public void reportRecordWritten(long inputFieldCount, long recordTimeMs) { + Date recordDate = new Date(recordTimeMs); + + totalRecordStats.incrementInputFieldCount(inputFieldCount); + totalRecordStats.incrementProcessedRecordCount(1); + totalRecordStats.setLatestRecordTimeStamp(recordDate); + + incrementalRecordStats.incrementInputFieldCount(inputFieldCount); + incrementalRecordStats.incrementProcessedRecordCount(1); + incrementalRecordStats.setLatestRecordTimeStamp(recordDate); + + boolean isFirstReport = totalRecordStats.getEarliestRecordTimeStamp() == null; + if (isFirstReport) { + totalRecordStats.setEarliestRecordTimeStamp(recordDate); + incrementalRecordStats.setEarliestRecordTimeStamp(recordDate); + } + + // report at various boundaries + long totalRecords = getInputRecordCount(); + if (reportingBoundaryFunction.apply(totalRecords)) { + logStatus(totalRecords); + } + + if (persistDataCountsOnNextRecord) { + DataCounts copy = new DataCounts(runningTotalStats()); + dataCountsPersister.persistDataCounts(jobId, copy, new LoggingActionListener()); + persistDataCountsOnNextRecord = false; + } + } + + /** + * Update only the incremental stats with the newest record time + * + * @param latestRecordTimeMs latest record time as epoch millis + */ + public void reportLatestTimeIncrementalStats(long latestRecordTimeMs) { + incrementalRecordStats.setLatestRecordTimeStamp(new Date(latestRecordTimeMs)); + } + + /** + * Increments the date parse error count + */ + public void reportDateParseError(long inputFieldCount) { + totalRecordStats.incrementInvalidDateCount(1); + totalRecordStats.incrementInputFieldCount(inputFieldCount); + + incrementalRecordStats.incrementInvalidDateCount(1); + incrementalRecordStats.incrementInputFieldCount(inputFieldCount); + } + + /** + * Increments the missing field count + * Records with missing fields are still processed + */ + public void reportMissingField() { + totalRecordStats.incrementMissingFieldCount(1); + incrementalRecordStats.incrementMissingFieldCount(1); + } + + public void reportMissingFields(long missingCount) { + totalRecordStats.incrementMissingFieldCount(missingCount); + incrementalRecordStats.incrementMissingFieldCount(missingCount); + } + + /** + * Add newBytes to the total volume processed + */ + public void reportBytesRead(long newBytes) { + totalRecordStats.incrementInputBytes(newBytes); + incrementalRecordStats.incrementInputBytes(newBytes); + } + + /** + * Increments the out of order record count + */ + public void reportOutOfOrderRecord(long inputFieldCount) { + totalRecordStats.incrementOutOfOrderTimeStampCount(1); + totalRecordStats.incrementInputFieldCount(inputFieldCount); + + incrementalRecordStats.incrementOutOfOrderTimeStampCount(1); + incrementalRecordStats.incrementInputFieldCount(inputFieldCount); + } + + /** + * Total records seen = records written to the Engine (processed record + * count) + date parse error records count + out of order record count. + *

+ * Records with missing fields are counted as they are still written. + */ + public long getInputRecordCount() { + return totalRecordStats.getInputRecordCount(); + } + + public long getProcessedRecordCount() { + return totalRecordStats.getProcessedRecordCount(); + } + + public long getDateParseErrorsCount() { + return totalRecordStats.getInvalidDateCount(); + } + + public long getMissingFieldErrorCount() { + return totalRecordStats.getMissingFieldCount(); + } + + public long getOutOfOrderRecordCount() { + return totalRecordStats.getOutOfOrderTimeStampCount(); + } + + public long getBytesRead() { + return totalRecordStats.getInputBytes(); + } + + public Date getLatestRecordTime() { + return totalRecordStats.getLatestRecordTimeStamp(); + } + + public long getProcessedFieldCount() { + totalRecordStats.calcProcessedFieldCount(getAnalysedFieldsPerRecord()); + return totalRecordStats.getProcessedFieldCount(); + } + + public long getInputFieldCount() { + return totalRecordStats.getInputFieldCount(); + } + + public int getAcceptablePercentDateParseErrors() { + return acceptablePercentDateParseErrors; + } + + public int getAcceptablePercentOutOfOrderErrors() { + return acceptablePercentOutOfOrderErrors; + } + + public void setAnalysedFieldsPerRecord(long value) { + analyzedFieldsPerRecord = value; + } + + public long getAnalysedFieldsPerRecord() { + return analyzedFieldsPerRecord; + } + + + /** + * Report the counts now regardless of whether or not we are at a reporting boundary. + */ + public void finishReporting() { + dataCountsPersister.persistDataCounts(jobId, runningTotalStats(), new LoggingActionListener()); + } + + /** + * Log the status. This is done progressively less frequently as the job + * processes more data. Logging every 10000 records when the data rate is + * 40000 per second quickly rolls the logs. + */ + protected void logStatus(long totalRecords) { + if (++logCount % logEvery != 0) { + return; + } + + String status = String.format(Locale.ROOT, + "[%s] %d records written to autodetect; missingFieldCount=%d, invalidDateCount=%d, outOfOrderCount=%d", jobId, + getProcessedRecordCount(), getMissingFieldErrorCount(), getDateParseErrorsCount(), getOutOfOrderRecordCount()); + + logger.info(status); + + int log10TotalRecords = (int) Math.floor(Math.log10(totalRecords)); + // Start reducing the logging rate after 10 million records have been seen + if (log10TotalRecords > 6) { + logEvery = (int) Math.pow(10.0, log10TotalRecords - 6); + logCount = 0; + } + } + + private boolean reportEvery100Records(long totalRecords) { + if (totalRecords > 1000) { + lastRecordCountQuotient = totalRecords / 1000; + reportingBoundaryFunction = this::reportEvery1000Records; + return false; + } + + long quotient = totalRecords / 100; + if (quotient > lastRecordCountQuotient) { + lastRecordCountQuotient = quotient; + return true; + } + + return false; + } + + private boolean reportEvery1000Records(long totalRecords) { + + if (totalRecords > 20000) { + lastRecordCountQuotient = totalRecords / 10000; + reportingBoundaryFunction = this::reportEvery10000Records; + return false; + } + + long quotient = totalRecords / 1000; + if (quotient > lastRecordCountQuotient) { + lastRecordCountQuotient = quotient; + return true; + } + + return false; + } + + private boolean reportEvery10000Records(long totalRecords) { + long quotient = totalRecords / 10000; + if (quotient > lastRecordCountQuotient) { + lastRecordCountQuotient = quotient; + return true; + } + + return false; + } + + public void startNewIncrementalCount() { + incrementalRecordStats = new DataCounts(jobId); + } + + public DataCounts incrementalStats() { + incrementalRecordStats.calcProcessedFieldCount(getAnalysedFieldsPerRecord()); + return incrementalRecordStats; + } + + public synchronized DataCounts runningTotalStats() { + totalRecordStats.calcProcessedFieldCount(getAnalysedFieldsPerRecord()); + return totalRecordStats; + } + + @Override + public void close() { + persistDataCountsDatafeedAction.cancel(); + } + + /** + * Log success/error + */ + private class LoggingActionListener implements ActionListener { + @Override + public void onResponse(Boolean aBoolean) { + logger.trace("[{}] Persisted DataCounts", jobId); + } + + @Override + public void onFailure(Exception e) { + logger.debug(new ParameterizedMessage("[{}] Error persisting DataCounts stats", jobId), e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java new file mode 100644 index 00000000000..62342124526 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeController.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.ml.job.process.logging.CppLogMessageHandler; +import org.elasticsearch.xpack.ml.utils.NamedPipeHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.TimeoutException; + + +/** + * Maintains the connection to the native controller daemon that can start other processes. + */ +public class NativeController { + private static final Logger LOGGER = Loggers.getLogger(NativeController.class); + + // The controller process should already be running by the time this class tries to connect to it, so the timeout can be short + private static final Duration CONTROLLER_CONNECT_TIMEOUT = Duration.ofSeconds(2); + + private static final String START_COMMAND = "start"; + + private final CppLogMessageHandler cppLogHandler; + private final OutputStream commandStream; + private Thread logTailThread; + + public NativeController(Environment env, NamedPipeHelper namedPipeHelper) throws IOException { + ProcessPipes processPipes = new ProcessPipes(env, namedPipeHelper, ProcessCtrl.CONTROLLER, null, + true, true, false, false, false, false); + processPipes.connectStreams(CONTROLLER_CONNECT_TIMEOUT); + cppLogHandler = new CppLogMessageHandler(null, processPipes.getLogStream().get()); + commandStream = processPipes.getCommandStream().get(); + } + + public void tailLogsInThread() { + logTailThread = new Thread(() -> { + try { + cppLogHandler.tailStream(); + cppLogHandler.close(); + } catch (IOException e) { + LOGGER.error("Error tailing C++ controller logs", e); + } + LOGGER.info("Native controller process has stopped - no new native processes can be started"); + }); + logTailThread.start(); + } + + public long getPid() throws TimeoutException { + return cppLogHandler.getPid(CONTROLLER_CONNECT_TIMEOUT); + } + + public void startProcess(List command) throws IOException { + // Sanity check to avoid hard-to-debug errors - tabs and newlines will confuse the controller process + for (String arg : command) { + if (arg.contains("\t")) { + throw new IllegalArgumentException("argument contains a tab character: " + arg + " in " + command); + } + if (arg.contains("\n")) { + throw new IllegalArgumentException("argument contains a newline character: " + arg + " in " + command); + } + } + + synchronized (commandStream) { + LOGGER.debug("Starting process with command: " + command); + commandStream.write(START_COMMAND.getBytes(StandardCharsets.UTF_8)); + for (String arg : command) { + commandStream.write('\t'); + commandStream.write(arg.getBytes(StandardCharsets.UTF_8)); + } + commandStream.write('\n'); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java new file mode 100644 index 00000000000..be6a279085f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessCtrl.java @@ -0,0 +1,316 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.IgnoreDowntime; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + + +/** + * Utility class for running a Ml process
+ * The process runs in a clean environment. + */ +public class ProcessCtrl { + + /** + * Autodetect API native program name - always loaded from the same directory as the controller process + */ + public static final String AUTODETECT = "autodetect"; + static final String AUTODETECT_PATH = "./" + AUTODETECT; + + /** + * The normalization native program name - always loaded from the same directory as the controller process + */ + public static final String NORMALIZE = "normalize"; + static final String NORMALIZE_PATH = "./" + NORMALIZE; + + /** + * Process controller native program name + */ + public static final String CONTROLLER = "controller"; + + /** + * Name of the config setting containing the path to the logs directory + */ + private static final int DEFAULT_MAX_NUM_RECORDS = 500; + /** + * The maximum number of anomaly records that will be written each bucket + */ + public static final Setting MAX_ANOMALY_RECORDS_SETTING = Setting.intSetting("max.anomaly.records", DEFAULT_MAX_NUM_RECORDS, + Property.NodeScope); + + /** + * This must match the value defined in CLicenseValidator::validate() in the C++ code + */ + static final long VALIDATION_NUMBER = 926213; + + /* + * General arguments + */ + static final String JOB_ID_ARG = "--jobid="; + static final String LICENSE_VALIDATION_ARG = "--licenseValidation="; + + /* + * Arguments used by both autodetect and normalize + */ + static final String BUCKET_SPAN_ARG = "--bucketspan="; + public static final String DELETE_STATE_FILES_ARG = "--deleteStateFiles"; + static final String IGNORE_DOWNTIME_ARG = "--ignoreDowntime"; + static final String LENGTH_ENCODED_INPUT_ARG = "--lengthEncodedInput"; + static final String MODEL_CONFIG_ARG = "--modelconfig="; + public static final String QUANTILES_STATE_PATH_ARG = "--quantilesState="; + static final String MULTIPLE_BUCKET_SPANS_ARG = "--multipleBucketspans="; + static final String PER_PARTITION_NORMALIZATION = "--perPartitionNormalization"; + + /* + * Arguments used by autodetect + */ + static final String BATCH_SPAN_ARG = "--batchspan="; + static final String LATENCY_ARG = "--latency="; + static final String RESULT_FINALIZATION_WINDOW_ARG = "--resultFinalizationWindow="; + static final String MULTIVARIATE_BY_FIELDS_ARG = "--multivariateByFields"; + static final String PERIOD_ARG = "--period="; + static final String PERSIST_INTERVAL_ARG = "--persistInterval="; + static final String MAX_QUANTILE_INTERVAL_ARG = "--maxQuantileInterval="; + static final String SUMMARY_COUNT_FIELD_ARG = "--summarycountfield="; + static final String TIME_FIELD_ARG = "--timefield="; + + private static final int SECONDS_IN_HOUR = 3600; + + /** + * Roughly how often should the C++ process persist state? A staggering + * factor that varies by job is added to this. + */ + static final long DEFAULT_BASE_PERSIST_INTERVAL = 10800; // 3 hours + + /** + * Roughly how often should the C++ process output quantiles when no + * anomalies are being detected? A staggering factor that varies by job is + * added to this. + */ + static final int BASE_MAX_QUANTILE_INTERVAL = 21600; // 6 hours + + /** + * Name of the model config file + */ + static final String ML_MODEL_CONF = "mlmodel.conf"; + + /** + * Persisted quantiles are written to disk so they can be read by + * the autodetect program. All quantiles files have this extension. + */ + private static final String QUANTILES_FILE_EXTENSION = ".json"; + + /** + * Config setting storing the flag that disables model persistence + */ + public static final Setting DONT_PERSIST_MODEL_STATE_SETTING = Setting.boolSetting("no.model.state.persist", false, + Property.NodeScope); + + static String maxAnomalyRecordsArg(Settings settings) { + return "--maxAnomalyRecords=" + MAX_ANOMALY_RECORDS_SETTING.get(settings); + } + + private ProcessCtrl() { + + } + + /** + * This random time of up to 1 hour is added to intervals at which we + * tell the C++ process to perform periodic operations. This means that + * when there are many jobs there is a certain amount of staggering of + * their periodic operations. A given job will always be given the same + * staggering interval (for a given JVM implementation). + * + * @param jobId The ID of the job to calculate the staggering interval for + * @return The staggering interval + */ + static int calculateStaggeringInterval(String jobId) { + Random rng = new Random(jobId.hashCode()); + return rng.nextInt(SECONDS_IN_HOUR); + } + + public static List buildAutodetectCommand(Environment env, Settings settings, Job job, Logger logger, boolean ignoreDowntime, + long controllerPid) { + List command = new ArrayList<>(); + command.add(AUTODETECT_PATH); + + String jobId = JOB_ID_ARG + job.getId(); + command.add(jobId); + + command.add(makeLicenseArg(controllerPid)); + + AnalysisConfig analysisConfig = job.getAnalysisConfig(); + if (analysisConfig != null) { + addIfNotNull(analysisConfig.getBucketSpan(), BUCKET_SPAN_ARG, command); + addIfNotNull(analysisConfig.getBatchSpan(), BATCH_SPAN_ARG, command); + addIfNotNull(analysisConfig.getLatency(), LATENCY_ARG, command); + addIfNotNull(analysisConfig.getPeriod(), PERIOD_ARG, command); + addIfNotNull(analysisConfig.getSummaryCountFieldName(), + SUMMARY_COUNT_FIELD_ARG, command); + addIfNotNull(analysisConfig.getMultipleBucketSpans(), + MULTIPLE_BUCKET_SPANS_ARG, command); + if (Boolean.TRUE.equals(analysisConfig.getOverlappingBuckets())) { + Long window = analysisConfig.getResultFinalizationWindow(); + if (window == null) { + window = AnalysisConfig.DEFAULT_RESULT_FINALIZATION_WINDOW; + } + command.add(RESULT_FINALIZATION_WINDOW_ARG + window); + } + if (Boolean.TRUE.equals(analysisConfig.getMultivariateByFields())) { + command.add(MULTIVARIATE_BY_FIELDS_ARG); + } + + if (analysisConfig.getUsePerPartitionNormalization()) { + command.add(PER_PARTITION_NORMALIZATION); + } + } + + // Input is always length encoded + command.add(LENGTH_ENCODED_INPUT_ARG); + + // Limit the number of output records + command.add(maxAnomalyRecordsArg(settings)); + + // always set the time field + String timeFieldArg = TIME_FIELD_ARG + getTimeFieldOrDefault(job); + command.add(timeFieldArg); + + int intervalStagger = calculateStaggeringInterval(job.getId()); + logger.debug("Periodic operations staggered by " + intervalStagger +" seconds for job '" + job.getId() + "'"); + + // Supply a URL for persisting/restoring model state unless model + // persistence has been explicitly disabled. + if (DONT_PERSIST_MODEL_STATE_SETTING.get(settings)) { + logger.info("Will not persist model state - " + DONT_PERSIST_MODEL_STATE_SETTING + " setting was set"); + } else { + // Persist model state every few hours even if the job isn't closed + long persistInterval = (job.getBackgroundPersistInterval() == null) ? + (DEFAULT_BASE_PERSIST_INTERVAL + intervalStagger) : + job.getBackgroundPersistInterval(); + command.add(PERSIST_INTERVAL_ARG + persistInterval); + } + + int maxQuantileInterval = BASE_MAX_QUANTILE_INTERVAL + intervalStagger; + command.add(MAX_QUANTILE_INTERVAL_ARG + maxQuantileInterval); + + ignoreDowntime = ignoreDowntime + || job.getIgnoreDowntime() == IgnoreDowntime.ONCE + || job.getIgnoreDowntime() == IgnoreDowntime.ALWAYS; + + if (ignoreDowntime) { + command.add(IGNORE_DOWNTIME_ARG); + } + + if (modelConfigFilePresent(env)) { + String modelConfigFile = MlPlugin.resolveConfigFile(env, ML_MODEL_CONF).toString(); + command.add(MODEL_CONFIG_ARG + modelConfigFile); + } + + return command; + } + + private static String getTimeFieldOrDefault(Job job) { + DataDescription dataDescription = job.getDataDescription(); + boolean useDefault = dataDescription == null + || Strings.isNullOrEmpty(dataDescription.getTimeField()); + return useDefault ? DataDescription.DEFAULT_TIME_FIELD : dataDescription.getTimeField(); + } + + private static void addIfNotNull(T object, String argKey, List command) { + if (object != null) { + String param = argKey + object; + command.add(param); + } + } + + /** + * Return true if there is a file ES_HOME/config/mlmodel.conf + */ + public static boolean modelConfigFilePresent(Environment env) { + Path modelConfPath = MlPlugin.resolveConfigFile(env, ML_MODEL_CONF); + + return Files.isRegularFile(modelConfPath); + } + + /** + * Build the command to start the normalizer process. + */ + public static List buildNormalizerCommand(Environment env, String jobId, String quantilesState, Integer bucketSpan, + boolean perPartitionNormalization, long controllerPid) throws IOException { + + List command = new ArrayList<>(); + command.add(NORMALIZE_PATH); + addIfNotNull(bucketSpan, BUCKET_SPAN_ARG, command); + command.add(makeLicenseArg(controllerPid)); + command.add(LENGTH_ENCODED_INPUT_ARG); + if (perPartitionNormalization) { + command.add(PER_PARTITION_NORMALIZATION); + } + + if (quantilesState != null) { + Path quantilesStateFilePath = writeNormalizerInitState(jobId, quantilesState, env); + + String stateFileArg = QUANTILES_STATE_PATH_ARG + quantilesStateFilePath; + command.add(stateFileArg); + command.add(DELETE_STATE_FILES_ARG); + } + + if (modelConfigFilePresent(env)) { + Path modelConfPath = MlPlugin.resolveConfigFile(env, ML_MODEL_CONF); + command.add(MODEL_CONFIG_ARG + modelConfPath.toAbsolutePath().getFileName()); + } + + return command; + } + + /** + * Write the normalizer init state to file. + */ + public static Path writeNormalizerInitState(String jobId, String state, Environment env) + throws IOException { + // createTempFile has a race condition where it may return the same + // temporary file name to different threads if called simultaneously + // from multiple threads, hence add the thread ID to avoid this + Path stateFile = Files.createTempFile(env.tmpFile(), jobId + "_quantiles_" + Thread.currentThread().getId(), + QUANTILES_FILE_EXTENSION); + + try (BufferedWriter osw = Files.newBufferedWriter(stateFile, StandardCharsets.UTF_8);) { + osw.write(state); + } + + return stateFile; + } + + /** + * The number must be equal to the daemon controller's PID modulo a magic number. + */ + private static String makeLicenseArg(long controllerPid) { + // Get a random int rather than long so we don't overflow when multiplying by VALIDATION_NUMBER + long rand = Randomness.get().nextInt(); + long val = controllerPid + (((rand < 0) ? -rand : rand) + 1) * VALIDATION_NUMBER; + return LICENSE_VALIDATION_ARG + val; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessPipes.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessPipes.java new file mode 100644 index 00000000000..41a7df348b1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/ProcessPipes.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.xpack.ml.utils.NamedPipeHelper; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Duration; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +/** + * Utility class for telling a Ml C++ process which named pipes to use, + * and then waiting for them to connect once the C++ process is running. + */ +public class ProcessPipes { + + public static final String LOG_PIPE_ARG = "--logPipe="; + public static final String COMMAND_PIPE_ARG = "--commandPipe="; + public static final String INPUT_ARG = "--input="; + public static final String INPUT_IS_PIPE_ARG = "--inputIsPipe"; + public static final String OUTPUT_ARG = "--output="; + public static final String OUTPUT_IS_PIPE_ARG = "--outputIsPipe"; + public static final String RESTORE_ARG = "--restore="; + public static final String RESTORE_IS_PIPE_ARG = "--restoreIsPipe"; + public static final String PERSIST_ARG = "--persist="; + public static final String PERSIST_IS_PIPE_ARG = "--persistIsPipe"; + + private final NamedPipeHelper namedPipeHelper; + + /** + * null indicates a pipe won't be used + */ + private final String logPipeName; + private final String commandPipeName; + private final String processInPipeName; + private final String processOutPipeName; + private final String restorePipeName; + private final String persistPipeName; + + private InputStream logStream; + private OutputStream commandStream; + private OutputStream processInStream; + private InputStream processOutStream; + private OutputStream restoreStream; + private InputStream persistStream; + + /** + * Construct, stating which pipes are expected to be created. The corresponding C++ process creates the named pipes, so + * only one combination of wanted pipes will work with any given C++ process. The arguments to this constructor + * must be carefully chosen with reference to the corresponding C++ code. + * @param processName The name of the process that pipes are to be opened to. + * Must not be a full path, nor have the .exe extension on Windows. + * @param jobId The job ID of the process to which pipes are to be opened, if the process is associated with a specific job. + * May be null or empty for processes not associated with a specific job. + */ + public ProcessPipes(Environment env, NamedPipeHelper namedPipeHelper, String processName, String jobId, + boolean wantLogPipe, boolean wantCommandPipe, boolean wantProcessInPipe, boolean wantProcessOutPipe, + boolean wantRestorePipe, boolean wantPersistPipe) { + this.namedPipeHelper = namedPipeHelper; + + // The way the pipe names are formed MUST match what is done in the controller main() + // function, as it does not get any command line arguments when started as a daemon. If + // you change the code here then you MUST also change the C++ code in controller's + // main() function. + StringBuilder prefixBuilder = new StringBuilder(); + prefixBuilder.append(namedPipeHelper.getDefaultPipeDirectoryPrefix(env)).append(Objects.requireNonNull(processName)).append('_'); + if (!Strings.isNullOrEmpty(jobId)) { + prefixBuilder.append(jobId).append('_'); + } + String prefix = prefixBuilder.toString(); + String suffix = String.format(Locale.ROOT, "_%d", JvmInfo.jvmInfo().getPid()); + logPipeName = wantLogPipe ? String.format(Locale.ROOT, "%slog%s", prefix, suffix) : null; + commandPipeName = wantCommandPipe ? String.format(Locale.ROOT, "%scommand%s", prefix, suffix) : null; + processInPipeName = wantProcessInPipe ? String.format(Locale.ROOT, "%sinput%s", prefix, suffix) : null; + processOutPipeName = wantProcessOutPipe ? String.format(Locale.ROOT, "%soutput%s", prefix, suffix) : null; + restorePipeName = wantRestorePipe ? String.format(Locale.ROOT, "%srestore%s", prefix, suffix) : null; + persistPipeName = wantPersistPipe ? String.format(Locale.ROOT, "%spersist%s", prefix, suffix) : null; + } + + /** + * Augments a list of command line arguments, for example that built up by the AutodetectBuilder class. + */ + public void addArgs(List command) { + if (logPipeName != null) { + command.add(LOG_PIPE_ARG + logPipeName); + } + if (commandPipeName != null) { + command.add(COMMAND_PIPE_ARG + commandPipeName); + } + // The following are specified using two arguments, as the C++ processes could already accept input from files on disk + if (processInPipeName != null) { + command.add(INPUT_ARG + processInPipeName); + command.add(INPUT_IS_PIPE_ARG); + } + if (processOutPipeName != null) { + command.add(OUTPUT_ARG + processOutPipeName); + command.add(OUTPUT_IS_PIPE_ARG); + } + if (restorePipeName != null) { + command.add(RESTORE_ARG + restorePipeName); + command.add(RESTORE_IS_PIPE_ARG); + } + if (persistPipeName != null) { + command.add(PERSIST_ARG + persistPipeName); + command.add(PERSIST_IS_PIPE_ARG); + } + } + + /** + * Connect the pipes created by the C++ process. This must be called after the corresponding C++ process has been started. + * @param timeout Needs to be long enough for the C++ process perform all startup tasks that precede creation of named pipes. + * There should not be very many of these, so a short timeout should be fine. However, at least a couple of + * seconds is recommended due to the vagaries of process scheduling and the way VMs can completely stall for + * some hypervisor actions. + */ + public void connectStreams(Duration timeout) throws IOException { + // The order here is important. It must match the order that the C++ process tries to connect to the pipes, otherwise + // a timeout is guaranteed. Also change api::CIoManager in the C++ code if changing the order here. + if (logPipeName != null) { + logStream = namedPipeHelper.openNamedPipeInputStream(logPipeName, timeout); + } + if (commandPipeName != null) { + commandStream = namedPipeHelper.openNamedPipeOutputStream(commandPipeName, timeout); + } + if (processInPipeName != null) { + processInStream = namedPipeHelper.openNamedPipeOutputStream(processInPipeName, timeout); + } + if (processOutPipeName != null) { + processOutStream = namedPipeHelper.openNamedPipeInputStream(processOutPipeName, timeout); + } + if (restorePipeName != null) { + restoreStream = namedPipeHelper.openNamedPipeOutputStream(restorePipeName, timeout); + } + if (persistPipeName != null) { + persistStream = namedPipeHelper.openNamedPipeInputStream(persistPipeName, timeout); + } + } + + public Optional getLogStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (logPipeName == null) { + return Optional.empty(); + } + if (logStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(logStream); + } + + public Optional getCommandStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (commandPipeName == null) { + return Optional.empty(); + } + if (commandStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(commandStream); + } + + public Optional getProcessInStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (processInPipeName == null) { + return Optional.empty(); + } + if (processInStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(processInStream); + } + + public Optional getProcessOutStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (processOutPipeName == null) { + return Optional.empty(); + } + if (processOutStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(processOutStream); + } + + public Optional getRestoreStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (restorePipeName == null) { + return Optional.empty(); + } + if (restoreStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(restoreStream); + } + + public Optional getPersistStream() { + // Distinguish between pipe not wanted and pipe wanted but not successfully connected + if (persistPipeName == null) { + return Optional.empty(); + } + if (persistStream == null) { + throw new IllegalStateException("process streams must be connected before use"); + } + return Optional.of(persistStream); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java new file mode 100644 index 00000000000..d47c277a38f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectBuilder.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.process.NativeController; +import org.elasticsearch.xpack.ml.job.process.ProcessCtrl; +import org.elasticsearch.xpack.ml.job.process.ProcessPipes; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.AnalysisLimitsWriter; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.FieldConfigWriter; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ModelDebugConfigWriter; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.config.MlFilter; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeoutException; + +/** + * The autodetect process builder. + */ +public class AutodetectBuilder { + private static final String CONF_EXTENSION = ".conf"; + private static final String LIMIT_CONFIG_ARG = "--limitconfig="; + private static final String MODEL_DEBUG_CONFIG_ARG = "--modeldebugconfig="; + private static final String FIELD_CONFIG_ARG = "--fieldconfig="; + + private Job job; + private List filesToDelete; + private Logger logger; + private boolean ignoreDowntime; + private Set referencedFilters; + private Quantiles quantiles; + private Environment env; + private Settings settings; + private NativeController controller; + private ProcessPipes processPipes; + + /** + * Constructs an autodetect process builder + * + * @param job The job configuration + * @param filesToDelete This method will append File objects that need to be + * deleted when the process completes + * @param logger The job's logger + */ + public AutodetectBuilder(Job job, List filesToDelete, Logger logger, Environment env, Settings settings, + NativeController controller, ProcessPipes processPipes) { + this.env = env; + this.settings = settings; + this.controller = controller; + this.processPipes = processPipes; + this.job = Objects.requireNonNull(job); + this.filesToDelete = Objects.requireNonNull(filesToDelete); + this.logger = Objects.requireNonNull(logger); + ignoreDowntime = false; + referencedFilters = new HashSet<>(); + } + + /** + * Set ignoreDowntime + * + * @param ignoreDowntime If true set the ignore downtime flag overriding the + * setting in the job configuration + */ + public AutodetectBuilder ignoreDowntime(boolean ignoreDowntime) { + this.ignoreDowntime = ignoreDowntime; + return this; + } + + public AutodetectBuilder referencedFilters(Set filters) { + referencedFilters = filters; + return this; + } + + /** + * Set quantiles to restore the normalizer state if any. + * + * @param quantiles the quantiles + */ + public AutodetectBuilder quantiles(Quantiles quantiles) { + this.quantiles = quantiles; + return this; + } + + /** + * Requests that the controller daemon start an autodetect process. + */ + public void build() throws IOException, TimeoutException { + + List command = ProcessCtrl.buildAutodetectCommand(env, settings, job, logger, ignoreDowntime, controller.getPid()); + + buildLimits(command); + buildModelDebugConfig(command); + + buildQuantiles(command); + buildFieldConfig(command); + processPipes.addArgs(command); + controller.startProcess(command); + } + + private void buildLimits(List command) throws IOException { + if (job.getAnalysisLimits() != null) { + Path limitConfigFile = Files.createTempFile(env.tmpFile(), "limitconfig", CONF_EXTENSION); + filesToDelete.add(limitConfigFile); + writeLimits(job.getAnalysisLimits(), limitConfigFile); + String limits = LIMIT_CONFIG_ARG + limitConfigFile.toString(); + command.add(limits); + } + } + + /** + * Write the Ml autodetect model options to emptyConfFile. + */ + private static void writeLimits(AnalysisLimits options, Path emptyConfFile) throws IOException { + + try (OutputStreamWriter osw = new OutputStreamWriter(Files.newOutputStream(emptyConfFile), StandardCharsets.UTF_8)) { + new AnalysisLimitsWriter(options, osw).write(); + } + } + + private void buildModelDebugConfig(List command) throws IOException { + if (job.getModelDebugConfig() != null) { + Path modelDebugConfigFile = Files.createTempFile(env.tmpFile(), "modeldebugconfig", CONF_EXTENSION); + filesToDelete.add(modelDebugConfigFile); + writeModelDebugConfig(job.getModelDebugConfig(), modelDebugConfigFile); + String modelDebugConfig = MODEL_DEBUG_CONFIG_ARG + modelDebugConfigFile.toString(); + command.add(modelDebugConfig); + } + } + + private static void writeModelDebugConfig(ModelDebugConfig config, Path emptyConfFile) + throws IOException { + try (OutputStreamWriter osw = new OutputStreamWriter( + Files.newOutputStream(emptyConfFile), + StandardCharsets.UTF_8)) { + new ModelDebugConfigWriter(config, osw).write(); + } + } + + private void buildQuantiles(List command) throws IOException { + if (quantiles != null && !quantiles.getQuantileState().isEmpty()) { + logger.info("Restoring quantiles for job '" + job.getId() + "'"); + + Path normalizersStateFilePath = ProcessCtrl.writeNormalizerInitState( + job.getId(), quantiles.getQuantileState(), env); + + String quantilesStateFileArg = ProcessCtrl.QUANTILES_STATE_PATH_ARG + normalizersStateFilePath; + command.add(quantilesStateFileArg); + command.add(ProcessCtrl.DELETE_STATE_FILES_ARG); + } + } + + private void buildFieldConfig(List command) throws IOException { + if (job.getAnalysisConfig() != null) { + // write to a temporary field config file + Path fieldConfigFile = Files.createTempFile(env.tmpFile(), "fieldconfig", CONF_EXTENSION); + filesToDelete.add(fieldConfigFile); + try (OutputStreamWriter osw = new OutputStreamWriter( + Files.newOutputStream(fieldConfigFile), + StandardCharsets.UTF_8)) { + new FieldConfigWriter(job.getAnalysisConfig(), referencedFilters, osw, logger).write(); + } + + String fieldConfig = FIELD_CONFIG_ARG + fieldConfigFile.toString(); + command.add(fieldConfig); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java new file mode 100644 index 00000000000..24486e8b728 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicator.java @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.CheckedSupplier; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.job.process.CountingInputStream; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.DataToProcessWriter; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.DataToProcessWriterFactory; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Supplier; + +public class AutodetectCommunicator implements Closeable { + + private static final Logger LOGGER = Loggers.getLogger(AutodetectCommunicator.class); + private static final Duration FLUSH_PROCESS_CHECK_FREQUENCY = Duration.ofSeconds(1); + + private final Job job; + private final DataCountsReporter dataCountsReporter; + private final AutodetectProcess autodetectProcess; + private final AutoDetectResultProcessor autoDetectResultProcessor; + private final Consumer handler; + + final AtomicReference inUse = new AtomicReference<>(); + + public AutodetectCommunicator(Job job, AutodetectProcess process, DataCountsReporter dataCountsReporter, + AutoDetectResultProcessor autoDetectResultProcessor, Consumer handler) { + this.job = job; + this.autodetectProcess = process; + this.dataCountsReporter = dataCountsReporter; + this.autoDetectResultProcessor = autoDetectResultProcessor; + this.handler = handler; + } + + public void writeJobInputHeader() throws IOException { + createProcessWriter(Optional.empty()).writeHeader(); + } + + private DataToProcessWriter createProcessWriter(Optional dataDescription) { + return DataToProcessWriterFactory.create(true, autodetectProcess, dataDescription.orElse(job.getDataDescription()), + job.getAnalysisConfig(), dataCountsReporter); + } + + public DataCounts writeToJob(InputStream inputStream, DataLoadParams params) throws IOException { + return checkAndRun(() -> Messages.getMessage(Messages.JOB_DATA_CONCURRENT_USE_UPLOAD, job.getId()), () -> { + if (params.isResettingBuckets()) { + autodetectProcess.writeResetBucketsControlMessage(params); + } + CountingInputStream countingStream = new CountingInputStream(inputStream, dataCountsReporter); + + DataToProcessWriter autoDetectWriter = createProcessWriter(params.getDataDescription()); + DataCounts results = autoDetectWriter.write(countingStream); + autoDetectWriter.flush(); + return results; + }, false); + } + + @Override + public void close() throws IOException { + checkAndRun(() -> Messages.getMessage(Messages.JOB_DATA_CONCURRENT_USE_CLOSE, job.getId()), () -> { + dataCountsReporter.close(); + autodetectProcess.close(); + autoDetectResultProcessor.awaitCompletion(); + handler.accept(null); + return null; + }, true); + } + + + public void writeUpdateModelDebugMessage(ModelDebugConfig config) throws IOException { + checkAndRun(() -> Messages.getMessage(Messages.JOB_DATA_CONCURRENT_USE_UPDATE, job.getId()), () -> { + autodetectProcess.writeUpdateModelDebugMessage(config); + return null; + }, false); + } + + public void writeUpdateDetectorRulesMessage(int detectorIndex, List rules) throws IOException { + checkAndRun(() -> Messages.getMessage(Messages.JOB_DATA_CONCURRENT_USE_UPDATE, job.getId()), () -> { + autodetectProcess.writeUpdateDetectorRulesMessage(detectorIndex, rules); + return null; + }, false); + } + + public void flushJob(InterimResultsParams params) throws IOException { + checkAndRun(() -> Messages.getMessage(Messages.JOB_DATA_CONCURRENT_USE_FLUSH, job.getId()), () -> { + String flushId = autodetectProcess.flushJob(params); + waitFlushToCompletion(flushId); + return null; + }, false); + } + + private void waitFlushToCompletion(String flushId) throws IOException { + LOGGER.info("[{}] waiting for flush", job.getId()); + + try { + boolean isFlushComplete = autoDetectResultProcessor.waitForFlushAcknowledgement(flushId, FLUSH_PROCESS_CHECK_FREQUENCY); + while (isFlushComplete == false) { + checkProcessIsAlive(); + isFlushComplete = autoDetectResultProcessor.waitForFlushAcknowledgement(flushId, FLUSH_PROCESS_CHECK_FREQUENCY); + } + } finally { + autoDetectResultProcessor.clearAwaitingFlush(flushId); + } + + // We also have to wait for the normalizer to become idle so that we block + // clients from querying results in the middle of normalization. + autoDetectResultProcessor.waitUntilRenormalizerIsIdle(); + + LOGGER.info("[{}] Flush completed", job.getId()); + } + + /** + * Throws an exception if the process has exited + */ + private void checkProcessIsAlive() { + if (!autodetectProcess.isProcessAlive()) { + ParameterizedMessage message = + new ParameterizedMessage("[{}] Unexpected death of autodetect: {}", job.getId(), autodetectProcess.readError()); + LOGGER.error(message); + throw ExceptionsHelper.serverError(message.getFormattedMessage()); + } + } + + public ZonedDateTime getProcessStartTime() { + return autodetectProcess.getProcessStartTime(); + } + + public ModelSizeStats getModelSizeStats() { + return autoDetectResultProcessor.modelSizeStats(); + } + + public DataCounts getDataCounts() { + return dataCountsReporter.runningTotalStats(); + } + + private T checkAndRun(Supplier errorMessage, CheckedSupplier callback, boolean wait) throws IOException { + CountDownLatch latch = new CountDownLatch(1); + if (inUse.compareAndSet(null, latch)) { + try { + checkProcessIsAlive(); + return callback.get(); + } finally { + latch.countDown(); + inUse.set(null); + } + } else { + if (wait) { + latch = inUse.get(); + if (latch != null) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ElasticsearchStatusException(errorMessage.get(), RestStatus.TOO_MANY_REQUESTS); + } + } + checkProcessIsAlive(); + return callback.get(); + } else { + throw new ElasticsearchStatusException(errorMessage.get(), RestStatus.TOO_MANY_REQUESTS); + } + } + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java new file mode 100644 index 00000000000..cb659f287c9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.ml.job.results.AutodetectResult; + +import java.io.Closeable; +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.Iterator; +import java.util.List; + +/** + * Interface representing the native C++ autodetect process + */ +public interface AutodetectProcess extends Closeable { + + /** + * Write the record to autodetect. The record parameter should not be encoded + * (i.e. length encoded) the implementation will appy the corrrect encoding. + * + * @param record Plain array of strings, implementors of this class should + * encode the record appropriately + * @throws IOException If the write failed + */ + void writeRecord(String[] record) throws IOException; + + /** + * Write the reset buckets control message + * + * @param params Reset bucket options + * @throws IOException If write reset message fails + */ + void writeResetBucketsControlMessage(DataLoadParams params) throws IOException; + + /** + * Update the model debug configuration + * + * @param modelDebugConfig New model debug config + * @throws IOException If the write fails + */ + void writeUpdateModelDebugMessage(ModelDebugConfig modelDebugConfig) throws IOException; + + /** + * Write message to update the detector rules + * + * @param detectorIndex Index of the detector to update + * @param rules Detector rules + * @throws IOException If the write fails + */ + void writeUpdateDetectorRulesMessage(int detectorIndex, List rules) throws IOException; + + /** + * Flush the job pushing any stale data into autodetect. + * Every flush command generates a unique flush Id which will be output + * in a flush acknowledgment by the autodetect process once the flush has + * been processed. + * + * @param params Should interim results be generated + * @return The flush Id + * @throws IOException If the flush failed + */ + String flushJob(InterimResultsParams params) throws IOException; + + /** + * Flush the output data stream + */ + void flushStream() throws IOException; + + /** + * @return stream of autodetect results. + */ + Iterator readAutodetectResults(); + + /** + * The time the process was started + * @return Process start time + */ + ZonedDateTime getProcessStartTime(); + + /** + * Returns true if the process still running. + * Methods such as {@link #flushJob(InterimResultsParams)} are essentially + * asynchronous the command will be continue to execute in the process after + * the call has returned. This method tests whether something catastrophic + * occurred in the process during its execution. + * @return True if the process is still running + */ + boolean isProcessAlive(); + + /** + * Read any content in the error output buffer. + * @return An error message or empty String if no error. + */ + String readError(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java new file mode 100644 index 00000000000..37d08506acb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessFactory.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.config.MlFilter; + +import java.util.Set; +import java.util.concurrent.ExecutorService; + +/** + * Factory interface for creating implementations of {@link AutodetectProcess} + */ +public interface AutodetectProcessFactory { + + /** + * Create an implementation of {@link AutodetectProcess} + * + * @param job Job configuration for the analysis process + * @param modelSnapshot The model snapshot to restore from + * @param quantiles The quantiles to push to the native process + * @param filters The filters to push to the native process + * @param ignoreDowntime Should gaps in data be treated as anomalous or as a maintenance window after a job re-start + * @param executorService Executor service used to start the async tasks a job needs to operate the analytical process + * @return The process + */ + AutodetectProcess createAutodetectProcess(Job job, ModelSnapshot modelSnapshot, Quantiles quantiles, Set filters, + boolean ignoreDowntime, ExecutorService executorService); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java new file mode 100644 index 00000000000..a27ab36e342 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -0,0 +1,337 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.UpdateJobStateAction; +import org.elasticsearch.xpack.ml.job.JobManager; +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobRenormalizedResultsPersister; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerFactory; +import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; +import org.elasticsearch.xpack.ml.job.process.normalizer.ScoresUpdater; +import org.elasticsearch.xpack.ml.job.process.normalizer.ShortCircuitingRenormalizer; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +public class AutodetectProcessManager extends AbstractComponent { + + // TODO (norelease) default needs to be reconsidered + // Cannot be dynamic because the thread pool that is sized to match cannot be resized + public static final Setting MAX_RUNNING_JOBS_PER_NODE = + Setting.intSetting("max_running_jobs", 10, 1, 512, Setting.Property.NodeScope); + + private final Client client; + private final ThreadPool threadPool; + private final JobManager jobManager; + private final JobProvider jobProvider; + private final AutodetectProcessFactory autodetectProcessFactory; + private final NormalizerFactory normalizerFactory; + + private final JobResultsPersister jobResultsPersister; + private final JobDataCountsPersister jobDataCountsPersister; + + private final ConcurrentMap autoDetectCommunicatorByJob; + + private final int maxAllowedRunningJobs; + + public AutodetectProcessManager(Settings settings, Client client, ThreadPool threadPool, JobManager jobManager, + JobProvider jobProvider, JobResultsPersister jobResultsPersister, + JobDataCountsPersister jobDataCountsPersister, + AutodetectProcessFactory autodetectProcessFactory, NormalizerFactory normalizerFactory) { + super(settings); + this.client = client; + this.threadPool = threadPool; + this.maxAllowedRunningJobs = MAX_RUNNING_JOBS_PER_NODE.get(settings); + this.autodetectProcessFactory = autodetectProcessFactory; + this.normalizerFactory = normalizerFactory; + this.jobManager = jobManager; + this.jobProvider = jobProvider; + + this.jobResultsPersister = jobResultsPersister; + this.jobDataCountsPersister = jobDataCountsPersister; + + this.autoDetectCommunicatorByJob = new ConcurrentHashMap<>(); + } + + /** + * Passes data to the native process. + * This is a blocking call that won't return until all the data has been + * written to the process. + * + * An ElasticsearchStatusException will be thrown is any of these error conditions occur: + *

    + *
  1. If a configured field is missing from the CSV header
    2. + *
  2. If JSON data is malformed and we cannot recover parsing
    3. + *
  3. If a high proportion of the records the timestamp field that cannot be parsed
    4. + *
  4. If a high proportion of the records chronologically out of order
    5. + *
+ * + * @param jobId the jobId + * @param input Data input stream + * @param params Data processing parameters + * @return Count of records, fields, bytes, etc written + */ + public DataCounts processData(String jobId, InputStream input, DataLoadParams params) { + Allocation allocation = jobManager.getJobAllocation(jobId); + if (allocation.getState() != JobState.OPENED) { + throw new IllegalArgumentException("job [" + jobId + "] state is [" + allocation.getState() + "], but must be [" + + JobState.OPENED + "] for processing data"); + } + + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + throw new IllegalStateException("job [" + jobId + "] with state [" + allocation.getState() + "] hasn't been started"); + } + try { + return communicator.writeToJob(input, params); + // TODO check for errors from autodetect + } catch (IOException e) { + String msg = String.format(Locale.ROOT, "Exception writing to process for job %s", jobId); + if (e.getCause() instanceof TimeoutException) { + logger.warn("Connection to process was dropped due to a timeout - if you are feeding this job from a connector it " + + "may be that your connector stalled for too long", e.getCause()); + } else { + logger.error("Unexpected exception", e); + } + throw ExceptionsHelper.serverError(msg, e); + } + } + + /** + * Flush the running job, ensuring that the native process has had the + * opportunity to process all data previously sent to it with none left + * sitting in buffers. + * + * @param jobId The job to flush + * @param params Parameters about whether interim results calculation + * should occur and for which period of time + */ + public void flushJob(String jobId, InterimResultsParams params) { + logger.debug("Flushing job {}", jobId); + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + String message = String.format(Locale.ROOT, "[%s] Cannot flush: no active autodetect process for job", jobId); + logger.debug(message); + throw new IllegalArgumentException(message); + } + try { + communicator.flushJob(params); + // TODO check for errors from autodetect + } catch (IOException ioe) { + String msg = String.format(Locale.ROOT, "[%s] exception while flushing job", jobId); + logger.error(msg); + throw ExceptionsHelper.serverError(msg, ioe); + } + } + + public void writeUpdateModelDebugMessage(String jobId, ModelDebugConfig config) throws IOException { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + logger.debug("Cannot update model debug config: no active autodetect process for job {}", jobId); + return; + } + communicator.writeUpdateModelDebugMessage(config); + // TODO check for errors from autodetects + } + + public void writeUpdateDetectorRulesMessage(String jobId, int detectorIndex, List rules) throws IOException { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + logger.debug("Cannot update model debug config: no active autodetect process for job {}", jobId); + return; + } + communicator.writeUpdateDetectorRulesMessage(detectorIndex, rules); + // TODO check for errors from autodetects + } + + public void openJob(String jobId, boolean ignoreDowntime, Consumer handler) { + gatherRequiredInformation(jobId, (dataCounts, modelSnapshot, quantiles, filters) -> { + autoDetectCommunicatorByJob.computeIfAbsent(jobId, id -> { + AutodetectCommunicator communicator = + create(id, dataCounts, modelSnapshot, quantiles, filters, ignoreDowntime, handler); + try { + communicator.writeJobInputHeader(); + } catch (IOException ioe) { + String msg = String.format(Locale.ROOT, "[%s] exception while opening job", jobId); + logger.error(msg); + throw ExceptionsHelper.serverError(msg, ioe); + } + setJobState(jobId, JobState.OPENED); + return communicator; + }); + }, handler); + } + + // TODO: add a method on JobProvider that fetches all required info via 1 msearch call, so that we have a single lambda + // instead of 4 nested lambdas. + void gatherRequiredInformation(String jobId, MultiConsumer handler, Consumer errorHandler) { + Job job = jobManager.getJobOrThrowIfUnknown(jobId); + jobProvider.dataCounts(jobId, dataCounts -> { + jobProvider.modelSnapshots(jobId, 0, 1, page -> { + ModelSnapshot modelSnapshot = page.results().isEmpty() ? null : page.results().get(0); + jobProvider.getQuantiles(jobId, quantiles -> { + Set ids = job.getAnalysisConfig().extractReferencedFilters(); + jobProvider.getFilters(filterDocument -> handler.accept(dataCounts, modelSnapshot, quantiles, filterDocument), + errorHandler, ids); + }, errorHandler); + }, errorHandler); + }, errorHandler); + } + + interface MultiConsumer { + + void accept(DataCounts dataCounts, ModelSnapshot modelSnapshot, Quantiles quantiles, Set filters); + + } + + AutodetectCommunicator create(String jobId, DataCounts dataCounts, ModelSnapshot modelSnapshot, Quantiles quantiles, + Set filters, boolean ignoreDowntime, Consumer handler) { + if (autoDetectCommunicatorByJob.size() == maxAllowedRunningJobs) { + throw new ElasticsearchStatusException("max running job capacity [" + maxAllowedRunningJobs + "] reached", + RestStatus.CONFLICT); + } + + Job job = jobManager.getJobOrThrowIfUnknown(jobId); + // A TP with no queue, so that we fail immediately if there are no threads available + ExecutorService executorService = threadPool.executor(MlPlugin.AUTODETECT_PROCESS_THREAD_POOL_NAME); + try (DataCountsReporter dataCountsReporter = new DataCountsReporter(threadPool, settings, job.getId(), dataCounts, + jobDataCountsPersister)) { + ScoresUpdater scoresUpdater = new ScoresUpdater(job, jobProvider, new JobRenormalizedResultsPersister(settings, client), + normalizerFactory); + Renormalizer renormalizer = new ShortCircuitingRenormalizer(jobId, scoresUpdater, + threadPool.executor(MlPlugin.THREAD_POOL_NAME), job.getAnalysisConfig().getUsePerPartitionNormalization()); + + AutodetectProcess process = autodetectProcessFactory.createAutodetectProcess(job, modelSnapshot, quantiles, filters, + ignoreDowntime, executorService); + boolean usePerPartitionNormalization = job.getAnalysisConfig().getUsePerPartitionNormalization(); + AutoDetectResultProcessor processor = new AutoDetectResultProcessor(jobId, renormalizer, jobResultsPersister); + try { + executorService.submit(() -> processor.process(process, usePerPartitionNormalization)); + } catch (EsRejectedExecutionException e) { + // If submitting the operation to read the results from the process fails we need to close + // the process too, so that other submitted operations to threadpool are stopped. + try { + IOUtils.close(process); + } catch (IOException ioe) { + logger.error("Can't close autodetect", ioe); + } + throw e; + } + return new AutodetectCommunicator(job, process, dataCountsReporter, processor, handler); + } + } + + /** + * Stop the running job and mark it as finished.
+ * @param jobId The job to stop + * + */ + public void closeJob(String jobId) { + logger.debug("Closing job {}", jobId); + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.remove(jobId); + if (communicator == null) { + logger.debug("Cannot close: no active autodetect process for job {}", jobId); + return; + } + + try { + communicator.close(); + setJobState(jobId, JobState.CLOSED); + } catch (Exception e) { + logger.warn("Exception closing stopped process input stream", e); + throw ExceptionsHelper.serverError("Exception closing stopped process input stream", e); + } + } + + int numberOfOpenJobs() { + return autoDetectCommunicatorByJob.size(); + } + + boolean jobHasActiveAutodetectProcess(String jobId) { + return autoDetectCommunicatorByJob.get(jobId) != null; + } + + public Duration jobUpTime(String jobId) { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + return Duration.ZERO; + } + return Duration.between(communicator.getProcessStartTime(), ZonedDateTime.now()); + } + + private void setJobState(String jobId, JobState state) { + UpdateJobStateAction.Request request = new UpdateJobStateAction.Request(jobId, state); + client.execute(UpdateJobStateAction.INSTANCE, request, new ActionListener() { + @Override + public void onResponse(UpdateJobStateAction.Response response) { + if (response.isAcknowledged()) { + logger.info("Successfully set job state to [{}] for job [{}]", state, jobId); + } else { + logger.info("Changing job state to [{}] for job [{}] wasn't acked", state, jobId); + } + } + + @Override + public void onFailure(Exception e) { + logger.error("Could not set job state to [" + state + "] for job [" + jobId +"]", e); + } + }); + } + + public void setJobState(String jobId, JobState state, Consumer handler, Consumer errorHandler) { + UpdateJobStateAction.Request request = new UpdateJobStateAction.Request(jobId, state); + client.execute(UpdateJobStateAction.INSTANCE, request, ActionListener.wrap(r -> handler.accept(null), errorHandler)); + } + + public Optional> getStatistics(String jobId) { + AutodetectCommunicator communicator = autoDetectCommunicatorByJob.get(jobId); + if (communicator == null) { + return Optional.empty(); + } + return Optional.of(new Tuple<>(communicator.getDataCounts(), communicator.getModelSizeStats())); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java new file mode 100644 index 00000000000..03e8dbd8a53 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/BlackHoleAutodetectProcess.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.ml.job.results.AutodetectResult; + +import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; + +/** + * A placeholder class simulating the actions of the native Autodetect process. + * Most methods consume data without performing any action however, after a call to + * {@link #flushJob(InterimResultsParams)} a {@link org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledgement} + * message is expected on the {@link #readAutodetectResults()} ()} stream. This class writes the flush + * acknowledgement immediately. + */ +public class BlackHoleAutodetectProcess implements AutodetectProcess { + + private static final Logger LOGGER = Loggers.getLogger(BlackHoleAutodetectProcess.class); + private static final String FLUSH_ID = "flush-1"; + + private final ZonedDateTime startTime; + + private final BlockingQueue results = new ArrayBlockingQueue<>(128); + + public BlackHoleAutodetectProcess() { + startTime = ZonedDateTime.now(); + } + + @Override + public void writeRecord(String[] record) throws IOException { + } + + @Override + public void writeResetBucketsControlMessage(DataLoadParams params) throws IOException { + } + + @Override + public void writeUpdateModelDebugMessage(ModelDebugConfig modelDebugConfig) throws IOException { + } + + @Override + public void writeUpdateDetectorRulesMessage(int detectorIndex, List rules) throws IOException { + } + + /** + * Accept the request do nothing with it but write the flush acknowledgement to {@link #readAutodetectResults()} + * @param params Should interim results be generated + * @return {@link #FLUSH_ID} + */ + @Override + public String flushJob(InterimResultsParams params) throws IOException { + FlushAcknowledgement flushAcknowledgement = new FlushAcknowledgement(FLUSH_ID); + AutodetectResult result = new AutodetectResult(null, null, null, null, null, null, null, null, flushAcknowledgement); + results.add(result); + return FLUSH_ID; + } + + @Override + public void flushStream() throws IOException { + } + + @Override + public void close() throws IOException { + } + + @Override + public Iterator readAutodetectResults() { + // Create a custom iterator here, because ArrayBlockingQueue iterator and stream are not blocking when empty: + return new Iterator() { + + AutodetectResult result; + + @Override + public boolean hasNext() { + try { + result = results.take(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + + @Override + public AutodetectResult next() { + return result; + } + }; + } + + @Override + public ZonedDateTime getProcessStartTime() { + return startTime; + } + + @Override + public boolean isProcessAlive() { + return true; + } + + @Override + public String readError() { + return ""; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java new file mode 100644 index 00000000000..e624b18322f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcess.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.StateProcessor; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.ControlMsgToProcessWriter; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.LengthEncodedWriter; +import org.elasticsearch.xpack.ml.job.process.logging.CppLogMessageHandler; +import org.elasticsearch.xpack.ml.job.results.AutodetectResult; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.ZonedDateTime; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Autodetect process using native code. + */ +class NativeAutodetectProcess implements AutodetectProcess { + private static final Logger LOGGER = Loggers.getLogger(NativeAutodetectProcess.class); + + private final String jobId; + private final CppLogMessageHandler cppLogHandler; + private final OutputStream processInStream; + private final InputStream processOutStream; + private final LengthEncodedWriter recordWriter; + private final ZonedDateTime startTime; + private final int numberOfAnalysisFields; + private final List filesToDelete; + private Future logTailFuture; + private Future stateProcessorFuture; + private AutodetectResultsParser resultsParser; + + NativeAutodetectProcess(String jobId, InputStream logStream, OutputStream processInStream, InputStream processOutStream, + int numberOfAnalysisFields, List filesToDelete, AutodetectResultsParser resultsParser) { + this.jobId = jobId; + cppLogHandler = new CppLogMessageHandler(jobId, logStream); + this.processInStream = new BufferedOutputStream(processInStream); + this.processOutStream = processOutStream; + this.recordWriter = new LengthEncodedWriter(this.processInStream); + startTime = ZonedDateTime.now(); + this.numberOfAnalysisFields = numberOfAnalysisFields; + this.filesToDelete = filesToDelete; + this.resultsParser = resultsParser; + } + + public void start(ExecutorService executorService, StateProcessor stateProcessor, InputStream persistStream) { + logTailFuture = executorService.submit(() -> { + try (CppLogMessageHandler h = cppLogHandler) { + h.tailStream(); + } catch (IOException e) { + LOGGER.error(new ParameterizedMessage("[{}] Error tailing C++ process logs", new Object[] { jobId }), e); + } + }); + stateProcessorFuture = executorService.submit(() -> { + stateProcessor.process(jobId, persistStream); + }); + } + + @Override + public void writeRecord(String[] record) throws IOException { + recordWriter.writeRecord(record); + } + + @Override + public void writeResetBucketsControlMessage(DataLoadParams params) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeResetBucketsMessage(params); + } + + @Override + public void writeUpdateModelDebugMessage(ModelDebugConfig modelDebugConfig) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeUpdateModelDebugMessage(modelDebugConfig); + } + + @Override + public void writeUpdateDetectorRulesMessage(int detectorIndex, List rules) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeUpdateDetectorRulesMessage(detectorIndex, rules); + } + + @Override + public String flushJob(InterimResultsParams params) throws IOException { + ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(recordWriter, numberOfAnalysisFields); + writer.writeCalcInterimMessage(params); + return writer.writeFlushMessage(); + } + + @Override + public void flushStream() throws IOException { + recordWriter.flush(); + } + + @Override + public void close() throws IOException { + try { + // closing its input causes the process to exit + processInStream.close(); + // wait for the process to exit by waiting for end-of-file on the named pipe connected to its logger + // this may take a long time as it persists the model state + logTailFuture.get(30, TimeUnit.MINUTES); + // the state processor should have stopped by now as the process should have exit + stateProcessorFuture.get(1, TimeUnit.SECONDS); + if (cppLogHandler.seenFatalError()) { + throw ExceptionsHelper.serverError(cppLogHandler.getErrors()); + } + LOGGER.debug("[{}] Autodetect process exited", jobId); + } catch (ExecutionException | TimeoutException e) { + LOGGER.warn(new ParameterizedMessage("[{}] Exception closing the running autodetect process", + new Object[] { jobId }), e); + } catch (InterruptedException e) { + LOGGER.warn("[{}] Exception closing the running autodetect process", jobId); + Thread.currentThread().interrupt(); + } finally { + deleteAssociatedFiles(); + } + } + + void deleteAssociatedFiles() throws IOException { + if (filesToDelete == null) { + return; + } + + for (Path fileToDelete : filesToDelete) { + if (Files.deleteIfExists(fileToDelete)) { + LOGGER.debug("[{}] Deleted file {}", jobId, fileToDelete.toString()); + } else { + LOGGER.warn("[{}] Failed to delete file {}", jobId, fileToDelete.toString()); + } + } + } + + @Override + public Iterator readAutodetectResults() { + return resultsParser.parseResults(processOutStream); + } + + @Override + public ZonedDateTime getProcessStartTime() { + return startTime; + } + + @Override + public boolean isProcessAlive() { + // Sanity check: make sure the process hasn't terminated already + return !cppLogHandler.hasLogStreamEnded(); + } + + @Override + public String readError() { + return cppLogHandler.getErrors(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java new file mode 100644 index 00000000000..22cca1d7eb5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/NativeAutodetectProcessFactory.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.NativeController; +import org.elasticsearch.xpack.ml.job.process.ProcessCtrl; +import org.elasticsearch.xpack.ml.job.process.ProcessPipes; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutodetectResultsParser; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.StateProcessor; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.NamedPipeHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeoutException; + +public class NativeAutodetectProcessFactory implements AutodetectProcessFactory { + + private static final Logger LOGGER = Loggers.getLogger(NativeAutodetectProcessFactory.class); + private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper(); + private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(2); + + private final Client client; + private final Environment env; + private final Settings settings; + private final JobProvider jobProvider; + private final NativeController nativeController; + + public NativeAutodetectProcessFactory(JobProvider jobProvider, Environment env, Settings settings, + NativeController nativeController, Client client) { + this.env = Objects.requireNonNull(env); + this.settings = Objects.requireNonNull(settings); + this.jobProvider = Objects.requireNonNull(jobProvider); + this.nativeController = Objects.requireNonNull(nativeController); + this.client = client; + } + + @Override + public AutodetectProcess createAutodetectProcess(Job job, ModelSnapshot modelSnapshot, Quantiles quantiles, Set filters, + boolean ignoreDowntime, ExecutorService executorService) { + List filesToDelete = new ArrayList<>(); + ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, ProcessCtrl.AUTODETECT, job.getId(), + true, false, true, true, modelSnapshot != null, !ProcessCtrl.DONT_PERSIST_MODEL_STATE_SETTING.get(settings)); + createNativeProcess(job, quantiles, filters, processPipes, ignoreDowntime, filesToDelete); + int numberOfAnalysisFields = job.getAnalysisConfig().analysisFields().size(); + + StateProcessor stateProcessor = new StateProcessor(settings, client); + AutodetectResultsParser resultsParser = new AutodetectResultsParser(settings); + NativeAutodetectProcess autodetect = new NativeAutodetectProcess( + job.getId(), processPipes.getLogStream().get(), processPipes.getProcessInStream().get(), + processPipes.getProcessOutStream().get(), numberOfAnalysisFields, filesToDelete, resultsParser + ); + try { + autodetect.start(executorService, stateProcessor, processPipes.getPersistStream().get()); + if (modelSnapshot != null) { + // TODO (norelease): I don't think we should do this in the background. If this happens then we should wait + // until restore it is done before we can accept data. + executorService.execute(() -> { + try (OutputStream r = processPipes.getRestoreStream().get()) { + jobProvider.restoreStateToStream(job.getId(), modelSnapshot, r); + } catch (Exception e) { + LOGGER.error("Error restoring model state for job " + job.getId(), e); + } + }); + } + return autodetect; + } catch (EsRejectedExecutionException e) { + try { + IOUtils.close(autodetect); + } catch (IOException ioe) { + LOGGER.error("Can't close autodetect", ioe); + } + throw e; + } + } + + private void createNativeProcess(Job job, Quantiles quantiles, Set filters, ProcessPipes processPipes, + boolean ignoreDowntime, List filesToDelete) { + try { + AutodetectBuilder autodetectBuilder = new AutodetectBuilder(job, filesToDelete, LOGGER, env, + settings, nativeController, processPipes) + .ignoreDowntime(ignoreDowntime) + .referencedFilters(filters); + + // if state is null or empty it will be ignored + // else it is used to restore the quantiles + if (quantiles != null) { + autodetectBuilder.quantiles(quantiles); + } + + autodetectBuilder.build(); + processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT); + } catch (IOException | TimeoutException e) { + String msg = "Failed to launch autodetect for job " + job.getId(); + LOGGER.error(msg); + throw ExceptionsHelper.serverError(msg, e); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java new file mode 100644 index 00000000000..472154d4916 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.output; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.AutodetectResult; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.ml.job.results.PerPartitionMaxProbabilities; + +import java.time.Duration; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +/** + * A runnable class that reads the autodetect process output in the + * {@link #process(AutodetectProcess, boolean)} method and persists parsed + * results via the {@linkplain JobResultsPersister} passed in the constructor. + *

+ * Has methods to register and remove alert observers. + * Also has a method to wait for a flush to be complete. + * + * Buckets are the written last after records, influencers etc + * when the end of bucket is reached. Therefore results aren't persisted + * until the bucket is read, this means that interim results for all + * result types can be safely deleted when the bucket is read and before + * the new results are updated. This is specifically for the case where + * a flush command is issued repeatedly in the same bucket to generate + * interim results and the old interim results have to be cleared out + * before the new ones are written. + */ +public class AutoDetectResultProcessor { + + private static final Logger LOGGER = Loggers.getLogger(AutoDetectResultProcessor.class); + + private final String jobId; + private final Renormalizer renormalizer; + private final JobResultsPersister persister; + + final CountDownLatch completionLatch = new CountDownLatch(1); + private final FlushListener flushListener; + + private volatile ModelSizeStats latestModelSizeStats; + + public AutoDetectResultProcessor(String jobId, Renormalizer renormalizer, JobResultsPersister persister) { + this(jobId, renormalizer, persister, new FlushListener()); + } + + AutoDetectResultProcessor(String jobId, Renormalizer renormalizer, JobResultsPersister persister, FlushListener flushListener) { + this.jobId = jobId; + this.renormalizer = renormalizer; + this.persister = persister; + this.flushListener = flushListener; + + ModelSizeStats.Builder builder = new ModelSizeStats.Builder(jobId); + latestModelSizeStats = builder.build(); + } + + public void process(AutodetectProcess process, boolean isPerPartitionNormalization) { + Context context = new Context(jobId, isPerPartitionNormalization, persister.bulkPersisterBuilder(jobId)); + try { + int bucketCount = 0; + Iterator iterator = process.readAutodetectResults(); + while (iterator.hasNext()) { + AutodetectResult result = iterator.next(); + processResult(context, result); + if (result.getBucket() != null) { + bucketCount++; + LOGGER.trace("[{}] Bucket number {} parsed from output", jobId, bucketCount); + } + } + context.bulkResultsPersister.executeRequest(); + LOGGER.info("[{}] {} buckets parsed from autodetect output", jobId, bucketCount); + LOGGER.info("[{}] Parse results Complete", jobId); + } catch (Exception e) { + LOGGER.error(new ParameterizedMessage("[{}] error parsing autodetect output", new Object[] {jobId}), e); + } finally { + waitUntilRenormalizerIsIdle(); + flushListener.clear(); + completionLatch.countDown(); + } + } + + void processResult(Context context, AutodetectResult result) { + Bucket bucket = result.getBucket(); + if (bucket != null) { + if (context.deleteInterimRequired) { + // Delete any existing interim results generated by a Flush command + // which have not been replaced or superseded by new results. + LOGGER.trace("[{}] Deleting interim results", context.jobId); + persister.deleteInterimResults(context.jobId); + context.deleteInterimRequired = false; + } + + // persist after deleting interim results in case the new + // results are also interim + context.bulkResultsPersister.persistBucket(bucket).executeRequest(); + context.bulkResultsPersister = persister.bulkPersisterBuilder(context.jobId); + } + List records = result.getRecords(); + if (records != null && !records.isEmpty()) { + context.bulkResultsPersister.persistRecords(records); + if (context.isPerPartitionNormalization) { + context.bulkResultsPersister.persistPerPartitionMaxProbabilities(new PerPartitionMaxProbabilities(records)); + } + } + List influencers = result.getInfluencers(); + if (influencers != null && !influencers.isEmpty()) { + context.bulkResultsPersister.persistInfluencers(influencers); + } + CategoryDefinition categoryDefinition = result.getCategoryDefinition(); + if (categoryDefinition != null) { + persister.persistCategoryDefinition(categoryDefinition); + } + ModelDebugOutput modelDebugOutput = result.getModelDebugOutput(); + if (modelDebugOutput != null) { + persister.persistModelDebugOutput(modelDebugOutput); + } + ModelSizeStats modelSizeStats = result.getModelSizeStats(); + if (modelSizeStats != null) { + LOGGER.trace("[{}] Parsed ModelSizeStats: {} / {} / {} / {} / {} / {}", + context.jobId, modelSizeStats.getModelBytes(), modelSizeStats.getTotalByFieldCount(), + modelSizeStats.getTotalOverFieldCount(), modelSizeStats.getTotalPartitionFieldCount(), + modelSizeStats.getBucketAllocationFailuresCount(), modelSizeStats.getMemoryStatus()); + + latestModelSizeStats = modelSizeStats; + persister.persistModelSizeStats(modelSizeStats); + } + ModelSnapshot modelSnapshot = result.getModelSnapshot(); + if (modelSnapshot != null) { + persister.persistModelSnapshot(modelSnapshot); + } + Quantiles quantiles = result.getQuantiles(); + if (quantiles != null) { + persister.persistQuantiles(quantiles); + + LOGGER.debug("[{}] Quantiles parsed from output - will trigger renormalization of scores", context.jobId); + renormalizer.renormalize(quantiles); + } + FlushAcknowledgement flushAcknowledgement = result.getFlushAcknowledgement(); + if (flushAcknowledgement != null) { + LOGGER.debug("[{}] Flush acknowledgement parsed from output for ID {}", context.jobId, flushAcknowledgement.getId()); + // Commit previous writes here, effectively continuing + // the flush from the C++ autodetect process right + // through to the data store + context.bulkResultsPersister.executeRequest(); + persister.commitResultWrites(context.jobId); + flushListener.acknowledgeFlush(flushAcknowledgement.getId()); + // Interim results may have been produced by the flush, + // which need to be + // deleted when the next finalized results come through + context.deleteInterimRequired = true; + } + } + + public void awaitCompletion() { + try { + completionLatch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + /** + * Blocks until a flush is acknowledged or the timeout expires, whichever happens first. + * + * @param flushId the id of the flush request to wait for + * @param timeout the timeout + * @return {@code true} if the flush has completed or the parsing finished; {@code false} if the timeout expired + */ + public boolean waitForFlushAcknowledgement(String flushId, Duration timeout) { + return flushListener.waitForFlush(flushId, timeout); + } + + public void clearAwaitingFlush(String flushId) { + flushListener.clear(flushId); + } + + public void waitUntilRenormalizerIsIdle() { + renormalizer.waitUntilIdle(); + } + + static class Context { + + private final String jobId; + private final boolean isPerPartitionNormalization; + private JobResultsPersister.Builder bulkResultsPersister; + + boolean deleteInterimRequired; + + Context(String jobId, boolean isPerPartitionNormalization, JobResultsPersister.Builder bulkResultsPersister) { + this.jobId = jobId; + this.isPerPartitionNormalization = isPerPartitionNormalization; + this.deleteInterimRequired = false; + this.bulkResultsPersister = bulkResultsPersister; + } + } + + public ModelSizeStats modelSizeStats() { + return latestModelSizeStats; + } + +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java new file mode 100644 index 00000000000..368edf2e07b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParser.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.output; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.ml.job.results.AutodetectResult; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Iterator; + + +/** + * Parses the JSON output of the autodetect program. + *

+ * Expects an array of buckets so the first element will always be the + * start array symbol and the data must be terminated with the end array symbol. + */ +public class AutodetectResultsParser extends AbstractComponent { + + public AutodetectResultsParser(Settings settings) { + super(settings); + } + + public Iterator parseResults(InputStream in) throws ElasticsearchParseException { + try { + XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(NamedXContentRegistry.EMPTY, in); + XContentParser.Token token = parser.nextToken(); + // if start of an array ignore it, we expect an array of buckets + if (token != XContentParser.Token.START_ARRAY) { + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + return new AutodetectResultIterator(in, parser); + } catch (IOException e) { + consumeAndCloseStream(in); + throw new ElasticsearchParseException(e.getMessage(), e); + } + } + + private void consumeAndCloseStream(InputStream in) { + try { + // read anything left in the stream before + // closing the stream otherwise if the process + // tries to write more after the close it gets + // a SIGPIPE + byte[] buff = new byte[512]; + while (in.read(buff) >= 0) { + // Do nothing + } + in.close(); + } catch (IOException e) { + logger.warn("Error closing result parser input stream", e); + } + } + + private class AutodetectResultIterator implements Iterator { + + private final InputStream in; + private final XContentParser parser; + private XContentParser.Token token; + + private AutodetectResultIterator(InputStream in, XContentParser parser) { + this.in = in; + this.parser = parser; + token = parser.currentToken(); + } + + @Override + public boolean hasNext() { + try { + token = parser.nextToken(); + } catch (IOException e) { + throw new ElasticsearchParseException(e.getMessage(), e); + } + if (token == XContentParser.Token.END_ARRAY) { + consumeAndCloseStream(in); + return false; + } else if (token != XContentParser.Token.START_OBJECT) { + logger.error("Expecting Json Field name token after the Start Object token"); + throw new ElasticsearchParseException("unexpected token [" + token + "]"); + } + return true; + } + + @Override + public AutodetectResult next() { + return AutodetectResult.PARSER.apply(parser, null); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgement.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgement.java new file mode 100644 index 00000000000..93cac7bd0d1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushAcknowledgement.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.output; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Simple class to parse and store a flush ID. + */ +public class FlushAcknowledgement extends ToXContentToBytes implements Writeable { + /** + * Field Names + */ + public static final ParseField TYPE = new ParseField("flush"); + public static final ParseField ID = new ParseField("id"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new FlushAcknowledgement((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), ID); + } + + private String id; + + public FlushAcknowledgement(String id) { + this.id = id; + } + + public FlushAcknowledgement(StreamInput in) throws IOException { + id = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + } + + public String getId() { + return id; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID.getPreferredName(), id); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + FlushAcknowledgement other = (FlushAcknowledgement) obj; + return Objects.equals(id, other.id); + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java new file mode 100644 index 00000000000..a268669978e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/FlushListener.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.output; + +import java.time.Duration; +import java.util.Iterator; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +class FlushListener { + + final ConcurrentMap awaitingFlushed = new ConcurrentHashMap<>(); + final AtomicBoolean cleared = new AtomicBoolean(false); + + boolean waitForFlush(String flushId, Duration timeout) { + if (cleared.get()) { + return false; + } + + CountDownLatch latch = awaitingFlushed.computeIfAbsent(flushId, (key) -> new CountDownLatch(1)); + try { + return latch.await(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + + void acknowledgeFlush(String flushId) { + // acknowledgeFlush(...) could be called before waitForFlush(...) + // a flush api call writes a flush command to the analytical process and then via a different thread the + // result reader then reads whether the flush has been acked. + CountDownLatch latch = awaitingFlushed.computeIfAbsent(flushId, (key) -> new CountDownLatch(1)); + latch.countDown(); + } + + void clear(String flushId) { + awaitingFlushed.remove(flushId); + } + + void clear() { + if (cleared.compareAndSet(false, true)) { + Iterator> latches = awaitingFlushed.entrySet().iterator(); + while (latches.hasNext()) { + latches.next().getValue().countDown(); + latches.remove(); + } + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/StateProcessor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/StateProcessor.java new file mode 100644 index 00000000000..d3376adaa4d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/StateProcessor.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.output; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Reads the autodetect persisted state and writes the results via the {@linkplain JobResultsPersister} passed in the constructor. + */ +public class StateProcessor extends AbstractComponent { + + private static final int READ_BUF_SIZE = 8192; + private final Client client; + + public StateProcessor(Settings settings, Client client) { + super(settings); + this.client = client; + } + + public void process(String jobId, InputStream in) { + try { + BytesReference bytesRef = null; + int searchFrom = 0; + byte[] readBuf = new byte[READ_BUF_SIZE]; + for (int bytesRead = in.read(readBuf); bytesRead != -1; bytesRead = in.read(readBuf)) { + if (bytesRef == null) { + searchFrom = 0; + bytesRef = new BytesArray(readBuf, 0, bytesRead); + } else { + searchFrom = bytesRef.length(); + bytesRef = new CompositeBytesReference(bytesRef, new BytesArray(readBuf, 0, bytesRead)); + } + bytesRef = splitAndPersist(jobId, bytesRef, searchFrom); + readBuf = new byte[READ_BUF_SIZE]; + } + } catch (IOException e) { + logger.info(new ParameterizedMessage("[{}] Error reading autodetect state output", jobId), e); + } + logger.info("[{}] State output finished", jobId); + } + + /** + * Splits bulk data streamed from the C++ process on '\0' characters. The + * data is expected to be a series of Elasticsearch bulk requests in UTF-8 JSON + * (as would be uploaded to the public REST API) separated by zero bytes ('\0'). + */ + private BytesReference splitAndPersist(String jobId, BytesReference bytesRef, int searchFrom) { + int splitFrom = 0; + while (true) { + int nextZeroByte = findNextZeroByte(bytesRef, searchFrom, splitFrom); + if (nextZeroByte == -1) { + // No more zero bytes in this block + break; + } + // No validation - assume the native process has formatted the state correctly + persist(jobId, bytesRef.slice(splitFrom, nextZeroByte - splitFrom)); + splitFrom = nextZeroByte + 1; + } + if (splitFrom >= bytesRef.length()) { + return null; + } + return bytesRef.slice(splitFrom, bytesRef.length() - splitFrom); + } + + void persist(String jobId, BytesReference bytes) { + try { + logger.trace("[{}] ES API CALL: bulk index", jobId); + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(bytes, null, null); + client.bulk(bulkRequest).actionGet(); + } catch (Exception e) { + logger.error(new ParameterizedMessage("[{}] Error persisting bulk state", jobId), e); + } + } + + private static int findNextZeroByte(BytesReference bytesRef, int searchFrom, int splitFrom) { + for (int i = Math.max(searchFrom, splitFrom); i < bytesRef.length(); ++i) { + if (bytesRef.get(i) == 0) { + return i; + } + } + return -1; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/DataLoadParams.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/DataLoadParams.java new file mode 100644 index 00000000000..a63f140f904 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/DataLoadParams.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.params; + +import org.elasticsearch.xpack.ml.job.config.DataDescription; + +import java.util.Objects; +import java.util.Optional; + +public class DataLoadParams { + private final TimeRange resetTimeRange; + private final Optional dataDescription; + + public DataLoadParams(TimeRange resetTimeRange, Optional dataDescription) { + this.resetTimeRange = Objects.requireNonNull(resetTimeRange); + this.dataDescription = Objects.requireNonNull(dataDescription); + } + + public boolean isResettingBuckets() { + return !getStart().isEmpty(); + } + + public String getStart() { + return resetTimeRange.getStart(); + } + + public String getEnd() { + return resetTimeRange.getEnd(); + } + + public Optional getDataDescription() { + return dataDescription; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/InterimResultsParams.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/InterimResultsParams.java new file mode 100644 index 00000000000..67ce83227a1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/InterimResultsParams.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.params; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.util.Objects; + +public class InterimResultsParams { + private final boolean calcInterim; + private final TimeRange timeRange; + private final Long advanceTimeSeconds; + + private InterimResultsParams(boolean calcInterim, TimeRange timeRange, Long advanceTimeSeconds) { + this.calcInterim = calcInterim; + this.timeRange = Objects.requireNonNull(timeRange); + this.advanceTimeSeconds = advanceTimeSeconds; + } + + public boolean shouldCalculateInterim() { + return calcInterim; + } + + public boolean shouldAdvanceTime() { + return advanceTimeSeconds != null; + } + + public String getStart() { + return timeRange.getStart(); + } + + public String getEnd() { + return timeRange.getEnd(); + } + + public long getAdvanceTime() { + if (!shouldAdvanceTime()) { + throw new IllegalStateException(); + } + return advanceTimeSeconds; + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InterimResultsParams that = (InterimResultsParams) o; + return calcInterim == that.calcInterim && + Objects.equals(timeRange, that.timeRange) && + Objects.equals(advanceTimeSeconds, that.advanceTimeSeconds); + } + + @Override + public int hashCode() { + return Objects.hash(calcInterim, timeRange, advanceTimeSeconds); + } + + public static class Builder { + private boolean calcInterim = false; + private TimeRange timeRange; + private String advanceTime; + + private Builder() { + calcInterim = false; + timeRange = TimeRange.builder().build(); + advanceTime = ""; + } + + public Builder calcInterim(boolean value) { + calcInterim = value; + return this; + } + + public Builder forTimeRange(TimeRange timeRange) { + this.timeRange = timeRange; + return this; + } + + public Builder advanceTime(String timestamp) { + advanceTime = ExceptionsHelper.requireNonNull(timestamp, "advance"); + return this; + } + + public InterimResultsParams build() { + checkValidFlushArgumentsCombination(); + Long advanceTimeSeconds = checkAdvanceTimeParam(); + return new InterimResultsParams(calcInterim, timeRange, advanceTimeSeconds); + } + + private void checkValidFlushArgumentsCombination() { + if (!calcInterim) { + checkFlushParamIsEmpty(TimeRange.START_PARAM, timeRange.getStart()); + checkFlushParamIsEmpty(TimeRange.END_PARAM, timeRange.getEnd()); + } else if (!isValidTimeRange(timeRange)) { + String msg = Messages.getMessage(Messages.REST_INVALID_FLUSH_PARAMS_MISSING, "start"); + throw new IllegalArgumentException(msg); + } + } + + private Long checkAdvanceTimeParam() { + if (advanceTime != null && !advanceTime.isEmpty()) { + return paramToEpochIfValidOrThrow("advance_time", advanceTime) / TimeRange.MILLISECONDS_IN_SECOND; + } + return null; + } + + private long paramToEpochIfValidOrThrow(String paramName, String date) { + if (TimeRange.NOW.equals(date)) { + return System.currentTimeMillis(); + } + long epoch = 0; + if (date.isEmpty() == false) { + epoch = TimeUtils.dateStringToEpoch(date); + if (epoch < 0) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName, date); + throw new ElasticsearchParseException(msg); + } + } + return epoch; + } + + private void checkFlushParamIsEmpty(String paramName, String paramValue) { + if (!paramValue.isEmpty()) { + String msg = Messages.getMessage(Messages.REST_INVALID_FLUSH_PARAMS_UNEXPECTED, paramName); + throw new IllegalArgumentException(msg); + } + } + + private boolean isValidTimeRange(TimeRange timeRange) { + return !timeRange.getStart().isEmpty() || (timeRange.getStart().isEmpty() && timeRange.getEnd().isEmpty()); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java new file mode 100644 index 00000000000..84c5be5fe48 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/params/TimeRange.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.params; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.util.Objects; + +public class TimeRange { + + public static final String START_PARAM = "start"; + public static final String END_PARAM = "end"; + public static final String NOW = "now"; + public static final int MILLISECONDS_IN_SECOND = 1000; + + private final Long start; + private final Long end; + + private TimeRange(Long start, Long end) { + this.start = start; + this.end = end; + } + + public String getStart() { + return start == null ? "" : String.valueOf(start); + } + + public String getEnd() { + return end == null ? "" : String.valueOf(end); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TimeRange timeRange = (TimeRange) o; + return Objects.equals(start, timeRange.start) && + Objects.equals(end, timeRange.end); + } + + @Override + public int hashCode() { + return Objects.hash(start, end); + } + + public static class Builder { + + private String start = ""; + private String end = ""; + + private Builder() { + } + + public Builder startTime(String start) { + this.start = ExceptionsHelper.requireNonNull(start, "start"); + return this; + } + + public Builder endTime(String end) { + this.end = ExceptionsHelper.requireNonNull(end, "end"); + return this; + } + + /** + * Create a new TimeRange instance after validating the start and end params. + * Throws {@link ElasticsearchStatusException} if the validation fails + * @return The time range + */ + public TimeRange build() { + return createTimeRange(start, end); + } + + private TimeRange createTimeRange(String start, String end) { + Long epochStart = null; + Long epochEnd = null; + if (!start.isEmpty()) { + epochStart = paramToEpochIfValidOrThrow(START_PARAM, start) / MILLISECONDS_IN_SECOND; + epochEnd = paramToEpochIfValidOrThrow(END_PARAM, end) / MILLISECONDS_IN_SECOND; + if (end.isEmpty() || epochEnd.equals(epochStart)) { + epochEnd = epochStart + 1; + } + if (epochEnd < epochStart) { + String msg = Messages.getMessage(Messages.REST_START_AFTER_END, end, start); + throw new IllegalArgumentException(msg); + } + } else { + if (!end.isEmpty()) { + epochEnd = paramToEpochIfValidOrThrow(END_PARAM, end) / MILLISECONDS_IN_SECOND; + } + } + return new TimeRange(epochStart, epochEnd); + } + + /** + * Returns epoch milli seconds + */ + private long paramToEpochIfValidOrThrow(String paramName, String date) { + if (NOW.equals(date)) { + return System.currentTimeMillis(); + } + long epoch = 0; + if (date.isEmpty() == false) { + epoch = TimeUtils.dateStringToEpoch(date); + if (epoch < 0) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName, date); + throw new ElasticsearchParseException(msg); + } + } + + return epoch; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java new file mode 100644 index 00000000000..7b6430ab880 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/CategorizerState.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.state; + + +/** + * The categorizer state does not need to be loaded on the Java side. + * However, the Java process DOES set up a mapping on the Elasticsearch + * index to tell Elasticsearch not to analyse the categorizer state documents + * in any way. + */ +public class CategorizerState { + /** + * The type of this class used when persisting the data + */ + public static final String TYPE = "categorizer_state"; + + public static final String categorizerStateDocId(String jobId, int docNum) { + return jobId + "_" + docNum; + } + + private CategorizerState() { + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/DataCounts.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/DataCounts.java new file mode 100644 index 00000000000..784779cb30d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/DataCounts.java @@ -0,0 +1,419 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.state; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Job processed record counts. + *

+ * The getInput... methods return the actual number of + * fields/records sent the the API including invalid records. + * The getProcessed... methods are the number sent to the + * Engine. + *

+ * The inputRecordCount field is calculated so it + * should not be set in deserialisation but it should be serialised + * so the field is visible. + */ + +public class DataCounts extends ToXContentToBytes implements Writeable { + + private static final String DOCUMENT_SUFFIX = "-data-counts"; + public static final String PROCESSED_RECORD_COUNT_STR = "processed_record_count"; + public static final String PROCESSED_FIELD_COUNT_STR = "processed_field_count"; + public static final String INPUT_BYTES_STR = "input_bytes"; + public static final String INPUT_RECORD_COUNT_STR = "input_record_count"; + public static final String INPUT_FIELD_COUNT_STR = "input_field_count"; + public static final String INVALID_DATE_COUNT_STR = "invalid_date_count"; + public static final String MISSING_FIELD_COUNT_STR = "missing_field_count"; + public static final String OUT_OF_ORDER_TIME_COUNT_STR = "out_of_order_timestamp_count"; + public static final String EARLIEST_RECORD_TIME_STR = "earliest_record_timestamp"; + public static final String LATEST_RECORD_TIME_STR = "latest_record_timestamp"; + + public static final ParseField PROCESSED_RECORD_COUNT = new ParseField(PROCESSED_RECORD_COUNT_STR); + public static final ParseField PROCESSED_FIELD_COUNT = new ParseField(PROCESSED_FIELD_COUNT_STR); + public static final ParseField INPUT_BYTES = new ParseField(INPUT_BYTES_STR); + public static final ParseField INPUT_RECORD_COUNT = new ParseField(INPUT_RECORD_COUNT_STR); + public static final ParseField INPUT_FIELD_COUNT = new ParseField(INPUT_FIELD_COUNT_STR); + public static final ParseField INVALID_DATE_COUNT = new ParseField(INVALID_DATE_COUNT_STR); + public static final ParseField MISSING_FIELD_COUNT = new ParseField(MISSING_FIELD_COUNT_STR); + public static final ParseField OUT_OF_ORDER_TIME_COUNT = new ParseField(OUT_OF_ORDER_TIME_COUNT_STR); + public static final ParseField EARLIEST_RECORD_TIME = new ParseField(EARLIEST_RECORD_TIME_STR); + public static final ParseField LATEST_RECORD_TIME = new ParseField(LATEST_RECORD_TIME_STR); + + public static final ParseField TYPE = new ParseField("data_counts"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("data_counts", a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3], + (long) a[4], (long) a[5], (long) a[6], (long) a[7], (Date) a[8], (Date) a[9])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_RECORD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), PROCESSED_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_BYTES); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INPUT_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), INVALID_DATE_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), MISSING_FIELD_COUNT); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), OUT_OF_ORDER_TIME_COUNT); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + EARLIEST_RECORD_TIME.getPreferredName() + "]"); + }, EARLIEST_RECORD_TIME, ValueType.VALUE); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LATEST_RECORD_TIME.getPreferredName() + "]"); + }, LATEST_RECORD_TIME, ValueType.VALUE); + PARSER.declareLong((t, u) -> {;}, INPUT_RECORD_COUNT); + } + + public static String documentId(String jobId) { + return jobId + DOCUMENT_SUFFIX; + } + + private final String jobId; + private long processedRecordCount; + private long processedFieldCount; + private long inputBytes; + private long inputFieldCount; + private long invalidDateCount; + private long missingFieldCount; + private long outOfOrderTimeStampCount; + // NORELEASE: Use Jodatime instead + private Date earliestRecordTimeStamp; + private Date latestRecordTimeStamp; + + public DataCounts(String jobId, long processedRecordCount, long processedFieldCount, long inputBytes, + long inputFieldCount, long invalidDateCount, long missingFieldCount, long outOfOrderTimeStampCount, + Date earliestRecordTimeStamp, Date latestRecordTimeStamp) { + this.jobId = jobId; + this.processedRecordCount = processedRecordCount; + this.processedFieldCount = processedFieldCount; + this.inputBytes = inputBytes; + this.inputFieldCount = inputFieldCount; + this.invalidDateCount = invalidDateCount; + this.missingFieldCount = missingFieldCount; + this.outOfOrderTimeStampCount = outOfOrderTimeStampCount; + this.latestRecordTimeStamp = latestRecordTimeStamp; + this.earliestRecordTimeStamp = earliestRecordTimeStamp; + } + + public DataCounts(String jobId) { + this.jobId = jobId; + } + + public DataCounts(DataCounts lhs) { + jobId = lhs.jobId; + processedRecordCount = lhs.processedRecordCount; + processedFieldCount = lhs.processedFieldCount; + inputBytes = lhs.inputBytes; + inputFieldCount = lhs.inputFieldCount; + invalidDateCount = lhs.invalidDateCount; + missingFieldCount = lhs.missingFieldCount; + outOfOrderTimeStampCount = lhs.outOfOrderTimeStampCount; + latestRecordTimeStamp = lhs.latestRecordTimeStamp; + earliestRecordTimeStamp = lhs.earliestRecordTimeStamp; + } + + public DataCounts(StreamInput in) throws IOException { + jobId = in.readString(); + processedRecordCount = in.readVLong(); + processedFieldCount = in.readVLong(); + inputBytes = in.readVLong(); + inputFieldCount = in.readVLong(); + invalidDateCount = in.readVLong(); + missingFieldCount = in.readVLong(); + outOfOrderTimeStampCount = in.readVLong(); + if (in.readBoolean()) { + latestRecordTimeStamp = new Date(in.readVLong()); + } + if (in.readBoolean()) { + earliestRecordTimeStamp = new Date(in.readVLong()); + } + in.readVLong(); // throw away inputRecordCount + } + + public String getJobid() { + return jobId; + } + + /** + * Number of records processed by this job. + * This value is the number of records sent passed on to + * the engine i.e. {@linkplain #getInputRecordCount()} minus + * records with bad dates or out of order + * + * @return Number of records processed by this job {@code long} + */ + public long getProcessedRecordCount() { + return processedRecordCount; + } + + public void incrementProcessedRecordCount(long additional) { + processedRecordCount += additional; + } + + /** + * Number of data points (processed record count * the number + * of analysed fields) processed by this job. This count does + * not include the time field. + * + * @return Number of data points processed by this job {@code long} + */ + public long getProcessedFieldCount() { + return processedFieldCount; + } + + public void calcProcessedFieldCount(long analysisFieldsPerRecord) { + processedFieldCount = + (processedRecordCount * analysisFieldsPerRecord) + - missingFieldCount; + + // processedFieldCount could be a -ve value if no + // records have been written in which case it should be 0 + processedFieldCount = (processedFieldCount < 0) ? 0 : processedFieldCount; + } + + /** + * Total number of input records read. + * This = processed record count + date parse error records count + * + out of order record count. + *

+ * Records with missing fields are counted as they are still written. + * + * @return Total number of input records read {@code long} + */ + public long getInputRecordCount() { + return processedRecordCount + outOfOrderTimeStampCount + + invalidDateCount; + } + + /** + * The total number of bytes sent to this job. + * This value includes the bytes from any records + * that have been discarded for any reason + * e.g. because the date cannot be read + * + * @return Volume in bytes + */ + public long getInputBytes() { + return inputBytes; + } + + public void incrementInputBytes(long additional) { + inputBytes += additional; + } + + /** + * The total number of fields sent to the job + * including fields that aren't analysed. + * + * @return The total number of fields sent to the job + */ + public long getInputFieldCount() { + return inputFieldCount; + } + + public void incrementInputFieldCount(long additional) { + inputFieldCount += additional; + } + + /** + * The number of records with an invalid date field that could + * not be parsed or converted to epoch time. + * + * @return The number of records with an invalid date field + */ + public long getInvalidDateCount() { + return invalidDateCount; + } + + public void incrementInvalidDateCount(long additional) { + invalidDateCount += additional; + } + + + /** + * The number of missing fields that had been + * configured for analysis. + * + * @return The number of missing fields + */ + public long getMissingFieldCount() { + return missingFieldCount; + } + + public void incrementMissingFieldCount(long additional) { + missingFieldCount += additional; + } + + /** + * The number of records with a timestamp that is + * before the time of the latest record. Records should + * be in ascending chronological order + * + * @return The number of records with a timestamp that is before the time of the latest record + */ + public long getOutOfOrderTimeStampCount() { + return outOfOrderTimeStampCount; + } + + public void incrementOutOfOrderTimeStampCount(long additional) { + outOfOrderTimeStampCount += additional; + } + + /** + * The time of the first record seen. + * + * @return The first record time + */ + public Date getEarliestRecordTimeStamp() { + return earliestRecordTimeStamp; + } + + /** + * If {@code earliestRecordTimeStamp} has not been set (i.e. is {@code null}) + * then set it to {@code timeStamp} + * + * @param timeStamp Candidate time + * @throws IllegalStateException if {@code earliestRecordTimeStamp} is already set + */ + public void setEarliestRecordTimeStamp(Date timeStamp) { + if (earliestRecordTimeStamp != null) { + throw new IllegalStateException("earliestRecordTimeStamp can only be set once"); + } + earliestRecordTimeStamp = timeStamp; + } + + + /** + * The time of the latest record seen. + * + * @return Latest record time + */ + public Date getLatestRecordTimeStamp() { + return latestRecordTimeStamp; + } + + public void setLatestRecordTimeStamp(Date latestRecordTimeStamp) { + this.latestRecordTimeStamp = latestRecordTimeStamp; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeVLong(processedRecordCount); + out.writeVLong(processedFieldCount); + out.writeVLong(inputBytes); + out.writeVLong(inputFieldCount); + out.writeVLong(invalidDateCount); + out.writeVLong(missingFieldCount); + out.writeVLong(outOfOrderTimeStampCount); + if (latestRecordTimeStamp != null) { + out.writeBoolean(true); + out.writeVLong(latestRecordTimeStamp.getTime()); + } else { + out.writeBoolean(false); + } + if (earliestRecordTimeStamp != null) { + out.writeBoolean(true); + out.writeVLong(earliestRecordTimeStamp.getTime()); + } else { + out.writeBoolean(false); + } + out.writeVLong(getInputRecordCount()); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder, params); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(PROCESSED_RECORD_COUNT.getPreferredName(), processedRecordCount); + builder.field(PROCESSED_FIELD_COUNT.getPreferredName(), processedFieldCount); + builder.field(INPUT_BYTES.getPreferredName(), inputBytes); + builder.field(INPUT_FIELD_COUNT.getPreferredName(), inputFieldCount); + builder.field(INVALID_DATE_COUNT.getPreferredName(), invalidDateCount); + builder.field(MISSING_FIELD_COUNT.getPreferredName(), missingFieldCount); + builder.field(OUT_OF_ORDER_TIME_COUNT.getPreferredName(), outOfOrderTimeStampCount); + if (earliestRecordTimeStamp != null) { + builder.dateField(EARLIEST_RECORD_TIME.getPreferredName(), EARLIEST_RECORD_TIME.getPreferredName() + "_string", + earliestRecordTimeStamp.getTime()); + } + if (latestRecordTimeStamp != null) { + builder.dateField(LATEST_RECORD_TIME.getPreferredName(), LATEST_RECORD_TIME.getPreferredName() + "_string", + latestRecordTimeStamp.getTime()); + } + builder.field(INPUT_RECORD_COUNT.getPreferredName(), getInputRecordCount()); + + return builder; + } + + /** + * Equality test + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof DataCounts == false) { + return false; + } + + DataCounts that = (DataCounts) other; + + return Objects.equals(this.jobId, that.jobId) && + this.processedRecordCount == that.processedRecordCount && + this.processedFieldCount == that.processedFieldCount && + this.inputBytes == that.inputBytes && + this.inputFieldCount == that.inputFieldCount && + this.invalidDateCount == that.invalidDateCount && + this.missingFieldCount == that.missingFieldCount && + this.outOfOrderTimeStampCount == that.outOfOrderTimeStampCount && + Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) && + Objects.equals(this.earliestRecordTimeStamp, that.earliestRecordTimeStamp); + + } + + @Override + public int hashCode() { + return Objects.hash(jobId, processedRecordCount, processedFieldCount, + inputBytes, inputFieldCount, invalidDateCount, missingFieldCount, + outOfOrderTimeStampCount, latestRecordTimeStamp, earliestRecordTimeStamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java new file mode 100644 index 00000000000..6ce95838814 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java @@ -0,0 +1,324 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.state; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.results.Result; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Locale; +import java.util.Objects; + +/** + * Provide access to the C++ model memory usage numbers for the Java process. + */ +public class ModelSizeStats extends ToXContentToBytes implements Writeable { + + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "model_size_stats"; + public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE); + + /** + * Field Names + */ + public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes"); + public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count"); + public static final ParseField TOTAL_OVER_FIELD_COUNT_FIELD = new ParseField("total_over_field_count"); + public static final ParseField TOTAL_PARTITION_FIELD_COUNT_FIELD = new ParseField("total_partition_field_count"); + public static final ParseField BUCKET_ALLOCATION_FAILURES_COUNT_FIELD = new ParseField("bucket_allocation_failures_count"); + public static final ParseField MEMORY_STATUS_FIELD = new ParseField("memory_status"); + public static final ParseField LOG_TIME_FIELD = new ParseField("log_time"); + public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + RESULT_TYPE_FIELD.getPreferredName(), a -> new Builder((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareString((modelSizeStat, s) -> {}, Result.RESULT_TYPE); + PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); + PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalByFieldCount, TOTAL_BY_FIELD_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalOverFieldCount, TOTAL_OVER_FIELD_COUNT_FIELD); + PARSER.declareLong(Builder::setTotalPartitionFieldCount, TOTAL_PARTITION_FIELD_COUNT_FIELD); + PARSER.declareField(Builder::setLogTime, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LOG_TIME_FIELD.getPreferredName() + "]"); + }, LOG_TIME_FIELD, ValueType.VALUE); + PARSER.declareField(Builder::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP_FIELD.getPreferredName() + "]"); + }, TIMESTAMP_FIELD, ValueType.VALUE); + PARSER.declareField(Builder::setMemoryStatus, p -> MemoryStatus.fromString(p.text()), MEMORY_STATUS_FIELD, ValueType.STRING); + } + + public static String documentId(String jobId) { + return jobId + "-" + RESULT_TYPE_VALUE; + } + + /** + * The status of the memory monitored by the ResourceMonitor. OK is default, + * SOFT_LIMIT means that the models have done some aggressive pruning to + * keep the memory below the limit, and HARD_LIMIT means that samples have + * been dropped + */ + public enum MemoryStatus implements Writeable { + OK, SOFT_LIMIT, HARD_LIMIT; + + public static MemoryStatus fromString(String statusName) { + return valueOf(statusName.trim().toUpperCase(Locale.ROOT)); + } + + public static MemoryStatus readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown MemoryStatus ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + private final String jobId; + private final long modelBytes; + private final long totalByFieldCount; + private final long totalOverFieldCount; + private final long totalPartitionFieldCount; + private final long bucketAllocationFailuresCount; + private final MemoryStatus memoryStatus; + private final Date timestamp; + private final Date logTime; + + private ModelSizeStats(String jobId, String id, long modelBytes, long totalByFieldCount, long totalOverFieldCount, + long totalPartitionFieldCount, long bucketAllocationFailuresCount, MemoryStatus memoryStatus, + Date timestamp, Date logTime) { + this.jobId = jobId; + this.modelBytes = modelBytes; + this.totalByFieldCount = totalByFieldCount; + this.totalOverFieldCount = totalOverFieldCount; + this.totalPartitionFieldCount = totalPartitionFieldCount; + this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; + this.memoryStatus = memoryStatus; + this.timestamp = timestamp; + this.logTime = logTime; + } + + public ModelSizeStats(StreamInput in) throws IOException { + jobId = in.readString(); + modelBytes = in.readVLong(); + totalByFieldCount = in.readVLong(); + totalOverFieldCount = in.readVLong(); + totalPartitionFieldCount = in.readVLong(); + bucketAllocationFailuresCount = in.readVLong(); + memoryStatus = MemoryStatus.readFromStream(in); + logTime = new Date(in.readLong()); + timestamp = in.readBoolean() ? new Date(in.readLong()) : null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeVLong(modelBytes); + out.writeVLong(totalByFieldCount); + out.writeVLong(totalOverFieldCount); + out.writeVLong(totalPartitionFieldCount); + out.writeVLong(bucketAllocationFailuresCount); + memoryStatus.writeTo(out); + out.writeLong(logTime.getTime()); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + doXContentBody(builder); + builder.endObject(); + return builder; + } + + public XContentBuilder doXContentBody(XContentBuilder builder) throws IOException { + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes); + builder.field(TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), totalByFieldCount); + builder.field(TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), totalOverFieldCount); + builder.field(TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), totalPartitionFieldCount); + builder.field(BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName(), bucketAllocationFailuresCount); + builder.field(MEMORY_STATUS_FIELD.getPreferredName(), memoryStatus); + builder.field(LOG_TIME_FIELD.getPreferredName(), logTime.getTime()); + if (timestamp != null) { + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestamp.getTime()); + } + + return builder; + } + + public String getJobId() { + return jobId; + } + + public long getModelBytes() { + return modelBytes; + } + + public long getTotalByFieldCount() { + return totalByFieldCount; + } + + public long getTotalPartitionFieldCount() { + return totalPartitionFieldCount; + } + + public long getTotalOverFieldCount() { + return totalOverFieldCount; + } + + public long getBucketAllocationFailuresCount() { + return bucketAllocationFailuresCount; + } + + public MemoryStatus getMemoryStatus() { + return memoryStatus; + } + + public Date getTimestamp() { + return timestamp; + } + + public Date getLogTime() { + return logTime; + } + + @Override + public int hashCode() { + // this.id excluded here as it is generated by the datastore + return Objects.hash(jobId, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, + this.bucketAllocationFailuresCount, memoryStatus, timestamp, logTime); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof ModelSizeStats == false) { + return false; + } + + ModelSizeStats that = (ModelSizeStats) other; + + return this.modelBytes == that.modelBytes && this.totalByFieldCount == that.totalByFieldCount + && this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount + && this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount + && Objects.equals(this.memoryStatus, that.memoryStatus) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.logTime, that.logTime) + && Objects.equals(this.jobId, that.jobId); + } + + // NORELEASE This will not be needed once we are able to parse ModelSizeStats all at once. + public static class Builder { + + private final String jobId; + private String id; + private long modelBytes; + private long totalByFieldCount; + private long totalOverFieldCount; + private long totalPartitionFieldCount; + private long bucketAllocationFailuresCount; + private MemoryStatus memoryStatus; + private Date timestamp; + private Date logTime; + + public Builder(String jobId) { + this.jobId = jobId; + id = RESULT_TYPE_FIELD.getPreferredName(); + memoryStatus = MemoryStatus.OK; + logTime = new Date(); + } + + public void setId(String id) { + this.id = Objects.requireNonNull(id); + } + + public void setModelBytes(long modelBytes) { + this.modelBytes = modelBytes; + } + + public void setTotalByFieldCount(long totalByFieldCount) { + this.totalByFieldCount = totalByFieldCount; + } + + public void setTotalPartitionFieldCount(long totalPartitionFieldCount) { + this.totalPartitionFieldCount = totalPartitionFieldCount; + } + + public void setTotalOverFieldCount(long totalOverFieldCount) { + this.totalOverFieldCount = totalOverFieldCount; + } + + public void setBucketAllocationFailuresCount(long bucketAllocationFailuresCount) { + this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; + } + + public void setMemoryStatus(MemoryStatus memoryStatus) { + Objects.requireNonNull(memoryStatus, "[" + MEMORY_STATUS_FIELD.getPreferredName() + "] must not be null"); + this.memoryStatus = memoryStatus; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public void setLogTime(Date logTime) { + this.logTime = logTime; + } + + public ModelSizeStats build() { + return new ModelSizeStats(jobId, id, modelBytes, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, + bucketAllocationFailuresCount, memoryStatus, timestamp, logTime); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java new file mode 100644 index 00000000000..707b4329627 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java @@ -0,0 +1,302 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.state; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + + +/** + * ModelSnapshot Result POJO + */ +public class ModelSnapshot extends ToXContentToBytes implements Writeable { + /** + * Field Names + */ + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField DESCRIPTION = new ParseField("description"); + public static final ParseField RESTORE_PRIORITY = new ParseField("restore_priority"); + public static final ParseField SNAPSHOT_ID = new ParseField("snapshot_id"); + public static final ParseField SNAPSHOT_DOC_COUNT = new ParseField("snapshot_doc_count"); + public static final ParseField LATEST_RECORD_TIME = new ParseField("latest_record_time_stamp"); + public static final ParseField LATEST_RESULT_TIME = new ParseField("latest_result_time_stamp"); + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("model_snapshots"); + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("model_snapshot"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new ModelSnapshot((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareField(ModelSnapshot::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareString(ModelSnapshot::setDescription, DESCRIPTION); + PARSER.declareLong(ModelSnapshot::setRestorePriority, RESTORE_PRIORITY); + PARSER.declareString(ModelSnapshot::setSnapshotId, SNAPSHOT_ID); + PARSER.declareInt(ModelSnapshot::setSnapshotDocCount, SNAPSHOT_DOC_COUNT); + PARSER.declareObject(ModelSnapshot::setModelSizeStats, ModelSizeStats.PARSER, ModelSizeStats.RESULT_TYPE_FIELD); + PARSER.declareField(ModelSnapshot::setLatestRecordTimeStamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LATEST_RECORD_TIME.getPreferredName() + "]"); + }, LATEST_RECORD_TIME, ValueType.VALUE); + PARSER.declareField(ModelSnapshot::setLatestResultTimeStamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + LATEST_RESULT_TIME.getPreferredName() + "]"); + }, LATEST_RESULT_TIME, ValueType.VALUE); + PARSER.declareObject(ModelSnapshot::setQuantiles, Quantiles.PARSER, Quantiles.TYPE); + } + + private final String jobId; + private Date timestamp; + private String description; + private long restorePriority; + private String snapshotId; + private int snapshotDocCount; + private ModelSizeStats modelSizeStats; + private Date latestRecordTimeStamp; + private Date latestResultTimeStamp; + private Quantiles quantiles; + + public ModelSnapshot(String jobId) { + this.jobId = jobId; + } + + public ModelSnapshot(StreamInput in) throws IOException { + jobId = in.readString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + description = in.readOptionalString(); + restorePriority = in.readLong(); + snapshotId = in.readOptionalString(); + snapshotDocCount = in.readInt(); + if (in.readBoolean()) { + modelSizeStats = new ModelSizeStats(in); + } + if (in.readBoolean()) { + latestRecordTimeStamp = new Date(in.readLong()); + } + if (in.readBoolean()) { + latestResultTimeStamp = new Date(in.readLong()); + } + if (in.readBoolean()) { + quantiles = new Quantiles(in); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeOptionalString(description); + out.writeLong(restorePriority); + out.writeOptionalString(snapshotId); + out.writeInt(snapshotDocCount); + boolean hasModelSizeStats = modelSizeStats != null; + out.writeBoolean(hasModelSizeStats); + if (hasModelSizeStats) { + modelSizeStats.writeTo(out); + } + boolean hasLatestRecordTimeStamp = latestRecordTimeStamp != null; + out.writeBoolean(hasLatestRecordTimeStamp); + if (hasLatestRecordTimeStamp) { + out.writeLong(latestRecordTimeStamp.getTime()); + } + boolean hasLatestResultTimeStamp = latestResultTimeStamp != null; + out.writeBoolean(hasLatestResultTimeStamp); + if (hasLatestResultTimeStamp) { + out.writeLong(latestResultTimeStamp.getTime()); + } + boolean hasQuantiles = quantiles != null; + out.writeBoolean(hasQuantiles); + if (hasQuantiles) { + quantiles.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (description != null) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + builder.field(RESTORE_PRIORITY.getPreferredName(), restorePriority); + if (snapshotId != null) { + builder.field(SNAPSHOT_ID.getPreferredName(), snapshotId); + } + builder.field(SNAPSHOT_DOC_COUNT.getPreferredName(), snapshotDocCount); + if (modelSizeStats != null) { + builder.field(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName(), modelSizeStats); + } + if (latestRecordTimeStamp != null) { + builder.field(LATEST_RECORD_TIME.getPreferredName(), latestRecordTimeStamp.getTime()); + } + if (latestResultTimeStamp != null) { + builder.field(LATEST_RESULT_TIME.getPreferredName(), latestResultTimeStamp.getTime()); + } + if (quantiles != null) { + builder.field(Quantiles.TYPE.getPreferredName(), quantiles); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public String documentId() { + return jobId + "-" + snapshotId; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public long getRestorePriority() { + return restorePriority; + } + + public void setRestorePriority(long restorePriority) { + this.restorePriority = restorePriority; + } + + public String getSnapshotId() { + return snapshotId; + } + + public void setSnapshotId(String snapshotId) { + this.snapshotId = snapshotId; + } + + public int getSnapshotDocCount() { + return snapshotDocCount; + } + + public void setSnapshotDocCount(int snapshotDocCount) { + this.snapshotDocCount = snapshotDocCount; + } + + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + public void setModelSizeStats(ModelSizeStats.Builder modelSizeStats) { + this.modelSizeStats = modelSizeStats.build(); + } + + public Quantiles getQuantiles() { + return quantiles; + } + + public void setQuantiles(Quantiles q) { + quantiles = q; + } + + public Date getLatestRecordTimeStamp() { + return latestRecordTimeStamp; + } + + public void setLatestRecordTimeStamp(Date latestRecordTimeStamp) { + this.latestRecordTimeStamp = latestRecordTimeStamp; + } + + public Date getLatestResultTimeStamp() { + return latestResultTimeStamp; + } + + public void setLatestResultTimeStamp(Date latestResultTimeStamp) { + this.latestResultTimeStamp = latestResultTimeStamp; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, description, restorePriority, snapshotId, quantiles, + snapshotDocCount, modelSizeStats, latestRecordTimeStamp, latestResultTimeStamp); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof ModelSnapshot == false) { + return false; + } + + ModelSnapshot that = (ModelSnapshot) other; + + return Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.description, that.description) + && this.restorePriority == that.restorePriority + && Objects.equals(this.snapshotId, that.snapshotId) + && this.snapshotDocCount == that.snapshotDocCount + && Objects.equals(this.modelSizeStats, that.modelSizeStats) + && Objects.equals(this.quantiles, that.quantiles) + && Objects.equals(this.latestRecordTimeStamp, that.latestRecordTimeStamp) + && Objects.equals(this.latestResultTimeStamp, that.latestResultTimeStamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelState.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelState.java new file mode 100644 index 00000000000..576693cf79a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelState.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.state; + + +import org.elasticsearch.common.ParseField; + +/** + * The serialised models can get very large and only the C++ code + * understands how to decode them, hence there is no reason to load + * them into the Java process. + * However, the Java process DOES set up a mapping on the Elasticsearch + * index to tell Elasticsearch not to analyse the model state documents + * in any way. (Otherwise Elasticsearch would go into a spin trying to + * make sense of such large JSON documents.) + */ +public class ModelState { + /** + * The type of this class used when persisting the data + */ + public static final ParseField TYPE = new ParseField("model_state"); + + private ModelState() { + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/Quantiles.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/Quantiles.java new file mode 100644 index 00000000000..ae3b9cf2efe --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/Quantiles.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.state; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Quantiles Result POJO + */ +public class Quantiles extends ToXContentToBytes implements Writeable { + + /** + * Field Names + */ + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField QUANTILE_STATE = new ParseField("quantile_state"); + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("quantiles"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new Quantiles((String) a[0], (Date) a[1], (String) a[2])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> new Date(p.longValue()), TIMESTAMP, ValueType.LONG); + PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); + } + + public static String documentId(String jobId) { + return jobId + "-" + TYPE.getPreferredName(); + } + + private final String jobId; + private final Date timestamp; + private final String quantileState; + + public Quantiles(String jobId, Date timestamp, String quantileState) { + this.jobId = jobId; + this.timestamp = Objects.requireNonNull(timestamp); + this.quantileState = Objects.requireNonNull(quantileState); + } + + public Quantiles(StreamInput in) throws IOException { + jobId = in.readString(); + timestamp = new Date(in.readLong()); + quantileState = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(timestamp.getTime()); + out.writeOptionalString(quantileState); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + if (quantileState != null) { + builder.field(QUANTILE_STATE.getPreferredName(), quantileState); + } + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public Date getTimestamp() { + return timestamp; + } + + public String getQuantileState() { + return quantileState; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, quantileState); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Quantiles == false) { + return false; + } + + Quantiles that = (Quantiles) other; + + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.quantileState, that.quantileState); + + + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java new file mode 100644 index 00000000000..9647d4353f7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractDataToProcessWriter.java @@ -0,0 +1,293 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +public abstract class AbstractDataToProcessWriter implements DataToProcessWriter { + + private static final int TIME_FIELD_OUT_INDEX = 0; + private static final long MS_IN_SECOND = 1000; + + private final boolean includeControlField; + + protected final AutodetectProcess autodetectProcess; + protected final DataDescription dataDescription; + protected final AnalysisConfig analysisConfig; + protected final DataCountsReporter dataCountsReporter; + + private final Logger logger; + private final DateTransformer dateTransformer; + + protected Map inFieldIndexes; + protected List inputOutputMap; + + // epoch in seconds + private long latestEpochMs; + private long latestEpochMsThisUpload; + + protected AbstractDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + DataCountsReporter dataCountsReporter, Logger logger) { + this.includeControlField = includeControlField; + this.autodetectProcess = Objects.requireNonNull(autodetectProcess); + this.dataDescription = Objects.requireNonNull(dataDescription); + this.analysisConfig = Objects.requireNonNull(analysisConfig); + this.dataCountsReporter = Objects.requireNonNull(dataCountsReporter); + this.logger = Objects.requireNonNull(logger); + + Date date = dataCountsReporter.getLatestRecordTime(); + latestEpochMsThisUpload = 0; + latestEpochMs = 0; + if (date != null) { + latestEpochMs = date.getTime(); + } + + boolean isDateFormatString = dataDescription.isTransformTime() && !dataDescription.isEpochMs(); + if (isDateFormatString) { + dateTransformer = new DateFormatDateTransformer(dataDescription.getTimeFormat()); + } else { + dateTransformer = new DoubleDateTransformer(dataDescription.isEpochMs()); + } + } + + /** + * Set up the field index mappings. + * This must be called before {@linkplain DataToProcessWriter#write(java.io.InputStream)}. + *

+ * Finds the required input indexes in the header + * and sets the mappings to the corresponding output indexes. + */ + void buildFieldIndexMapping(String[] header) throws IOException { + Collection inputFields = inputFields(); + inFieldIndexes = inputFieldIndexes(header, inputFields); + checkForMissingFields(inputFields, inFieldIndexes, header); + + inputOutputMap = createInputOutputMap(inFieldIndexes); + dataCountsReporter.setAnalysedFieldsPerRecord(analysisConfig.analysisFields().size()); + } + + /** + * Write the header. + * The header is created from the list of analysis input fields, the time field and the control field. + */ + @Override + public void writeHeader() throws IOException { + Map outFieldIndexes = outputFieldIndexes(); + + // header is all the analysis input fields + the time field + control field + int numFields = outFieldIndexes.size(); + String[] record = new String[numFields]; + + Iterator> itr = outFieldIndexes.entrySet().iterator(); + while (itr.hasNext()) { + Map.Entry entry = itr.next(); + record[entry.getValue()] = entry.getKey(); + } + + // Write the header + autodetectProcess.writeRecord(record); + } + + /** + * Transform the input data and write to length encoded writer.
+ *

+ * Fields that aren't transformed i.e. those in inputOutputMap must be + * copied from input to output before this function is called. + *

+ * First all the transforms whose outputs the Date transform relies + * on are executed then the date transform then the remaining transforms. + * + * @param record The record that will be written to the length encoded writer after the time has been transformed. + * This should be the same size as the number of output (analysis fields) i.e. + * the size of the map returned by {@linkplain #outputFieldIndexes()} + * @param numberOfFieldsRead The total number read not just those included in the analysis + */ + protected boolean transformTimeAndWrite(String[] record, long numberOfFieldsRead) throws IOException { + long epochMs; + try { + epochMs = dateTransformer.transform(record[TIME_FIELD_OUT_INDEX]); + } catch (CannotParseTimestampException e) { + dataCountsReporter.reportDateParseError(numberOfFieldsRead); + logger.error(e.getMessage()); + return false; + } + + // Records have epoch seconds timestamp so compare for out of order in seconds + if (epochMs / MS_IN_SECOND < latestEpochMs / MS_IN_SECOND - analysisConfig.getLatency()) { + // out of order + dataCountsReporter.reportOutOfOrderRecord(inFieldIndexes.size()); + + if (epochMs > latestEpochMsThisUpload) { + // record this timestamp even if the record won't be processed + latestEpochMsThisUpload = epochMs; + dataCountsReporter.reportLatestTimeIncrementalStats(latestEpochMsThisUpload); + } + return false; + } + + record[TIME_FIELD_OUT_INDEX] = Long.toString(epochMs / MS_IN_SECOND); + + latestEpochMs = Math.max(latestEpochMs, epochMs); + latestEpochMsThisUpload = latestEpochMs; + + autodetectProcess.writeRecord(record); + dataCountsReporter.reportRecordWritten(numberOfFieldsRead, latestEpochMs); + + return true; + } + + @Override + public void flush() throws IOException { + autodetectProcess.flushStream(); + } + + /** + * Get all the expected input fields i.e. all the fields we + * must see in the csv header + */ + final Collection inputFields() { + Set requiredFields = new HashSet<>(analysisConfig.analysisFields()); + requiredFields.add(dataDescription.getTimeField()); + + return requiredFields; + } + + /** + * Find the indexes of the input fields from the header + */ + protected final Map inputFieldIndexes(String[] header, Collection inputFields) { + List headerList = Arrays.asList(header); // TODO header could be empty + + Map fieldIndexes = new HashMap(); + + for (String field : inputFields) { + int index = headerList.indexOf(field); + if (index >= 0) { + fieldIndexes.put(field, index); + } + } + + return fieldIndexes; + } + + Map getInputFieldIndexes() { + return inFieldIndexes; + } + + /** + * Create indexes of the output fields. + * This is the time field and all the fields configured for analysis + * and the control field. + * Time is the first field and the last is the control field + */ + protected final Map outputFieldIndexes() { + Map fieldIndexes = new HashMap(); + + // time field + fieldIndexes.put(dataDescription.getTimeField(), TIME_FIELD_OUT_INDEX); + + int index = TIME_FIELD_OUT_INDEX + 1; + List analysisFields = analysisConfig.analysisFields(); + Collections.sort(analysisFields); + + for (String field : analysisConfig.analysisFields()) { + fieldIndexes.put(field, index++); + } + + // control field + if (includeControlField) { + fieldIndexes.put(LengthEncodedWriter.CONTROL_FIELD_NAME, index); + } + + return fieldIndexes; + } + + /** + * The number of fields used in the analysis field, + * the time field and (sometimes) the control field + */ + protected int outputFieldCount() { + return analysisConfig.analysisFields().size() + (includeControlField ? 2 : 1); + } + + protected Map getOutputFieldIndexes() { + return outputFieldIndexes(); + } + + /** + * Create a map of input index to output index. This does not include the time or control fields. + * + * @param inFieldIndexes Map of field name to index in the input array + */ + private List createInputOutputMap(Map inFieldIndexes) { + List inputOutputMap = new ArrayList<>(); + + int outIndex = TIME_FIELD_OUT_INDEX; + Integer inIndex = inFieldIndexes.get(dataDescription.getTimeField()); + if (inIndex == null) { + throw new IllegalStateException( + String.format(Locale.ROOT, "Input time field '%s' not found", dataDescription.getTimeField())); + } + inputOutputMap.add(new InputOutputMap(inIndex, outIndex)); + + for (String field : analysisConfig.analysisFields()) { + ++outIndex; + inIndex = inFieldIndexes.get(field); + if (inIndex != null) { + inputOutputMap.add(new InputOutputMap(inIndex, outIndex)); + } + } + + return inputOutputMap; + } + + protected List getInputOutputMap() { + return inputOutputMap; + } + + /** + * Check that all the fields are present in the header. + * Either return true or throw a MissingFieldException + *

+ * Every input field should have an entry in inputFieldIndexes + * otherwise the field cannot be found. + */ + protected abstract boolean checkForMissingFields(Collection inputFields, Map inputFieldIndexes, + String[] header); + + /** + * Input and output array indexes map + */ + protected class InputOutputMap { + int inputIndex; + int outputIndex; + + public InputOutputMap(int in, int out) { + inputIndex = in; + outputIndex = out; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractJsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractJsonRecordReader.java new file mode 100644 index 00000000000..62511bbb9f5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AbstractJsonRecordReader.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.ElasticsearchParseException; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Objects; + +abstract class AbstractJsonRecordReader implements JsonRecordReader { + static final int PARSE_ERRORS_LIMIT = 100; + + // NORELEASE - Remove direct dependency on Jackson + protected final JsonParser parser; + protected final Map fieldMap; + protected final Logger logger; + protected int nestedLevel; + protected long fieldCount; + protected int errorCounter; + + /** + * Create a reader that parses the mapped fields from JSON. + * + * @param parser + * The JSON parser + * @param fieldMap + * Map to field name to record array index position + * @param logger + * the logger + */ + AbstractJsonRecordReader(JsonParser parser, Map fieldMap, Logger logger) { + this.parser = Objects.requireNonNull(parser); + this.fieldMap = Objects.requireNonNull(fieldMap); + this.logger = Objects.requireNonNull(logger); + } + + protected void initArrays(String[] record, boolean[] gotFields) { + Arrays.fill(gotFields, false); + Arrays.fill(record, ""); + } + + /** + * Returns null at the EOF or the next token + */ + protected JsonToken tryNextTokenOrReadToEndOnError() throws IOException { + try { + return parser.nextToken(); + } catch (JsonParseException e) { + logger.warn("Attempting to recover from malformed JSON data.", e); + for (int i = 0; i <= nestedLevel; ++i) { + readToEndOfObject(); + } + clearNestedLevel(); + } + + return parser.getCurrentToken(); + } + + protected abstract void clearNestedLevel(); + + /** + * In some cases the parser doesn't recognise the '}' of a badly formed + * JSON document and so may skip to the end of the second document. In this + * case we lose an extra record. + */ + protected void readToEndOfObject() throws IOException { + JsonToken token = null; + do { + try { + token = parser.nextToken(); + } catch (JsonParseException e) { + ++errorCounter; + if (errorCounter >= PARSE_ERRORS_LIMIT) { + logger.error("Failed to recover from malformed JSON data.", e); + throw new ElasticsearchParseException("The input JSON data is malformed."); + } + } + } + while (token != JsonToken.END_OBJECT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriter.java new file mode 100644 index 00000000000..023f35cf708 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/AnalysisLimitsWriter.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.Objects; + +import org.elasticsearch.xpack.ml.job.config.AnalysisLimits; + +import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.EQUALS; +import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.NEW_LINE; + +public class AnalysisLimitsWriter { + /* + * The configuration fields used in limits.conf + */ + private static final String MEMORY_STANZA_STR = "[memory]"; + private static final String RESULTS_STANZA_STR = "[results]"; + private static final String MODEL_MEMORY_LIMIT_CONFIG_STR = "modelmemorylimit"; + private static final String MAX_EXAMPLES_LIMIT_CONFIG_STR = "maxexamples"; + + private final AnalysisLimits limits; + private final OutputStreamWriter writer; + + public AnalysisLimitsWriter(AnalysisLimits limits, OutputStreamWriter writer) { + this.limits = Objects.requireNonNull(limits); + this.writer = Objects.requireNonNull(writer); + } + + public void write() throws IOException { + StringBuilder contents = new StringBuilder(MEMORY_STANZA_STR).append(NEW_LINE); + if (limits.getModelMemoryLimit() != null && limits.getModelMemoryLimit() != 0L) { + contents.append(MODEL_MEMORY_LIMIT_CONFIG_STR + EQUALS) + .append(limits.getModelMemoryLimit()).append(NEW_LINE); + } + + contents.append(RESULTS_STANZA_STR).append(NEW_LINE); + if (limits.getCategorizationExamplesLimit() != null) { + contents.append(MAX_EXAMPLES_LIMIT_CONFIG_STR + EQUALS) + .append(limits.getCategorizationExamplesLimit()) + .append(NEW_LINE); + } + + writer.write(contents.toString()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CannotParseTimestampException.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CannotParseTimestampException.java new file mode 100644 index 00000000000..e478d9d443d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CannotParseTimestampException.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +public class CannotParseTimestampException extends Exception { + + public CannotParseTimestampException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java new file mode 100644 index 00000000000..518a0c9e7af --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriter.java @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams; + +/** + * A writer for sending control messages to the C++ autodetect process. + * The data written to outputIndex is length encoded. + */ +public class ControlMsgToProcessWriter { + /** + * This should be the same size as the buffer in the C++ autodetect process. + */ + public static final int FLUSH_SPACES_LENGTH = 8192; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String FLUSH_MESSAGE_CODE = "f"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String INTERIM_MESSAGE_CODE = "i"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + public static final String RESET_BUCKETS_MESSAGE_CODE = "r"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + private static final String ADVANCE_TIME_MESSAGE_CODE = "t"; + + /** + * This must match the code defined in the api::CAnomalyDetector C++ class. + */ + public static final String UPDATE_MESSAGE_CODE = "u"; + + + private static final String EQUALS = " = "; + private static final char NEW_LINE = '\n'; + + /** + * An number to uniquely identify each flush so that subsequent code can + * wait for acknowledgement of the correct flush. + */ + private static AtomicLong ms_FlushNumber = new AtomicLong(1); + + private final LengthEncodedWriter lengthEncodedWriter; + private final int numberOfAnalysisFields; + + /** + * Construct the control message writer with a LengthEncodedWriter + * + * @param lengthEncodedWriter + * the writer + * @param numberOfAnalysisFields + * The number of fields configured for analysis not including the + * time field + */ + public ControlMsgToProcessWriter(LengthEncodedWriter lengthEncodedWriter, int numberOfAnalysisFields) { + this.lengthEncodedWriter = Objects.requireNonNull(lengthEncodedWriter); + this.numberOfAnalysisFields= numberOfAnalysisFields; + } + + /** + * Create the control message writer with a OutputStream. A + * LengthEncodedWriter is created on the OutputStream parameter + * + * @param os + * the output stream + * @param numberOfAnalysisFields + * The number of fields configured for analysis not including the + * time field + */ + public static ControlMsgToProcessWriter create(OutputStream os, int numberOfAnalysisFields) { + return new ControlMsgToProcessWriter(new LengthEncodedWriter(os), numberOfAnalysisFields); + } + + /** + * Send an instruction to calculate interim results to the C++ autodetect process. + * + * @param params Parameters indicating whether interim results should be written + * and for which buckets + */ + public void writeCalcInterimMessage(InterimResultsParams params) throws IOException { + if (params.shouldAdvanceTime()) { + writeMessage(ADVANCE_TIME_MESSAGE_CODE + params.getAdvanceTime()); + } + if (params.shouldCalculateInterim()) { + writeControlCodeFollowedByTimeRange(INTERIM_MESSAGE_CODE, params.getStart(), params.getEnd()); + } + } + + /** + * Send a flush message to the C++ autodetect process. + * This actually consists of two messages: one to carry the flush ID and the + * other (which might not be processed until much later) to fill the buffers + * and force prior messages through. + * + * @return an ID for this flush that will be echoed back by the C++ + * autodetect process once it is complete. + */ + public String writeFlushMessage() throws IOException { + String flushId = Long.toString(ms_FlushNumber.getAndIncrement()); + writeMessage(FLUSH_MESSAGE_CODE + flushId); + + char[] spaces = new char[FLUSH_SPACES_LENGTH]; + Arrays.fill(spaces, ' '); + writeMessage(new String(spaces)); + + lengthEncodedWriter.flush(); + return flushId; + } + + public void writeResetBucketsMessage(DataLoadParams params) throws IOException { + writeControlCodeFollowedByTimeRange(RESET_BUCKETS_MESSAGE_CODE, params.getStart(), params.getEnd()); + } + + private void writeControlCodeFollowedByTimeRange(String code, String start, String end) + throws IOException { + StringBuilder message = new StringBuilder(code); + if (start.isEmpty() == false) { + message.append(start); + message.append(' '); + message.append(end); + } + writeMessage(message.toString()); + } + + public void writeUpdateModelDebugMessage(ModelDebugConfig modelDebugConfig) throws IOException { + StringWriter configWriter = new StringWriter(); + configWriter.append(UPDATE_MESSAGE_CODE).append("[modelDebugConfig]\n"); + new ModelDebugConfigWriter(modelDebugConfig, configWriter).write(); + writeMessage(configWriter.toString()); + } + + public void writeUpdateDetectorRulesMessage(int detectorIndex, List rules) throws IOException { + StringWriter configWriter = new StringWriter(); + configWriter.append(UPDATE_MESSAGE_CODE).append("[detectorRules]\n"); + configWriter.append("detectorIndex=").append(Integer.toString(detectorIndex)).append("\n"); + + configWriter.append("rulesJson="); + + XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startArray(); + for (DetectionRule rule : rules) { + rule.toXContent(builder, ToXContent.EMPTY_PARAMS); + } + builder.endArray(); + configWriter.append(builder.string()); + + writeMessage(configWriter.toString()); + } + + /** + * Transform the supplied control message to length encoded values and + * write to the OutputStream. + * The number of blank fields to make up a full record is deduced from + * analysisConfig. + * + * @param message The control message to write. + */ + private void writeMessage(String message) throws IOException { + + // The fields consist of all the analysis fields plus the time and the + // control field, hence + 2 + lengthEncodedWriter.writeNumFields(numberOfAnalysisFields + 2); + + // Write blank values for all analysis fields and the time + for (int i = -1; i < numberOfAnalysisFields; ++i) { + lengthEncodedWriter.writeField(""); + } + + // The control field comes last + lengthEncodedWriter.writeField(message); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java new file mode 100644 index 00000000000..13957d70d5b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvDataToProcessWriter.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; +import org.supercsv.io.CsvListReader; +import org.supercsv.prefs.CsvPreference; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * A writer for transforming and piping CSV data from an + * inputstream to outputstream. + * The data written to outputIndex is length encoded each record + * consists of number of fields followed by length/value pairs. + * See CLengthEncodedInputParser.h in the C++ code for a more + * detailed description. + * A control field is added to the end of each length encoded + * line. + */ +class CsvDataToProcessWriter extends AbstractDataToProcessWriter { + + private static final Logger LOGGER = Loggers.getLogger(CsvDataToProcessWriter.class); + + /** + * Maximum number of lines allowed within a single CSV record. + *

+ * In the scenario where there is a misplaced quote, there is + * the possibility that it results to a single record expanding + * over many lines. Supercsv will eventually deplete all memory + * from the JVM. We set a limit to an arbitrary large number + * to prevent that from happening. Unfortunately, supercsv + * throws an exception which means we cannot recover and continue + * reading new records from the next line. + */ + private static final int MAX_LINES_PER_RECORD = 10000; + + CsvDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + DataCountsReporter dataCountsReporter) { + super(includeControlField, autodetectProcess, dataDescription, analysisConfig, dataCountsReporter, LOGGER); + } + + /** + * Read the csv inputIndex, transform to length encoded values and pipe to + * the OutputStream. If any of the expected fields in the + * analysis inputIndex or if the expected time field is missing from the CSV + * header a exception is thrown + */ + @Override + public DataCounts write(InputStream inputStream) throws IOException { + CsvPreference csvPref = new CsvPreference.Builder( + dataDescription.getQuoteCharacter(), + dataDescription.getFieldDelimiter(), + new String(new char[]{DataDescription.LINE_ENDING})) + .maxLinesPerRow(MAX_LINES_PER_RECORD).build(); + + dataCountsReporter.startNewIncrementalCount(); + + try (CsvListReader csvReader = new CsvListReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8), csvPref)) { + String[] header = csvReader.getHeader(true); + if (header == null) { // null if EoF + return dataCountsReporter.incrementalStats(); + } + + long inputFieldCount = Math.max(header.length - 1, 0); // time field doesn't count + + buildFieldIndexMapping(header); + + // backing array for the inputIndex + String[] inputRecord = new String[header.length]; + + int maxIndex = 0; + for (Integer index : inFieldIndexes.values()) { + maxIndex = Math.max(index, maxIndex); + } + + int numFields = outputFieldCount(); + String[] record = new String[numFields]; + + List line; + while ((line = csvReader.read()) != null) { + Arrays.fill(record, ""); + + if (maxIndex >= line.size()) { + LOGGER.warn("Not enough fields in csv record, expected at least " + maxIndex + ". " + line); + + for (InputOutputMap inOut : inputOutputMap) { + if (inOut.inputIndex >= line.size()) { + dataCountsReporter.reportMissingField(); + continue; + } + + String field = line.get(inOut.inputIndex); + record[inOut.outputIndex] = (field == null) ? "" : field; + } + } else { + for (InputOutputMap inOut : inputOutputMap) { + String field = line.get(inOut.inputIndex); + record[inOut.outputIndex] = (field == null) ? "" : field; + } + } + + fillRecordFromLine(line, inputRecord); + transformTimeAndWrite(record, inputFieldCount); + } + + // This function can throw + dataCountsReporter.finishReporting(); + } + + return dataCountsReporter.incrementalStats(); + } + + private static void fillRecordFromLine(List line, String[] record) { + Arrays.fill(record, ""); + for (int i = 0; i < Math.min(line.size(), record.length); i++) { + String value = line.get(i); + if (value != null) { + record[i] = value; + } + } + } + + @Override + protected boolean checkForMissingFields(Collection inputFields, Map inputFieldIndexes, String[] header) { + for (String field : inputFields) { + if (AnalysisConfig.AUTO_CREATED_FIELDS.contains(field)) { + continue; + } + Integer index = inputFieldIndexes.get(field); + if (index == null) { + String msg = String.format(Locale.ROOT, "Field configured for analysis '%s' is not in the CSV header '%s'", + field, Arrays.toString(header)); + + LOGGER.error(msg); + throw new IllegalArgumentException(msg); + } + } + + return true; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvRecordWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvRecordWriter.java new file mode 100644 index 00000000000..3e1e2fd61a1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/CsvRecordWriter.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; + +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.supercsv.io.CsvListWriter; +import org.supercsv.prefs.CsvPreference; + +/** + * Write the records to the outputIndex stream as UTF 8 encoded CSV + */ +public class CsvRecordWriter implements RecordWriter { + private final CsvListWriter writer; + + /** + * Create the writer on the OutputStream os. + * This object will never close os. + */ + public CsvRecordWriter(OutputStream os) { + writer = new CsvListWriter(new OutputStreamWriter(os, StandardCharsets.UTF_8), CsvPreference.STANDARD_PREFERENCE); + } + + @Override + public void writeRecord(String[] record) throws IOException { + writer.write(record); + } + + @Override + public void writeRecord(List record) throws IOException { + writer.write(record); + } + + @Override + public void flush() throws IOException { + writer.flush(); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DataToProcessWriter.java new file mode 100644 index 00000000000..a09be8e1015 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DataToProcessWriter.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; + +import java.io.IOException; +import java.io.InputStream; + +/** + * A writer for transforming and piping data from an + * inputstream to outputstream as the process expects. + */ +public interface DataToProcessWriter { + + /** + * Write the header. + * The header is created from the list of analysis input fields, + * the time field and the control field. + */ + void writeHeader() throws IOException; + + /** + * Reads the inputIndex, transform to length encoded values and pipe + * to the OutputStream. + * If any of the fields in analysisFields or the + * DataDescriptions timeField is missing from the CSV header + * a MissingFieldException is thrown + * + * @return Counts of the records processed, bytes read etc + */ + DataCounts write(InputStream inputStream) throws IOException; + + /** + * Flush the outputstream + */ + void flush() throws IOException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DataToProcessWriterFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DataToProcessWriterFactory.java new file mode 100644 index 00000000000..13771a09d2b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DataToProcessWriterFactory.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; + +/** + * Factory for creating the suitable writer depending on + * whether the data format is JSON or not, and on the kind + * of date transformation that should occur. + */ +public final class DataToProcessWriterFactory { + + private DataToProcessWriterFactory() { + + } + + /** + * Constructs a {@link DataToProcessWriter} depending on + * the data format and the time transformation. + * + * @return A {@link JsonDataToProcessWriter} if the data + * format is JSON or otherwise a {@link CsvDataToProcessWriter} + */ + public static DataToProcessWriter create(boolean includeControlField, AutodetectProcess autodetectProcess, + DataDescription dataDescription, AnalysisConfig analysisConfig, + DataCountsReporter dataCountsReporter) { + switch (dataDescription.getFormat()) { + case JSON: + return new JsonDataToProcessWriter(includeControlField, autodetectProcess, dataDescription, analysisConfig, + dataCountsReporter); + case DELIMITED: + return new CsvDataToProcessWriter(includeControlField, autodetectProcess, dataDescription, analysisConfig, + dataCountsReporter); + default: + throw new IllegalArgumentException(); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DateFormatDateTransformer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DateFormatDateTransformer.java new file mode 100644 index 00000000000..74c257224d2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DateFormatDateTransformer.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import org.elasticsearch.xpack.ml.utils.time.DateTimeFormatterTimestampConverter; +import org.elasticsearch.xpack.ml.utils.time.TimestampConverter; + +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; +import java.util.Locale; + +/** + * A transformer that attempts to parse a String timestamp as a data according to a time format. + * It converts that to a long that represents the equivalent milliseconds since the epoch. + */ +public class DateFormatDateTransformer implements DateTransformer { + + private final String timeFormat; + private final TimestampConverter dateToEpochConverter; + + public DateFormatDateTransformer(String timeFormat) { + this.timeFormat = timeFormat; + dateToEpochConverter = DateTimeFormatterTimestampConverter.ofPattern(timeFormat, ZoneOffset.UTC); + } + + @Override + public long transform(String timestamp) throws CannotParseTimestampException { + try { + return dateToEpochConverter.toEpochMillis(timestamp); + } catch (DateTimeParseException e) { + String message = String.format(Locale.ROOT, "Cannot parse date '%s' with format string '%s'", timestamp, timeFormat); + throw new CannotParseTimestampException(message, e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DateTransformer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DateTransformer.java new file mode 100644 index 00000000000..a612e866a5b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DateTransformer.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +/** + * An interface for transforming a String timestamp into epoch_millis. + */ +public interface DateTransformer { + /** + * + * @param timestamp A String representing a timestamp + * @return Milliseconds since the epoch that the timestamp corresponds to + * @throws CannotParseTimestampException If the timestamp cannot be parsed + */ + long transform(String timestamp) throws CannotParseTimestampException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DoubleDateTransformer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DoubleDateTransformer.java new file mode 100644 index 00000000000..a9735fb0035 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/DoubleDateTransformer.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.util.Locale; + +/** + * A transformer that attempts to parse a String timestamp + * as a double and convert that to a long that represents + * an epoch. If m_IsMillisecond is true, it will convert to seconds. + */ +public class DoubleDateTransformer implements DateTransformer { + + private static final long MS_IN_SECOND = 1000; + + private final boolean isMillisecond; + + public DoubleDateTransformer(boolean isMillisecond) { + this.isMillisecond = isMillisecond; + } + + @Override + public long transform(String timestamp) throws CannotParseTimestampException { + try { + long longValue = Double.valueOf(timestamp).longValue(); + return isMillisecond ? longValue : longValue * MS_IN_SECOND; + } catch (NumberFormatException e) { + String message = String.format(Locale.ROOT, "Cannot parse timestamp '%s' as epoch value", timestamp); + throw new CannotParseTimestampException(message, e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java new file mode 100644 index 00000000000..13dd9125dc6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java @@ -0,0 +1,159 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.EQUALS; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import org.apache.logging.log4j.Logger; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.DefaultDetectorDescription; +import org.elasticsearch.xpack.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.utils.MlStrings; + +public class FieldConfigWriter { + private static final String DETECTOR_PREFIX = "detector."; + private static final String DETECTOR_CLAUSE_SUFFIX = ".clause"; + private static final String DETECTOR_RULES_SUFFIX = ".rules"; + private static final String INFLUENCER_PREFIX = "influencer."; + private static final String CATEGORIZATION_FIELD_OPTION = " categorizationfield="; + private static final String CATEGORIZATION_FILTER_PREFIX = "categorizationfilter."; + private static final String FILTER_PREFIX = "filter."; + + // Note: for the Engine API summarycountfield is currently passed as a + // command line option to autodetect rather than in the field config file + + private static final char NEW_LINE = '\n'; + + private final AnalysisConfig config; + private final Set filters; + private final OutputStreamWriter writer; + private final Logger logger; + + public FieldConfigWriter(AnalysisConfig config, Set filters, + OutputStreamWriter writer, Logger logger) { + this.config = Objects.requireNonNull(config); + this.filters = Objects.requireNonNull(filters); + this.writer = Objects.requireNonNull(writer); + this.logger = Objects.requireNonNull(logger); + } + + /** + * Write the Ml autodetect field options to the outputIndex stream. + */ + public void write() throws IOException { + StringBuilder contents = new StringBuilder(); + + writeDetectors(contents); + writeFilters(contents); + writeAsEnumeratedSettings(CATEGORIZATION_FILTER_PREFIX, config.getCategorizationFilters(), + contents, true); + + // As values are written as entire settings rather than part of a + // clause no quoting is needed + writeAsEnumeratedSettings(INFLUENCER_PREFIX, config.getInfluencers(), contents, false); + + logger.debug("FieldConfig:\n" + contents.toString()); + + writer.write(contents.toString()); + } + + private void writeDetectors(StringBuilder contents) throws IOException { + int counter = 0; + for (Detector detector : config.getDetectors()) { + int detectorId = counter++; + writeDetectorClause(detectorId, detector, contents); + writeDetectorRules(detectorId, detector, contents); + } + } + + private void writeDetectorClause(int detectorId, Detector detector, StringBuilder contents) { + contents.append(DETECTOR_PREFIX).append(detectorId) + .append(DETECTOR_CLAUSE_SUFFIX).append(EQUALS); + + DefaultDetectorDescription.appendOn(detector, contents); + + if (Strings.isNullOrEmpty(config.getCategorizationFieldName()) == false) { + contents.append(CATEGORIZATION_FIELD_OPTION) + .append(quoteField(config.getCategorizationFieldName())); + } + + contents.append(NEW_LINE); + } + + private void writeDetectorRules(int detectorId, Detector detector, StringBuilder contents) throws IOException { + List rules = detector.getDetectorRules(); + if (rules == null || rules.isEmpty()) { + return; + } + + contents.append(DETECTOR_PREFIX).append(detectorId) + .append(DETECTOR_RULES_SUFFIX).append(EQUALS); + + contents.append('['); + boolean first = true; + for (DetectionRule rule : detector.getDetectorRules()) { + if (first) { + first = false; + } else { + contents.append(','); + } + contents.append(rule.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS).string()); + } + contents.append(']'); + + contents.append(NEW_LINE); + } + + private void writeFilters(StringBuilder buffer) throws IOException { + for (MlFilter filter : filters) { + + StringBuilder filterAsJson = new StringBuilder(); + filterAsJson.append('['); + boolean first = true; + for (String item : filter.getItems()) { + if (first) { + first = false; + } else { + filterAsJson.append(','); + } + filterAsJson.append('"'); + filterAsJson.append(item); + filterAsJson.append('"'); + } + filterAsJson.append(']'); + buffer.append(FILTER_PREFIX).append(filter.getId()).append(EQUALS).append(filterAsJson) + .append(NEW_LINE); + } + } + + private static void writeAsEnumeratedSettings(String settingName, List values, StringBuilder buffer, boolean quote) { + if (values == null) { + return; + } + + int counter = 0; + for (String value : values) { + buffer.append(settingName).append(counter++).append(EQUALS) + .append(quote ? quoteField(value) : value).append(NEW_LINE); + } + } + + private static String quoteField(String field) { + return MlStrings.doubleQuoteIfNotAlphaNumeric(field); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java new file mode 100644 index 00000000000..351accad434 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonDataToProcessWriter.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.ml.job.process.DataCountsReporter; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; + +/** + * A writer for transforming and piping JSON data from an + * inputstream to outputstream. + * The data written to outputIndex is length encoded each record + * consists of number of fields followed by length/value pairs. + * See CLengthEncodedInputParser.h in the C++ code for a more + * detailed description. + */ +class JsonDataToProcessWriter extends AbstractDataToProcessWriter { + + private static final Logger LOGGER = Loggers.getLogger(JsonDataToProcessWriter.class); + + JsonDataToProcessWriter(boolean includeControlField, AutodetectProcess autodetectProcess, DataDescription dataDescription, + AnalysisConfig analysisConfig, DataCountsReporter dataCountsReporter) { + super(includeControlField, autodetectProcess, dataDescription, analysisConfig, dataCountsReporter, LOGGER); + } + + /** + * Read the JSON inputIndex, transform to length encoded values and pipe to + * the OutputStream. No transformation is applied to the data the timestamp + * is expected in seconds from the epoch. If any of the fields in + * analysisFields or the DataDescriptions + * timeField is missing from the JOSN inputIndex an exception is thrown + */ + @Override + public DataCounts write(InputStream inputStream) throws IOException { + dataCountsReporter.startNewIncrementalCount(); + + try (JsonParser parser = new JsonFactory().createParser(inputStream)) { + writeJson(parser); + + // this line can throw and will be propagated + dataCountsReporter.finishReporting(); + } + + return dataCountsReporter.incrementalStats(); + } + + private void writeJson(JsonParser parser) throws IOException { + Collection analysisFields = inputFields(); + + buildFieldIndexMapping(analysisFields.toArray(new String[0])); + + int numFields = outputFieldCount(); + String[] input = new String[numFields]; + String[] record = new String[numFields]; + + // We never expect to get the control field + boolean[] gotFields = new boolean[analysisFields.size()]; + + JsonRecordReader recordReader = new SimpleJsonRecordReader(parser, inFieldIndexes, LOGGER); + long inputFieldCount = recordReader.read(input, gotFields); + while (inputFieldCount >= 0) { + Arrays.fill(record, ""); + + inputFieldCount = Math.max(inputFieldCount - 1, 0); // time field doesn't count + + long missing = missingFieldCount(gotFields); + if (missing > 0) { + dataCountsReporter.reportMissingFields(missing); + } + + for (InputOutputMap inOut : inputOutputMap) { + String field = input[inOut.inputIndex]; + record[inOut.outputIndex] = (field == null) ? "" : field; + } + + transformTimeAndWrite(record, inputFieldCount); + + inputFieldCount = recordReader.read(input, gotFields); + } + } + + /** + * Don't enforce the check that all the fields are present in JSON docs. + * Always returns true + */ + @Override + protected boolean checkForMissingFields(Collection inputFields, + Map inputFieldIndexes, + String[] header) { + return true; + } + + /** + * Return the number of missing fields + */ + private static long missingFieldCount(boolean[] gotFieldFlags) { + long count = 0; + + for (int i = 0; i < gotFieldFlags.length; i++) { + if (gotFieldFlags[i] == false) { + ++count; + } + } + + return count; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonRecordReader.java new file mode 100644 index 00000000000..f8dc4ccbf37 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/JsonRecordReader.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; + +/** + * Interface for classes that read the various styles of JSON inputIndex. + */ +interface JsonRecordReader { + /** + * Read some JSON and write to the record array. + * + * @param record Read fields are written to this array. This array is first filled with empty + * strings and will never contain a null + * @param gotFields boolean array each element is true if that field + * was read + * @return The number of fields in the JSON doc or -1 if nothing was read + * because the end of the stream was reached + */ + long read(String[] record, boolean[] gotFields) throws IOException; +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/LengthEncodedWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/LengthEncodedWriter.java new file mode 100644 index 00000000000..bf380cc713b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/LengthEncodedWriter.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * Writes the data records to the outputIndex stream as length encoded pairs. + * Each record consists of number of fields followed by length/value pairs. The + * first call to one the of the writeRecord() methods should be + * with the header fields, once the headers are written records can be written + * sequentially. + *

+ * See CLengthEncodedInputParser.h in the C++ code for a more detailed + * description. + *

+ */ +public class LengthEncodedWriter implements RecordWriter { + private OutputStream outputStream; + private ByteBuffer lengthBuffer; + + /** + * Create the writer on the OutputStream os. + * This object will never close os. + */ + public LengthEncodedWriter(OutputStream os) { + outputStream = os; + // This will be used to convert 32 bit integers to network byte order + lengthBuffer = ByteBuffer.allocate(4); // 4 == sizeof(int) + } + + + /** + * Convert each String in the record array to a length/value encoded pair + * and write to the outputstream. + */ + @Override + public void writeRecord(String[] record) throws IOException { + writeNumFields(record.length); + + for (String field : record) { + writeField(field); + } + } + + /** + * Convert each String in the record list to a length/value encoded + * pair and write to the outputstream. + */ + @Override + public void writeRecord(List record) throws IOException { + writeNumFields(record.size()); + + for (String field : record) { + writeField(field); + } + } + + + /** + * Lower level functions to write records individually. + * After this function is called {@link #writeField(String)} + * must be called numFields times. + */ + public void writeNumFields(int numFields) throws IOException { + // number fields + lengthBuffer.clear(); + lengthBuffer.putInt(numFields); + outputStream.write(lengthBuffer.array()); + } + + + /** + * Lower level functions to write record fields individually. + * {@linkplain #writeNumFields(int)} must be called first + */ + public void writeField(String field) throws IOException { + byte[] utf8Bytes = field.getBytes(StandardCharsets.UTF_8); + lengthBuffer.clear(); + lengthBuffer.putInt(utf8Bytes.length); + outputStream.write(lengthBuffer.array()); + outputStream.write(utf8Bytes); + } + + @Override + public void flush() throws IOException { + outputStream.flush(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ModelDebugConfigWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ModelDebugConfigWriter.java new file mode 100644 index 00000000000..7bd44834caf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ModelDebugConfigWriter.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; + +import java.io.IOException; +import java.io.Writer; +import java.util.Objects; + +import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.EQUALS; +import static org.elasticsearch.xpack.ml.job.process.autodetect.writer.WriterConstants.NEW_LINE; + +public class ModelDebugConfigWriter { + + private final ModelDebugConfig modelDebugConfig; + private final Writer writer; + + public ModelDebugConfigWriter(ModelDebugConfig modelDebugConfig, Writer writer) { + this.modelDebugConfig = Objects.requireNonNull(modelDebugConfig); + this.writer = Objects.requireNonNull(writer); + } + + public void write() throws IOException { + StringBuilder contents = new StringBuilder(); + + contents.append("boundspercentile") + .append(EQUALS) + .append(modelDebugConfig.getBoundsPercentile()) + .append(NEW_LINE); + + String terms = modelDebugConfig.getTerms(); + contents.append(ModelDebugConfig.TERMS_FIELD.getPreferredName()) + .append(EQUALS) + .append(terms == null ? "" : terms) + .append(NEW_LINE); + + writer.write(contents.toString()); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/RecordWriter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/RecordWriter.java new file mode 100644 index 00000000000..38cf9caa22f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/RecordWriter.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; +import java.util.List; + +/** + * Interface for classes that write arrays of strings to the + * Ml analytics processes. + */ +public interface RecordWriter { + /** + * Value must match api::CAnomalyDetector::CONTROL_FIELD_NAME in the C++ + * code. + */ + String CONTROL_FIELD_NAME = "."; + + /** + * Write each String in the record array + */ + void writeRecord(String[] record) throws IOException; + + /** + * Write each String in the record list + */ + void writeRecord(List record) throws IOException; + + /** + * Flush the outputIndex stream. + */ + void flush() throws IOException; + +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/SimpleJsonRecordReader.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/SimpleJsonRecordReader.java new file mode 100644 index 00000000000..415c336e77a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/SimpleJsonRecordReader.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; + +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +class SimpleJsonRecordReader extends AbstractJsonRecordReader { + private Deque nestedFields; + private String nestedPrefix; + + /** + * Create a reader that parses the mapped fields from JSON. + * + * @param parser + * The JSON parser + * @param fieldMap + * Map to field name to record array index position + * @param logger + * logger + */ + SimpleJsonRecordReader(JsonParser parser, Map fieldMap, Logger logger) { + super(parser, fieldMap, logger); + } + + /** + * Read the JSON object and write to the record array. + * Nested objects are flattened with the field names separated by + * a '.'. + * e.g. for a record with a nested 'tags' object: + * "{"name":"my.test.metric1","tags":{"tag1":"blah","tag2":"boo"},"time":1350824400,"value":12345.678}" + * use 'tags.tag1' to reference the tag1 field in the nested object + *

+ * Array fields in the JSON are ignored + * + * @param record Read fields are written to this array. This array is first filled with empty + * strings and will never contain a null + * @param gotFields boolean array each element is true if that field + * was read + * @return The number of fields in the JSON doc or -1 if nothing was read + * because the end of the stream was reached + */ + @Override + public long read(String[] record, boolean[] gotFields) throws IOException { + initArrays(record, gotFields); + fieldCount = 0; + clearNestedLevel(); + + JsonToken token = tryNextTokenOrReadToEndOnError(); + while (!(token == JsonToken.END_OBJECT && nestedLevel == 0)) { + if (token == null) { + break; + } + + if (token == JsonToken.END_OBJECT) { + --nestedLevel; + String objectFieldName = nestedFields.pop(); + + int lastIndex = nestedPrefix.length() - objectFieldName.length() - 1; + nestedPrefix = nestedPrefix.substring(0, lastIndex); + } else if (token == JsonToken.FIELD_NAME) { + parseFieldValuePair(record, gotFields); + } + + token = tryNextTokenOrReadToEndOnError(); + } + + // null token means EOF + if (token == null) { + return -1; + } + return fieldCount; + } + + @Override + protected void clearNestedLevel() { + nestedLevel = 0; + nestedFields = new ArrayDeque(); + nestedPrefix = ""; + } + + private void parseFieldValuePair(String[] record, boolean[] gotFields) throws IOException { + String fieldName = parser.getCurrentName(); + JsonToken token = tryNextTokenOrReadToEndOnError(); + + if (token == null) { + return; + } + + if (token == JsonToken.START_OBJECT) { + ++nestedLevel; + nestedFields.push(fieldName); + nestedPrefix = nestedPrefix + fieldName + "."; + } else { + if (token == JsonToken.START_ARRAY || token.isScalarValue()) { + ++fieldCount; + + // Only do the donkey work of converting the field value to a + // string if we need it + Integer index = fieldMap.get(nestedPrefix + fieldName); + if (index != null) { + record[index] = parseSingleFieldValue(token); + gotFields[index] = true; + } else { + skipSingleFieldValue(token); + } + } + } + } + + private String parseSingleFieldValue(JsonToken token) throws IOException { + if (token == JsonToken.START_ARRAY) { + // Convert any scalar values in the array to a comma delimited + // string. (Arrays of more complex objects are ignored.) + StringBuilder strBuilder = new StringBuilder(); + boolean needComma = false; + while (token != JsonToken.END_ARRAY) { + token = tryNextTokenOrReadToEndOnError(); + if (token.isScalarValue()) { + if (needComma) { + strBuilder.append(','); + } else { + needComma = true; + } + strBuilder.append(tokenToString(token)); + } + } + + return strBuilder.toString(); + } + + return tokenToString(token); + } + + private void skipSingleFieldValue(JsonToken token) throws IOException { + // Scalar values don't need any extra skip code + if (token == JsonToken.START_ARRAY) { + // Consume the whole array but do nothing with it + int arrayDepth = 1; + do { + token = tryNextTokenOrReadToEndOnError(); + if (token == JsonToken.END_ARRAY) { + --arrayDepth; + } else if (token == JsonToken.START_ARRAY) { + ++arrayDepth; + } + } + while (token != null && arrayDepth > 0); + } + } + + /** + * Get the text representation of the current token unless it's a null. + * Nulls are replaced with empty strings to match the way the rest of the + * product treats them (which in turn is shaped by the fact that CSV + * cannot distinguish empty string and null). + */ + private String tokenToString(JsonToken token) throws IOException { + if (token == null || token == JsonToken.VALUE_NULL) { + return ""; + } + return parser.getText(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/WriterConstants.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/WriterConstants.java new file mode 100644 index 00000000000..db48e76b8a6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/WriterConstants.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.autodetect.writer; + +/** + * Common constants for process writer classes + */ +public final class WriterConstants { + public static final char NEW_LINE = '\n'; + public static final String EQUALS = " = "; + + private WriterConstants() { + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessage.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessage.java new file mode 100644 index 00000000000..af99df06ea3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessage.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.logging; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Provide access to the C++ log messages that arrive via a named pipe in JSON format. + */ +public class CppLogMessage extends ToXContentToBytes implements Writeable { + /** + * Field Names (these are defined by log4cxx; we have no control over them) + */ + public static final ParseField LOGGER_FIELD = new ParseField("logger"); + public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); + public static final ParseField LEVEL_FIELD = new ParseField("level"); + public static final ParseField PID_FIELD = new ParseField("pid"); + public static final ParseField THREAD_FIELD = new ParseField("thread"); + public static final ParseField MESSAGE_FIELD = new ParseField("message"); + public static final ParseField CLASS_FIELD = new ParseField("class"); + public static final ParseField METHOD_FIELD = new ParseField("method"); + public static final ParseField FILE_FIELD = new ParseField("file"); + public static final ParseField LINE_FIELD = new ParseField("line"); + + public static final ObjectParser PARSER = new ObjectParser<>( + LOGGER_FIELD.getPreferredName(), CppLogMessage::new); + + static { + PARSER.declareString(CppLogMessage::setLogger, LOGGER_FIELD); + PARSER.declareField(CppLogMessage::setTimestamp, p -> new Date(p.longValue()), TIMESTAMP_FIELD, ValueType.LONG); + PARSER.declareString(CppLogMessage::setLevel, LEVEL_FIELD); + PARSER.declareLong(CppLogMessage::setPid, PID_FIELD); + PARSER.declareString(CppLogMessage::setThread, THREAD_FIELD); + PARSER.declareString(CppLogMessage::setMessage, MESSAGE_FIELD); + PARSER.declareString(CppLogMessage::setClazz, CLASS_FIELD); + PARSER.declareString(CppLogMessage::setMethod, METHOD_FIELD); + PARSER.declareString(CppLogMessage::setFile, FILE_FIELD); + PARSER.declareLong(CppLogMessage::setLine, LINE_FIELD); + } + + /** + * Elasticsearch type + */ + public static final ParseField TYPE = new ParseField("cpp_log_message"); + + private String logger = ""; + private Date timestamp; + private String level = ""; + private long pid = 0; + private String thread = ""; + private String message = ""; + private String clazz = ""; + private String method = ""; + private String file = ""; + private long line = 0; + + public CppLogMessage() { + timestamp = new Date(); + } + + public CppLogMessage(StreamInput in) throws IOException { + logger = in.readString(); + timestamp = new Date(in.readVLong()); + level = in.readString(); + pid = in.readVLong(); + thread = in.readString(); + message = in.readString(); + clazz = in.readString(); + method = in.readString(); + file = in.readString(); + line = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(logger); + out.writeVLong(timestamp.getTime()); + out.writeString(level); + out.writeVLong(pid); + out.writeString(thread); + out.writeString(message); + out.writeString(clazz); + out.writeString(method); + out.writeString(file); + out.writeVLong(line); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(LOGGER_FIELD.getPreferredName(), logger); + builder.field(TIMESTAMP_FIELD.getPreferredName(), timestamp.getTime()); + builder.field(LEVEL_FIELD.getPreferredName(), level); + builder.field(PID_FIELD.getPreferredName(), pid); + builder.field(THREAD_FIELD.getPreferredName(), thread); + builder.field(MESSAGE_FIELD.getPreferredName(), message); + builder.field(CLASS_FIELD.getPreferredName(), clazz); + builder.field(METHOD_FIELD.getPreferredName(), method); + builder.field(FILE_FIELD.getPreferredName(), file); + builder.field(LINE_FIELD.getPreferredName(), line); + builder.endObject(); + return builder; + } + + public String getLogger() { + return logger; + } + + public void setLogger(String logger) { + this.logger = logger; + } + + public Date getTimestamp() { + return this.timestamp; + } + + public void setTimestamp(Date d) { + this.timestamp = d; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public long getPid() { + return pid; + } + + public void setPid(long pid) { + this.pid = pid; + } + + public String getThread() { + return thread; + } + + public void setThread(String thread) { + this.thread = thread; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + /** + * This is unreliable for some C++ compilers - probably best not to display it prominently + */ + public String getClazz() { + return clazz; + } + + public void setClazz(String clazz) { + this.clazz = clazz; + } + + /** + * This is unreliable for some C++ compilers - probably best not to display it prominently + */ + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getFile() { + return file; + } + + public void setFile(String file) { + this.file = file; + } + + public long getLine() { + return line; + } + + public void setLine(long line) { + this.line = line; + } + + @Override + public int hashCode() { + return Objects.hash(logger, timestamp, level, pid, thread, message, clazz, method, file, line); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (!(other instanceof CppLogMessage)) { + return false; + } + + CppLogMessage that = (CppLogMessage)other; + + return Objects.equals(this.logger, that.logger) && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.level, that.level) && this.pid == that.pid + && Objects.equals(this.thread, that.thread) && Objects.equals(this.message, that.message) + && Objects.equals(this.clazz, that.clazz) && Objects.equals(this.method, that.method) + && Objects.equals(this.file, that.file) && this.line == that.line; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessageHandler.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessageHandler.java new file mode 100644 index 00000000000..ef78ce53d09 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/logging/CppLogMessageHandler.java @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.logging; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.Deque; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Handle a stream of C++ log messages that arrive via a named pipe in JSON format. + * Retains the last few error messages so that they can be passed back in a REST response + * if the C++ process dies. + */ +public class CppLogMessageHandler implements Closeable { + + private static final Logger LOGGER = Loggers.getLogger(CppLogMessageHandler.class); + private static final int DEFAULT_READBUF_SIZE = 1024; + private static final int DEFAULT_ERROR_STORE_SIZE = 5; + + private final String jobId; + private final InputStream inputStream; + private final int readBufSize; + private final int errorStoreSize; + private final Deque errorStore; + private final CountDownLatch pidLatch; + private volatile boolean hasLogStreamEnded; + private volatile boolean seenFatalError; + private volatile long pid; + private volatile String cppCopyright; + + /** + * @param jobId May be null or empty if the logs are from a process not associated with a job. + * @param inputStream May not be null. + */ + public CppLogMessageHandler(String jobId, InputStream inputStream) { + this(inputStream, jobId, DEFAULT_READBUF_SIZE, DEFAULT_ERROR_STORE_SIZE); + } + + /** + * For testing - allows meddling with the logger, read buffer size and error store size. + */ + CppLogMessageHandler(InputStream inputStream, String jobId, int readBufSize, int errorStoreSize) { + this.jobId = jobId; + this.inputStream = Objects.requireNonNull(inputStream); + this.readBufSize = readBufSize; + this.errorStoreSize = errorStoreSize; + errorStore = ConcurrentCollections.newDeque(); + pidLatch = new CountDownLatch(1); + hasLogStreamEnded = false; + } + + @Override + public void close() throws IOException { + inputStream.close(); + } + + /** + * Tail the InputStream provided to the constructor, handling each complete log document as it arrives. + * This method will not return until either end-of-file is detected on the InputStream or the + * InputStream throws an exception. + */ + public void tailStream() throws IOException { + try { + XContent xContent = XContentFactory.xContent(XContentType.JSON); + BytesReference bytesRef = null; + byte[] readBuf = new byte[readBufSize]; + for (int bytesRead = inputStream.read(readBuf); bytesRead != -1; bytesRead = inputStream.read(readBuf)) { + if (bytesRef == null) { + bytesRef = new BytesArray(readBuf, 0, bytesRead); + } else { + bytesRef = new CompositeBytesReference(bytesRef, new BytesArray(readBuf, 0, bytesRead)); + } + bytesRef = parseMessages(xContent, bytesRef); + readBuf = new byte[readBufSize]; + } + } finally { + hasLogStreamEnded = true; + } + } + + public boolean hasLogStreamEnded() { + return hasLogStreamEnded; + } + + public boolean seenFatalError() { + return seenFatalError; + } + + /** + * Get the process ID of the C++ process whose log messages are being read. This will + * arrive in the first log message logged by the C++ process. They all log their version + * number immediately on startup so it should not take long to arrive, but will not be + * available instantly after the process starts. + */ + public long getPid(Duration timeout) throws TimeoutException { + // There's an assumption here that 0 is not a valid PID. This is certainly true for + // userland processes. On Windows the "System Idle Process" has PID 0 and on *nix + // PID 0 is for "sched", which is part of the kernel. + if (pid == 0) { + try { + pidLatch.await(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + if (pid == 0) { + throw new TimeoutException("Timed out waiting for C++ process PID"); + } + } + return pid; + } + + public String getCppCopyright() { + return cppCopyright; + } + + /** + * Expected to be called very infrequently. + */ + public String getErrors() { + String[] errorSnapshot = errorStore.toArray(new String[0]); + StringBuilder errors = new StringBuilder(); + for (String error : errorSnapshot) { + errors.append(error).append('\n'); + } + return errors.toString(); + } + + private BytesReference parseMessages(XContent xContent, BytesReference bytesRef) { + byte marker = xContent.streamSeparator(); + int from = 0; + while (true) { + int nextMarker = findNextMarker(marker, bytesRef, from); + if (nextMarker == -1) { + // No more markers in this block + break; + } + // Ignore blank lines + if (nextMarker > from) { + parseMessage(xContent, bytesRef.slice(from, nextMarker - from)); + } + from = nextMarker + 1; + } + if (from >= bytesRef.length()) { + return null; + } + return bytesRef.slice(from, bytesRef.length() - from); + } + + private void parseMessage(XContent xContent, BytesReference bytesRef) { + try { + XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, bytesRef); + CppLogMessage msg = CppLogMessage.PARSER.apply(parser, null); + Level level = Level.getLevel(msg.getLevel()); + if (level == null) { + // This isn't expected to ever happen + level = Level.WARN; + } else if (level.isMoreSpecificThan(Level.ERROR)) { + // Keep the last few error messages to report if the process dies + storeError(msg.getMessage()); + if (level.isMoreSpecificThan(Level.FATAL)) { + seenFatalError = true; + } + } + long latestPid = msg.getPid(); + if (pid != latestPid) { + pid = latestPid; + pidLatch.countDown(); + } + String latestMessage = msg.getMessage(); + if (cppCopyright == null && latestMessage.contains("Copyright")) { + cppCopyright = latestMessage; + } + // TODO: Is there a way to preserve the original timestamp when re-logging? + if (jobId != null) { + LOGGER.log(level, "[{}] {}/{} {}@{} {}", jobId, msg.getLogger(), latestPid, msg.getFile(), msg.getLine(), latestMessage); + } else { + LOGGER.log(level, "{}/{} {}@{} {}", msg.getLogger(), latestPid, msg.getFile(), msg.getLine(), latestMessage); + } + // TODO: Could send the message for indexing instead of or as well as logging it + } catch (IOException e) { + if (jobId != null) { + LOGGER.warn(new ParameterizedMessage("[{}] Failed to parse C++ log message: {}", + new Object[] {jobId, bytesRef.utf8ToString()}), e); + } else { + LOGGER.warn(new ParameterizedMessage("Failed to parse C++ log message: {}", new Object[] {bytesRef.utf8ToString()}), e); + } + } + } + + private void storeError(String error) { + if (Strings.isNullOrEmpty(error) || errorStoreSize <= 0) { + return; + } + if (errorStore.size() >= errorStoreSize) { + errorStore.removeFirst(); + } + errorStore.offerLast(error); + } + + private static int findNextMarker(byte marker, BytesReference bytesRef, int from) { + for (int i = from; i < bytesRef.length(); ++i) { + if (bytesRef.get(i) == marker) { + return i; + } + } + return -1; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/AbstractLeafNormalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/AbstractLeafNormalizable.java new file mode 100644 index 00000000000..17e6e965bdd --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/AbstractLeafNormalizable.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import java.util.Collections; +import java.util.List; + +abstract class AbstractLeafNormalizable extends Normalizable { + + AbstractLeafNormalizable(String indexName) { + super(indexName); + } + + @Override + public final boolean isContainerOnly() { + return false; + } + + @Override + public final List getChildrenTypes() { + return Collections.emptyList(); + } + + @Override + public final List getChildren() { + return Collections.emptyList(); + } + + @Override + public final List getChildren(ChildType type) { + throw new IllegalStateException(getClass().getSimpleName() + " has no children"); + } + + @Override + public final boolean setMaxChildrenScore(ChildType childrenType, double maxScore) { + throw new IllegalStateException(getClass().getSimpleName() + " has no children"); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java new file mode 100644 index 00000000000..af570970095 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketInfluencerNormalizable.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; + +import java.io.IOException; +import java.util.Objects; + + +class BucketInfluencerNormalizable extends AbstractLeafNormalizable { + private final BucketInfluencer bucketInfluencer; + + BucketInfluencerNormalizable(BucketInfluencer influencer, String indexName) { + super(indexName); + bucketInfluencer = Objects.requireNonNull(influencer); + } + + @Override + public String getId() { + return bucketInfluencer.getId(); + } + + @Override + public Level getLevel() { + return BucketInfluencer.BUCKET_TIME.equals(bucketInfluencer.getInfluencerFieldName()) ? + Level.ROOT : Level.BUCKET_INFLUENCER; + } + + @Override + public String getPartitionFieldName() { + return null; + } + + @Override + public String getPartitionFieldValue() { + return null; + } + + @Override + public String getPersonFieldName() { + return bucketInfluencer.getInfluencerFieldName(); + } + + @Override + public String getFunctionName() { + return null; + } + + @Override + public String getValueFieldName() { + return null; + } + + @Override + public double getProbability() { + return bucketInfluencer.getProbability(); + } + + @Override + public double getNormalizedScore() { + return bucketInfluencer.getAnomalyScore(); + } + + @Override + public void setNormalizedScore(double normalizedScore) { + bucketInfluencer.setAnomalyScore(normalizedScore); + } + + @Override + public void setParentScore(double parentScore) { + // Do nothing as it is not holding the parent score. + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return bucketInfluencer.toXContent(builder, params); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java new file mode 100644 index 00000000000..0ad9d4da99a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.results.Bucket; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable.ChildType.BUCKET_INFLUENCER; +import static org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable.ChildType.PARTITION_SCORE; +import static org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable.ChildType.RECORD; + + +public class BucketNormalizable extends Normalizable { + + private static final List CHILD_TYPES = Arrays.asList(BUCKET_INFLUENCER, RECORD, PARTITION_SCORE); + + private final Bucket bucket; + + private List records = Collections.emptyList(); + + public BucketNormalizable(Bucket bucket, String indexName) { + super(indexName); + this.bucket = Objects.requireNonNull(bucket); + } + + public Bucket getBucket() { + return bucket; + } + + @Override + public String getId() { + return bucket.getId(); + } + + @Override + public boolean isContainerOnly() { + return true; + } + + @Override + public Level getLevel() { + return Level.ROOT; + } + + @Override + public String getPartitionFieldName() { + return null; + } + + @Override + public String getPartitionFieldValue() { + return null; + } + + @Override + public String getPersonFieldName() { + return null; + } + + @Override + public String getFunctionName() { + return null; + } + + @Override + public String getValueFieldName() { + return null; + } + + @Override + public double getProbability() { + throw new IllegalStateException("Bucket is container only"); + } + + @Override + public double getNormalizedScore() { + return bucket.getAnomalyScore(); + } + + @Override + public void setNormalizedScore(double normalizedScore) { + bucket.setAnomalyScore(normalizedScore); + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + @Override + public List getChildrenTypes() { + return CHILD_TYPES; + } + + @Override + public List getChildren() { + List children = new ArrayList<>(); + for (ChildType type : getChildrenTypes()) { + children.addAll(getChildren(type)); + } + return children; + } + + @Override + public List getChildren(ChildType type) { + List children = new ArrayList<>(); + switch (type) { + case BUCKET_INFLUENCER: + children.addAll(bucket.getBucketInfluencers().stream() + .map(bi -> new BucketInfluencerNormalizable(bi, getOriginatingIndex())) + .collect(Collectors.toList())); + break; + case RECORD: + children.addAll(records); + break; + case PARTITION_SCORE: + children.addAll(bucket.getPartitionScores().stream() + .map(ps -> new PartitionScoreNormalizable(ps, getOriginatingIndex())) + .collect(Collectors.toList())); + break; + default: + throw new IllegalArgumentException("Invalid type: " + type); + } + return children; + } + + @Override + public boolean setMaxChildrenScore(ChildType childrenType, double maxScore) { + double oldScore = 0.0; + switch (childrenType) { + case BUCKET_INFLUENCER: + oldScore = bucket.getAnomalyScore(); + bucket.setAnomalyScore(maxScore); + break; + case RECORD: + oldScore = bucket.getMaxNormalizedProbability(); + bucket.setMaxNormalizedProbability(maxScore); + break; + case PARTITION_SCORE: + break; + default: + throw new IllegalArgumentException("Invalid type: " + childrenType); + } + return maxScore != oldScore; + } + + @Override + public void setParentScore(double parentScore) { + throw new IllegalStateException("Bucket has no parent"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return bucket.toXContent(builder, params); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java new file mode 100644 index 00000000000..d54666d686a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.results.Influencer; + +import java.io.IOException; +import java.util.Objects; + +class InfluencerNormalizable extends AbstractLeafNormalizable { + private final Influencer influencer; + + InfluencerNormalizable(Influencer influencer, String indexName) { + super(indexName); + this.influencer = Objects.requireNonNull(influencer); + } + + @Override + public String getId() { + return influencer.getId(); + } + + @Override + public Level getLevel() { + return Level.INFLUENCER; + } + + @Override + public String getPartitionFieldName() { + return null; + } + + @Override + public String getPartitionFieldValue() { + return null; + } + + @Override + public String getPersonFieldName() { + return influencer.getInfluencerFieldName(); + } + + @Override + public String getFunctionName() { + return null; + } + + @Override + public String getValueFieldName() { + return null; + } + + @Override + public double getProbability() { + return influencer.getProbability(); + } + + @Override + public double getNormalizedScore() { + return influencer.getAnomalyScore(); + } + + @Override + public void setNormalizedScore(double normalizedScore) { + influencer.setAnomalyScore(normalizedScore); + } + + @Override + public void setParentScore(double parentScore) { + throw new IllegalStateException("Influencer has no parent"); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return influencer.toXContent(builder, params); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Level.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Level.java new file mode 100644 index 00000000000..30aada22278 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Level.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + + +/** + * An enumeration of the different normalization levels. + * The string value of each level has to match the equivalent + * level names in the normalizer C++ process. + */ +enum Level { + ROOT("root"), + LEAF("leaf"), + BUCKET_INFLUENCER("inflb"), + INFLUENCER("infl"), + PARTITION("part"); + + private final String key; + + Level(String key) { + this.key = key; + } + + public String asString() { + return key; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java new file mode 100644 index 00000000000..fc7bd351884 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/MultiplyingNormalizerProcess.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.ml.job.process.normalizer.output.NormalizerResultHandler; + +import java.io.IOException; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; + +/** + * Normalizer process that doesn't use native code. + * + * Instead, all scores sent for normalization are multiplied by a supplied factor. Obviously this is useless + * for production operation of the product, but it serves two useful purposes in development: + * - By supplying a factor of 1.0 it can be used as a no-op when native processes are not available + * - It can be used to produce results in testing that do not vary based on changes to the real normalization algorithms + */ +public class MultiplyingNormalizerProcess implements NormalizerProcess { + private static final Logger LOGGER = Loggers.getLogger(MultiplyingNormalizerProcess.class); + + private final Settings settings; + private final double factor; + private final PipedInputStream processOutStream; + private XContentBuilder builder; + private boolean shouldIgnoreHeader; + + public MultiplyingNormalizerProcess(Settings settings, double factor) { + this.settings = settings; + this.factor = factor; + processOutStream = new PipedInputStream(); + try { + XContent xContent = XContentFactory.xContent(XContentType.JSON); + PipedOutputStream processInStream = new PipedOutputStream(processOutStream); + builder = new XContentBuilder(xContent, processInStream); + } catch (IOException e) { + LOGGER.error("Could not set up no-op pipe", e); + } + shouldIgnoreHeader = true; + } + + @Override + public void writeRecord(String[] record) throws IOException { + if (shouldIgnoreHeader) { + shouldIgnoreHeader = false; + return; + } + NormalizerResult result = new NormalizerResult(); + try { + // This isn't great as the order must match the order in Normalizer.normalize(), + // but it's only for developers who cannot run the native processes + result.setLevel(record[0]); + result.setPartitionFieldName(record[1]); + result.setPartitionFieldValue(record[2]); + result.setPersonFieldName(record[3]); + result.setFunctionName(record[4]); + result.setValueFieldName(record[5]); + result.setProbability(Double.parseDouble(record[6])); + result.setNormalizedScore(factor * Double.parseDouble(record[7])); + } catch (NumberFormatException | ArrayIndexOutOfBoundsException e) { + throw new IOException("Unable to write to no-op normalizer", e); + } + // Write lineified JSON + builder.lfAtEnd(); + result.toXContent(builder, null); + } + + @Override + public void close() throws IOException { + builder.close(); + } + + @Override + public NormalizerResultHandler createNormalizedResultsHandler() { + return new NormalizerResultHandler(settings, processOutStream); + } + + @Override + public boolean isProcessAlive() { + // Sanity check: make sure the process hasn't terminated already + return true; + } + + @Override + public String readError() { + return ""; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java new file mode 100644 index 00000000000..254d1a6cc51 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcess.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; +import org.elasticsearch.xpack.ml.job.process.logging.CppLogMessageHandler; +import org.elasticsearch.xpack.ml.job.process.autodetect.writer.LengthEncodedWriter; +import org.elasticsearch.xpack.ml.job.process.normalizer.output.NormalizerResultHandler; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * Normalizer process using native code. + */ +class NativeNormalizerProcess implements NormalizerProcess { + private static final Logger LOGGER = Loggers.getLogger(NativeNormalizerProcess.class); + + private final String jobId; + private final Settings settings; + private final CppLogMessageHandler cppLogHandler; + private final OutputStream processInStream; + private final InputStream processOutStream; + private final LengthEncodedWriter recordWriter; + private Future logTailThread; + + NativeNormalizerProcess(String jobId, Settings settings, InputStream logStream, OutputStream processInStream, + InputStream processOutStream, ExecutorService executorService) throws EsRejectedExecutionException { + this.jobId = jobId; + this.settings = settings; + cppLogHandler = new CppLogMessageHandler(jobId, logStream); + this.processInStream = new BufferedOutputStream(processInStream); + this.processOutStream = processOutStream; + this.recordWriter = new LengthEncodedWriter(this.processInStream); + logTailThread = executorService.submit(() -> { + try (CppLogMessageHandler h = cppLogHandler) { + h.tailStream(); + } catch (IOException e) { + LOGGER.error(new ParameterizedMessage("[{}] Error tailing C++ process logs", new Object[] { jobId }), e); + } + }); + } + + @Override + public void writeRecord(String[] record) throws IOException { + recordWriter.writeRecord(record); + } + + @Override + public void close() throws IOException { + try { + // closing its input causes the process to exit + processInStream.close(); + // wait for the process to exit by waiting for end-of-file on the named pipe connected to its logger + // this may take a long time as it persists the model state + logTailThread.get(5, TimeUnit.MINUTES); + if (cppLogHandler.seenFatalError()) { + throw ExceptionsHelper.serverError(cppLogHandler.getErrors()); + } + LOGGER.debug("[{}] Normalizer process exited", jobId); + } catch (ExecutionException | TimeoutException e) { + LOGGER.warn(new ParameterizedMessage("[{}] Exception closing the running normalizer process", new Object[] { jobId }), e); + } catch (InterruptedException e) { + LOGGER.warn("[{}] Exception closing the running normalizer process", jobId); + Thread.currentThread().interrupt(); + } + } + + @Override + public NormalizerResultHandler createNormalizedResultsHandler() { + return new NormalizerResultHandler(settings, processOutStream); + } + + @Override + public boolean isProcessAlive() { + // Sanity check: make sure the process hasn't terminated already + return !cppLogHandler.hasLogStreamEnded(); + } + + @Override + public String readError() { + return cppLogHandler.getErrors(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java new file mode 100644 index 00000000000..d0974cb377c --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NativeNormalizerProcessFactory.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.xpack.ml.job.process.NativeController; +import org.elasticsearch.xpack.ml.job.process.ProcessCtrl; +import org.elasticsearch.xpack.ml.job.process.ProcessPipes; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.NamedPipeHelper; + +import java.io.IOException; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeoutException; + +public class NativeNormalizerProcessFactory implements NormalizerProcessFactory { + + private static final Logger LOGGER = Loggers.getLogger(NativeNormalizerProcessFactory.class); + private static final NamedPipeHelper NAMED_PIPE_HELPER = new NamedPipeHelper(); + private static final Duration PROCESS_STARTUP_TIMEOUT = Duration.ofSeconds(2); + + private final Environment env; + private final Settings settings; + private final NativeController nativeController; + + public NativeNormalizerProcessFactory(Environment env, Settings settings, NativeController nativeController) { + this.env = Objects.requireNonNull(env); + this.settings = Objects.requireNonNull(settings); + this.nativeController = Objects.requireNonNull(nativeController); + } + + @Override + public NormalizerProcess createNormalizerProcess(String jobId, String quantilesState, Integer bucketSpan, + boolean perPartitionNormalization, ExecutorService executorService) { + ProcessPipes processPipes = new ProcessPipes(env, NAMED_PIPE_HELPER, ProcessCtrl.NORMALIZE, jobId, + true, false, true, true, false, false); + createNativeProcess(jobId, quantilesState, processPipes, bucketSpan, perPartitionNormalization); + + return new NativeNormalizerProcess(jobId, settings, processPipes.getLogStream().get(), + processPipes.getProcessInStream().get(), processPipes.getProcessOutStream().get(), executorService); + } + + private void createNativeProcess(String jobId, String quantilesState, ProcessPipes processPipes, Integer bucketSpan, + boolean perPartitionNormalization) { + + try { + List command = ProcessCtrl.buildNormalizerCommand(env, jobId, quantilesState, bucketSpan, + perPartitionNormalization, nativeController.getPid()); + processPipes.addArgs(command); + nativeController.startProcess(command); + processPipes.connectStreams(PROCESS_STARTUP_TIMEOUT); + } catch (IOException | TimeoutException e) { + String msg = "Failed to launch normalizer for job " + jobId; + LOGGER.error(msg); + throw ExceptionsHelper.serverError(msg, e); + } + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java new file mode 100644 index 00000000000..21784122ee5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizable.java @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.common.xcontent.ToXContent; + +import java.util.List; +import java.util.Objects; + +public abstract class Normalizable implements ToXContent { + public enum ChildType {BUCKET_INFLUENCER, RECORD, PARTITION_SCORE}; + + private final String indexName; + private boolean hadBigNormalizedUpdate; + + public Normalizable(String indexName) { + this.indexName = Objects.requireNonNull(indexName); + } + + /** + * The document ID of the underlying result. + * @return The document Id string + */ + public abstract String getId(); + + /** + * A {@code Normalizable} may be the owner of scores or just a + * container of other {@code Normalizable} objects. A container only + * {@code Normalizable} does not have any scores to be normalized. + * It contains scores that are aggregates of its children. + * + * @return true if this {@code Normalizable} is only a container + */ + abstract boolean isContainerOnly(); + + abstract Level getLevel(); + + abstract String getPartitionFieldName(); + + abstract String getPartitionFieldValue(); + + abstract String getPersonFieldName(); + + abstract String getFunctionName(); + + abstract String getValueFieldName(); + + abstract double getProbability(); + + abstract double getNormalizedScore(); + + abstract void setNormalizedScore(double normalizedScore); + + abstract List getChildrenTypes(); + + abstract List getChildren(); + + abstract List getChildren(ChildType type); + + /** + * Set the aggregate normalized score for a type of children + * + * @param type the child type + * @param maxScore the aggregate normalized score of the children + * @return true if the score has changed or false otherwise + */ + abstract boolean setMaxChildrenScore(ChildType type, double maxScore); + + /** + * If this {@code Normalizable} holds the score of its parent, + * set the parent score + * + * @param parentScore the score of the parent {@code Normalizable} + */ + abstract void setParentScore(double parentScore); + + public boolean hadBigNormalizedUpdate() { + return hadBigNormalizedUpdate; + } + + public void resetBigChangeFlag() { + hadBigNormalizedUpdate = false; + } + + public void raiseBigChangeFlag() { + hadBigNormalizedUpdate = true; + } + + public String getOriginatingIndex() { + return indexName; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java new file mode 100644 index 00000000000..afa23dbfe9b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Normalizer.java @@ -0,0 +1,215 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.process.normalizer.output.NormalizerResultHandler; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +/** + * Normalizes probabilities to scores in the range 0-100. + *
+ * Creates and initialises the normalizer process, pipes the probabilities + * through them and adds the normalized values to the records/buckets. + *
+ * Relies on the C++ normalizer process returning an answer for every input + * and in exactly the same order as the inputs. + */ +public class Normalizer { + private static final Logger LOGGER = Loggers.getLogger(Normalizer.class); + + private final String jobId; + private final NormalizerProcessFactory processFactory; + private final ExecutorService executorService; + + public Normalizer(String jobId, NormalizerProcessFactory processFactory, ExecutorService executorService) { + this.jobId = jobId; + this.processFactory = processFactory; + this.executorService = executorService; + } + + /** + * Launches a normalization process seeded with the quantiles state provided + * and normalizes the given results. + * + * @param bucketSpan If null the default is used + * @param perPartitionNormalization Is normalization per partition (rather than per job)? + * @param results Will be updated with the normalized results + * @param quantilesState The state to be used to seed the system change + * normalizer + */ + public void normalize(Integer bucketSpan, boolean perPartitionNormalization, + List results, String quantilesState) { + NormalizerProcess process = processFactory.createNormalizerProcess(jobId, quantilesState, bucketSpan, + perPartitionNormalization, executorService); + NormalizerResultHandler resultsHandler = process.createNormalizedResultsHandler(); + Future resultsHandlerFuture = executorService.submit(() -> { + try { + resultsHandler.process(); + } catch (IOException e) { + LOGGER.error(new ParameterizedMessage("[{}] Error reading normalizer results", new Object[] { jobId }), e); + } + }); + + try { + process.writeRecord(new String[] { + NormalizerResult.LEVEL_FIELD.getPreferredName(), + NormalizerResult.PARTITION_FIELD_NAME_FIELD.getPreferredName(), + NormalizerResult.PARTITION_FIELD_VALUE_FIELD.getPreferredName(), + NormalizerResult.PERSON_FIELD_NAME_FIELD.getPreferredName(), + NormalizerResult.FUNCTION_NAME_FIELD.getPreferredName(), + NormalizerResult.VALUE_FIELD_NAME_FIELD.getPreferredName(), + NormalizerResult.PROBABILITY_FIELD.getPreferredName(), + NormalizerResult.NORMALIZED_SCORE_FIELD.getPreferredName() + }); + + for (Normalizable result : results) { + writeNormalizableAndChildrenRecursively(result, process); + } + } catch (IOException e) { + LOGGER.error("[" + jobId + "] Error writing to the normalizer", e); + } finally { + try { + process.close(); + } catch (IOException e) { + LOGGER.error("[" + jobId + "] Error closing normalizer", e); + } + } + + // Wait for the results handler to finish + try { + resultsHandlerFuture.get(); + mergeNormalizedScoresIntoResults(resultsHandler.getNormalizedResults(), results); + } catch (ExecutionException e) { + LOGGER.error(new ParameterizedMessage("[{}] Error processing normalizer results", new Object[] { jobId }), e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private static void writeNormalizableAndChildrenRecursively(Normalizable normalizable, + NormalizerProcess process) throws IOException { + if (normalizable.isContainerOnly() == false) { + process.writeRecord(new String[] { + normalizable.getLevel().asString(), + Strings.coalesceToEmpty(normalizable.getPartitionFieldName()), + Strings.coalesceToEmpty(normalizable.getPartitionFieldValue()), + Strings.coalesceToEmpty(normalizable.getPersonFieldName()), + Strings.coalesceToEmpty(normalizable.getFunctionName()), + Strings.coalesceToEmpty(normalizable.getValueFieldName()), + Double.toString(normalizable.getProbability()), + Double.toString(normalizable.getNormalizedScore()) + }); + } + for (Normalizable child : normalizable.getChildren()) { + writeNormalizableAndChildrenRecursively(child, process); + } + } + + /** + * Updates the normalized scores on the results. + */ + private void mergeNormalizedScoresIntoResults(List normalizedScores, + List results) { + Iterator scoresIter = normalizedScores.iterator(); + for (Normalizable result : results) { + mergeRecursively(scoresIter, null, false, result); + } + if (scoresIter.hasNext()) { + LOGGER.error("[{}] Unused normalized scores remain after updating all results: {} for {}", + jobId, normalizedScores.size(), results.size()); + } + } + + /** + * Recursively merges the scores returned by the normalization process into the results + * + * @param scoresIter an Iterator of the scores returned by the normalization process + * @param parent the parent result + * @param parentHadBigChange whether the parent had a big change + * @param result the result to be updated + * @return the effective normalized score of the given result + */ + private double mergeRecursively(Iterator scoresIter, Normalizable parent, + boolean parentHadBigChange, Normalizable result) { + boolean hasBigChange = false; + if (result.isContainerOnly() == false) { + if (!scoresIter.hasNext()) { + String msg = "Error iterating normalized results"; + LOGGER.error("[{}] {}", jobId, msg); + throw new ElasticsearchException(msg); + } + + result.resetBigChangeFlag(); + if (parent != null && parentHadBigChange) { + result.setParentScore(parent.getNormalizedScore()); + result.raiseBigChangeFlag(); + } + + double normalizedScore = scoresIter.next().getNormalizedScore(); + hasBigChange = isBigUpdate(result.getNormalizedScore(), normalizedScore); + if (hasBigChange) { + result.setNormalizedScore(normalizedScore); + result.raiseBigChangeFlag(); + if (parent != null) { + parent.raiseBigChangeFlag(); + } + } + } + + for (Normalizable.ChildType childrenType : result.getChildrenTypes()) { + List children = result.getChildren(childrenType); + if (!children.isEmpty()) { + double maxChildrenScore = 0.0; + for (Normalizable child : children) { + maxChildrenScore = Math.max( + mergeRecursively(scoresIter, result, hasBigChange, child), + maxChildrenScore); + } + hasBigChange |= result.setMaxChildrenScore(childrenType, maxChildrenScore); + } + } + return result.getNormalizedScore(); + } + + /** + * Encapsulate the logic for deciding whether a change to a normalized score + * is "big". + *

+ * Current logic is that a big change is a change of at least 1 or more than + * than 50% of the higher of the two values. + * + * @param oldVal The old value of the normalized score + * @param newVal The new value of the normalized score + * @return true if the update is considered "big" + */ + private static boolean isBigUpdate(double oldVal, double newVal) { + if (Math.abs(oldVal - newVal) >= 1.0) { + return true; + } + if (oldVal > newVal) { + if (oldVal * 0.5 > newVal) { + return true; + } + } else { + if (newVal * 0.5 > oldVal) { + return true; + } + } + + return false; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerFactory.java new file mode 100644 index 00000000000..ccbbc0082b9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerFactory.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import java.util.Objects; +import java.util.concurrent.ExecutorService; + +public class NormalizerFactory { + + private final NormalizerProcessFactory processFactory; + private final ExecutorService executorService; + + public NormalizerFactory(NormalizerProcessFactory processFactory, ExecutorService executorService) { + this.processFactory = Objects.requireNonNull(processFactory); + this.executorService = Objects.requireNonNull(executorService); + } + + public Normalizer create(String jobId) { + return new Normalizer(jobId, processFactory, executorService); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcess.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcess.java new file mode 100644 index 00000000000..d0ce62612bb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcess.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.xpack.ml.job.process.normalizer.output.NormalizerResultHandler; + +import java.io.Closeable; +import java.io.IOException; + +/** + * Interface representing the native C++ normalizer process + */ +public interface NormalizerProcess extends Closeable { + + /** + * Write the record to normalizer. The record parameter should not be encoded + * (i.e. length encoded) the implementation will appy the corrrect encoding. + * + * @param record Plain array of strings, implementors of this class should + * encode the record appropriately + * @throws IOException If the write failed + */ + void writeRecord(String[] record) throws IOException; + + /** + * Create a result handler for this process's results. + * @return results handler + */ + NormalizerResultHandler createNormalizedResultsHandler(); + + /** + * Returns true if the process still running. + * @return True if the process is still running + */ + boolean isProcessAlive(); + + /** + * Read any content in the error output buffer. + * @return An error message or empty String if no error. + */ + String readError(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java new file mode 100644 index 00000000000..bdb63b77897 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerProcessFactory.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import java.util.concurrent.ExecutorService; + +/** + * Factory interface for creating implementations of {@link NormalizerProcess} + */ +public interface NormalizerProcessFactory { + /** + * Create an implementation of {@link NormalizerProcess} + * + * @param executorService Executor service used to start the async tasks a job needs to operate the analytical process + * @return The process + */ + NormalizerProcess createNormalizerProcess(String jobId, String quantilesState, Integer bucketSpan, boolean perPartitionNormalization, + ExecutorService executorService); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java new file mode 100644 index 00000000000..daac2cdf8eb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/NormalizerResult.java @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Parse the output of the normalizer process, for example: + * + * {"probability":0.01,"normalized_score":2.2} + */ +public class NormalizerResult extends ToXContentToBytes implements Writeable { + static final ParseField LEVEL_FIELD = new ParseField("level"); + static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name"); + static final ParseField PARTITION_FIELD_VALUE_FIELD = new ParseField("partition_field_value"); + static final ParseField PERSON_FIELD_NAME_FIELD = new ParseField("person_field_name"); + static final ParseField FUNCTION_NAME_FIELD = new ParseField("function_name"); + static final ParseField VALUE_FIELD_NAME_FIELD = new ParseField("value_field_name"); + static final ParseField PROBABILITY_FIELD = new ParseField("probability"); + static final ParseField NORMALIZED_SCORE_FIELD = new ParseField("normalized_score"); + + public static final ObjectParser PARSER = new ObjectParser<>( + LEVEL_FIELD.getPreferredName(), NormalizerResult::new); + + static { + PARSER.declareString(NormalizerResult::setLevel, LEVEL_FIELD); + PARSER.declareString(NormalizerResult::setPartitionFieldName, PARTITION_FIELD_NAME_FIELD); + PARSER.declareString(NormalizerResult::setPartitionFieldValue, PARTITION_FIELD_VALUE_FIELD); + PARSER.declareString(NormalizerResult::setPersonFieldName, PERSON_FIELD_NAME_FIELD); + PARSER.declareString(NormalizerResult::setFunctionName, FUNCTION_NAME_FIELD); + PARSER.declareString(NormalizerResult::setValueFieldName, VALUE_FIELD_NAME_FIELD); + PARSER.declareDouble(NormalizerResult::setProbability, PROBABILITY_FIELD); + PARSER.declareDouble(NormalizerResult::setNormalizedScore, NORMALIZED_SCORE_FIELD); + } + + private String level; + private String partitionFieldName; + private String partitionFieldValue; + private String personFieldName; + private String functionName; + private String valueFieldName; + private double probability; + private double normalizedScore; + + public NormalizerResult() { + } + + public NormalizerResult(StreamInput in) throws IOException { + level = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + personFieldName = in.readOptionalString(); + functionName = in.readOptionalString(); + valueFieldName = in.readOptionalString(); + probability = in.readDouble(); + normalizedScore = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(level); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(personFieldName); + out.writeOptionalString(functionName); + out.writeOptionalString(valueFieldName); + out.writeDouble(probability); + out.writeDouble(normalizedScore); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(LEVEL_FIELD.getPreferredName(), level); + builder.field(PARTITION_FIELD_NAME_FIELD.getPreferredName(), partitionFieldName); + builder.field(PARTITION_FIELD_VALUE_FIELD.getPreferredName(), partitionFieldValue); + builder.field(PERSON_FIELD_NAME_FIELD.getPreferredName(), personFieldName); + builder.field(FUNCTION_NAME_FIELD.getPreferredName(), functionName); + builder.field(VALUE_FIELD_NAME_FIELD.getPreferredName(), valueFieldName); + builder.field(PROBABILITY_FIELD.getPreferredName(), probability); + builder.field(NORMALIZED_SCORE_FIELD.getPreferredName(), normalizedScore); + builder.endObject(); + return builder; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String partitionFieldValue) { + this.partitionFieldValue = partitionFieldValue; + } + + public String getPersonFieldName() { + return personFieldName; + } + + public void setPersonFieldName(String personFieldName) { + this.personFieldName = personFieldName; + } + + public String getFunctionName() { + return functionName; + } + + public void setFunctionName(String functionName) { + this.functionName = functionName; + } + + public String getValueFieldName() { + return valueFieldName; + } + + public void setValueFieldName(String valueFieldName) { + this.valueFieldName = valueFieldName; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + public double getNormalizedScore() { + return normalizedScore; + } + + public void setNormalizedScore(double normalizedScore) { + this.normalizedScore = normalizedScore; + } + + @Override + public int hashCode() { + return Objects.hash(level, partitionFieldName, partitionFieldValue, personFieldName, + functionName, valueFieldName, probability, normalizedScore); + } + + /** + * Compare all the fields. + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (!(other instanceof NormalizerResult)) { + return false; + } + + NormalizerResult that = (NormalizerResult)other; + + return Objects.equals(this.level, that.level) + && Objects.equals(this.partitionFieldName, that.partitionFieldName) + && Objects.equals(this.partitionFieldValue, that.partitionFieldValue) + && Objects.equals(this.personFieldName, that.personFieldName) + && Objects.equals(this.functionName, that.functionName) + && Objects.equals(this.valueFieldName, that.valueFieldName) + && this.probability == that.probability + && this.normalizedScore == that.normalizedScore; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java new file mode 100644 index 00000000000..78c3455cebb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.results.PartitionScore; + +import java.io.IOException; +import java.util.Objects; + + +public class PartitionScoreNormalizable extends AbstractLeafNormalizable { + private final PartitionScore score; + + public PartitionScoreNormalizable(PartitionScore score, String indexName) { + super(indexName); + this.score = Objects.requireNonNull(score); + } + + @Override + public String getId() { + throw new IllegalStateException("PartitionScore has no ID as is should not be persisted outside of the owning bucket"); + } + + @Override + public Level getLevel() { + return Level.PARTITION; + } + + @Override + public String getPartitionFieldName() { + return score.getPartitionFieldName(); + } + + @Override + public String getPartitionFieldValue() { + return score.getPartitionFieldValue(); + } + + @Override + public String getPersonFieldName() { + return null; + } + + @Override + public String getFunctionName() { + return null; + } + + @Override + public String getValueFieldName() { + return null; + } + + @Override + public double getProbability() { + return score.getProbability(); + } + + @Override + public double getNormalizedScore() { + return score.getAnomalyScore(); + } + + @Override + public void setNormalizedScore(double normalizedScore) { + score.setAnomalyScore(normalizedScore); + } + + @Override + public void setParentScore(double parentScore) { + // Do nothing as it is not holding the parent score. + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return score.toXContent(builder, params); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java new file mode 100644 index 00000000000..eedf640d954 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; + +import java.io.IOException; +import java.util.Objects; + + +class RecordNormalizable extends AbstractLeafNormalizable { + private final AnomalyRecord record; + + RecordNormalizable(AnomalyRecord record, String indexName) { + super(indexName); + this.record = Objects.requireNonNull(record); + } + + @Override + public String getId() { + return record.getId(); + } + + @Override + public Level getLevel() { + return Level.LEAF; + } + + @Override + public String getPartitionFieldName() { + return record.getPartitionFieldName(); + } + + @Override + public String getPartitionFieldValue() { + return record.getPartitionFieldValue(); + } + + @Override + public String getPersonFieldName() { + String over = record.getOverFieldName(); + return over != null ? over : record.getByFieldName(); + } + + @Override + public String getFunctionName() { + return record.getFunction(); + } + + @Override + public String getValueFieldName() { + return record.getFieldName(); + } + + @Override + public double getProbability() { + return record.getProbability(); + } + + @Override + public double getNormalizedScore() { + return record.getNormalizedProbability(); + } + + @Override + public void setNormalizedScore(double normalizedScore) { + record.setNormalizedProbability(normalizedScore); + } + + @Override + public void setParentScore(double parentScore) { + record.setAnomalyScore(parentScore); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return record.toXContent(builder, params); + } + + public AnomalyRecord getRecord() { + return record; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java new file mode 100644 index 00000000000..56dd5d7be67 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/Renormalizer.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; + +public interface Renormalizer { + /** + * Update the anomaly score field on all previously persisted buckets + * and all contained records + */ + void renormalize(Quantiles quantiles); + + /** + * Blocks until the renormalizer is idle and no further quantiles updates are pending. + */ + void waitUntilIdle(); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java new file mode 100644 index 00000000000..9edac99a73f --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdater.java @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.BatchedDocumentsIterator; +import org.elasticsearch.xpack.ml.job.persistence.BatchedResultsIterator; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobRenormalizedResultsPersister; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.PerPartitionMaxProbabilities; + +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Thread safe class that updates the scores of all existing results + * with the renormalized scores + */ +public class ScoresUpdater { + private static final Logger LOGGER = Loggers.getLogger(ScoresUpdater.class); + + /** + * Target number of records to renormalize at a time + */ + private static final int TARGET_RECORDS_TO_RENORMALIZE = 100000; + + // 30 days + private static final long DEFAULT_RENORMALIZATION_WINDOW_MS = 2592000000L; + + private static final int DEFAULT_BUCKETS_IN_RENORMALIZATION_WINDOW = 100; + + private static final long SECONDS_IN_DAY = 86400; + private static final long MILLISECONDS_IN_SECOND = 1000; + + private final Job job; + private final JobProvider jobProvider; + private final JobRenormalizedResultsPersister updatesPersister; + private final NormalizerFactory normalizerFactory; + private int bucketSpan; + private long normalizationWindow; + private boolean perPartitionNormalization; + + public ScoresUpdater(Job job, JobProvider jobProvider, JobRenormalizedResultsPersister jobRenormalizedResultsPersister, + NormalizerFactory normalizerFactory) { + this.job = job; + this.jobProvider = Objects.requireNonNull(jobProvider); + updatesPersister = Objects.requireNonNull(jobRenormalizedResultsPersister); + this.normalizerFactory = Objects.requireNonNull(normalizerFactory); + bucketSpan = getBucketSpanOrDefault(job.getAnalysisConfig()); + normalizationWindow = getNormalizationWindowOrDefault(job); + perPartitionNormalization = getPerPartitionNormalizationOrDefault(job.getAnalysisConfig()); + } + + private static int getBucketSpanOrDefault(AnalysisConfig analysisConfig) { + if (analysisConfig != null && analysisConfig.getBucketSpan() != null) { + return analysisConfig.getBucketSpan().intValue(); + } + // A bucketSpan value of 0 will result to the default + // bucketSpan value being used in the back-end. + return 0; + } + + private long getNormalizationWindowOrDefault(Job job) { + if (job.getRenormalizationWindowDays() != null) { + return job.getRenormalizationWindowDays() * SECONDS_IN_DAY * MILLISECONDS_IN_SECOND; + } + return Math.max(DEFAULT_RENORMALIZATION_WINDOW_MS, + DEFAULT_BUCKETS_IN_RENORMALIZATION_WINDOW * bucketSpan * MILLISECONDS_IN_SECOND); + } + + private static boolean getPerPartitionNormalizationOrDefault(AnalysisConfig analysisConfig) { + return (analysisConfig != null) && analysisConfig.getUsePerPartitionNormalization(); + } + + /** + * Update the anomaly score field on all previously persisted buckets + * and all contained records + */ + public void update(String quantilesState, long endBucketEpochMs, long windowExtensionMs, boolean perPartitionNormalization) { + Normalizer normalizer = normalizerFactory.create(job.getId()); + int[] counts = {0, 0}; + updateBuckets(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts, + perPartitionNormalization); + updateInfluencers(normalizer, quantilesState, endBucketEpochMs, windowExtensionMs, counts); + + LOGGER.info("[{}] Normalization resulted in: {} updates, {} no-ops", job.getId(), counts[0], counts[1]); + } + + private void updateBuckets(Normalizer normalizer, String quantilesState, long endBucketEpochMs, + long windowExtensionMs, int[] counts, boolean perPartitionNormalization) { + BatchedDocumentsIterator> bucketsIterator = + jobProvider.newBatchedBucketsIterator(job.getId()) + .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs); + + // Make a list of buckets to be renormalized. + // This may be shorter than the original list of buckets for two + // reasons: + // 1) We don't bother with buckets that have raw score 0 and no + // records + // 2) We limit the total number of records to be not much more + // than 100000 + List bucketsToRenormalize = new ArrayList<>(); + int batchRecordCount = 0; + int skipped = 0; + + while (bucketsIterator.hasNext()) { + // Get a batch of buckets without their records to calculate + // how many buckets can be sensibly retrieved + Deque> buckets = bucketsIterator.next(); + if (buckets.isEmpty()) { + break; + } + + while (!buckets.isEmpty()) { + BatchedResultsIterator.ResultWithIndex current = buckets.removeFirst(); + Bucket currentBucket = current.result; + if (currentBucket.isNormalizable()) { + BucketNormalizable bucketNormalizable = new BucketNormalizable(current.result, current.indexName); + List recordNormalizables = + bucketRecordsAsNormalizables(currentBucket.getTimestamp().getTime()); + batchRecordCount += recordNormalizables.size(); + bucketNormalizable.setRecords(recordNormalizables); + bucketsToRenormalize.add(bucketNormalizable); + + } else { + ++skipped; + } + + if (batchRecordCount >= TARGET_RECORDS_TO_RENORMALIZE) { + normalizeBuckets(normalizer, bucketsToRenormalize, quantilesState, + batchRecordCount, skipped, counts, perPartitionNormalization); + + bucketsToRenormalize = new ArrayList<>(); + batchRecordCount = 0; + skipped = 0; + } + } + } + if (!bucketsToRenormalize.isEmpty()) { + normalizeBuckets(normalizer, bucketsToRenormalize, quantilesState, batchRecordCount, skipped, counts, + perPartitionNormalization); + } + } + + private List bucketRecordsAsNormalizables(long bucketTimeStamp) { + BatchedDocumentsIterator> + recordsIterator = jobProvider.newBatchedRecordsIterator(job.getId()) + .timeRange(bucketTimeStamp, bucketTimeStamp + 1); + + List recordNormalizables = new ArrayList<>(); + while (recordsIterator.hasNext()) { + for (BatchedResultsIterator.ResultWithIndex record : recordsIterator.next() ) { + recordNormalizables.add(new RecordNormalizable(record.result, record.indexName)); + } + } + + return recordNormalizables; + } + + private long calcNormalizationWindowStart(long endEpochMs, long windowExtensionMs) { + return Math.max(0, endEpochMs - normalizationWindow - windowExtensionMs); + } + + private void normalizeBuckets(Normalizer normalizer, List normalizableBuckets, + String quantilesState, int recordCount, int skipped, int[] counts, + boolean perPartitionNormalization) { + LOGGER.debug("[{}] Will renormalize a batch of {} buckets with {} records ({} empty buckets skipped)", + job.getId(), normalizableBuckets.size(), recordCount, skipped); + + List asNormalizables = normalizableBuckets.stream() + .map(Function.identity()).collect(Collectors.toList()); + normalizer.normalize(bucketSpan, perPartitionNormalization, asNormalizables, quantilesState); + + for (BucketNormalizable bn : normalizableBuckets) { + updateSingleBucket(counts, bn); + } + + updatesPersister.executeRequest(job.getId()); + } + + private void updateSingleBucket(int[] counts, BucketNormalizable bucketNormalizable) { + if (bucketNormalizable.hadBigNormalizedUpdate()) { + if (perPartitionNormalization) { + List anomalyRecords = bucketNormalizable.getRecords().stream() + .map(RecordNormalizable::getRecord).collect(Collectors.toList()); + PerPartitionMaxProbabilities ppProbs = new PerPartitionMaxProbabilities(anomalyRecords); + updatesPersister.updateResult(ppProbs.getId(), bucketNormalizable.getOriginatingIndex(), ppProbs); + } + updatesPersister.updateBucket(bucketNormalizable); + + ++counts[0]; + } else { + ++counts[1]; + } + + persistChanged(counts, bucketNormalizable.getRecords()); + } + + private void updateInfluencers(Normalizer normalizer, String quantilesState, long endBucketEpochMs, + long windowExtensionMs, int[] counts) { + BatchedDocumentsIterator> influencersIterator = + jobProvider.newBatchedInfluencersIterator(job.getId()) + .timeRange(calcNormalizationWindowStart(endBucketEpochMs, windowExtensionMs), endBucketEpochMs); + + while (influencersIterator.hasNext()) { + Deque> influencers = influencersIterator.next(); + if (influencers.isEmpty()) { + LOGGER.debug("[{}] No influencers to renormalize for job", job.getId()); + break; + } + + LOGGER.debug("[{}] Will renormalize a batch of {} influencers", job.getId(), influencers.size()); + List asNormalizables = influencers.stream() + .map(influencerResultIndex -> + new InfluencerNormalizable(influencerResultIndex.result, influencerResultIndex.indexName)) + .collect(Collectors.toList()); + normalizer.normalize(bucketSpan, perPartitionNormalization, asNormalizables, quantilesState); + + persistChanged(counts, asNormalizables); + } + + updatesPersister.executeRequest(job.getId()); + } + + private void persistChanged(int[] counts, List asNormalizables) { + List toUpdate = asNormalizables.stream().filter(n -> n.hadBigNormalizedUpdate()).collect(Collectors.toList()); + + counts[0] += toUpdate.size(); + counts[1] += asNormalizables.size() - toUpdate.size(); + if (!toUpdate.isEmpty()) { + updatesPersister.updateResults(toUpdate); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java new file mode 100644 index 00000000000..979bf00003d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/ShortCircuitingRenormalizer.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; + +import java.util.Deque; +import java.util.Objects; +import java.util.concurrent.ConcurrentLinkedDeque; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Semaphore; + +/** + * Renormalizer that discards outdated quantiles if even newer ones are received while waiting for a prior renormalization to complete. + */ +public class ShortCircuitingRenormalizer implements Renormalizer { + + private static final Logger LOGGER = Loggers.getLogger(ShortCircuitingRenormalizer.class); + + private final String jobId; + private final ScoresUpdater scoresUpdater; + private final ExecutorService executorService; + private final boolean isPerPartitionNormalization; + private final Deque quantilesDeque = new ConcurrentLinkedDeque<>(); + private final Deque latchDeque = new ConcurrentLinkedDeque<>(); + /** + * Each job may only have 1 normalization in progress at any time; the semaphore enforces this + */ + private final Semaphore semaphore = new Semaphore(1); + + public ShortCircuitingRenormalizer(String jobId, ScoresUpdater scoresUpdater, ExecutorService executorService, + boolean isPerPartitionNormalization) { + this.jobId = jobId; + this.scoresUpdater = scoresUpdater; + this.executorService = executorService; + this.isPerPartitionNormalization = isPerPartitionNormalization; + } + + public void renormalize(Quantiles quantiles) { + // This will throw NPE if quantiles is null, so do it first + QuantilesWithLatch quantilesWithLatch = new QuantilesWithLatch(quantiles, new CountDownLatch(1)); + // Needed to ensure work is not added while the tryFinishWork() method is running + synchronized (quantilesDeque) { + // Must add to latchDeque before quantilesDeque + latchDeque.addLast(quantilesWithLatch.getLatch()); + quantilesDeque.addLast(quantilesWithLatch); + executorService.submit(() -> doRenormalizations()); + } + } + + public void waitUntilIdle() { + try { + // We cannot tolerate more than one thread running this loop at any time, + // but need a different lock to the other synchronized parts of the code + synchronized (latchDeque) { + for (CountDownLatch latchToAwait = latchDeque.pollFirst(); latchToAwait != null; latchToAwait = latchDeque.pollFirst()) { + latchToAwait.await(); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private Quantiles getEarliestQuantiles() { + QuantilesWithLatch earliestQuantilesWithLatch = quantilesDeque.peekFirst(); + return (earliestQuantilesWithLatch != null) ? earliestQuantilesWithLatch.getQuantiles() : null; + } + + private QuantilesWithLatch getLatestQuantilesWithLatchAndClear() { + // We discard all but the latest quantiles + QuantilesWithLatch latestQuantilesWithLatch = null; + for (QuantilesWithLatch quantilesWithLatch = quantilesDeque.pollFirst(); quantilesWithLatch != null; + quantilesWithLatch = quantilesDeque.pollFirst()) { + // Count down the latches associated with any discarded quantiles + if (latestQuantilesWithLatch != null) { + latestQuantilesWithLatch.getLatch().countDown(); + } + latestQuantilesWithLatch = quantilesWithLatch; + } + return latestQuantilesWithLatch; + } + + private boolean tryStartWork() { + return semaphore.tryAcquire(); + } + + private boolean tryFinishWork() { + // We cannot tolerate new work being added in between the isEmpty() check and releasing the semaphore + synchronized (quantilesDeque) { + if (!quantilesDeque.isEmpty()) { + return false; + } + semaphore.release(); + return true; + } + } + + private void forceFinishWork() { + semaphore.release(); + } + + private void doRenormalizations() { + // Exit immediately if another normalization is in progress. This means we don't hog threads. + if (tryStartWork() == false) { + return; + } + + CountDownLatch latch = null; + try { + do { + // Note that if there is only one set of quantiles in the queue then both these references will point to the same quantiles. + Quantiles earliestQuantiles = getEarliestQuantiles(); + QuantilesWithLatch latestQuantilesWithLatch = getLatestQuantilesWithLatchAndClear(); + // We could end up with latestQuantilesWithLatch being null if the thread running this method + // was preempted before the tryStartWork() call, another thread already running this method + // did the work and exited, and then this thread got true returned by tryStartWork(). + if (latestQuantilesWithLatch != null) { + Quantiles latestQuantiles = latestQuantilesWithLatch.getQuantiles(); + latch = latestQuantilesWithLatch.getLatch(); + // We could end up with earliestQuantiles being null if quantiles were + // added between getting the earliest and latest quantiles. + if (earliestQuantiles == null) { + earliestQuantiles = latestQuantiles; + } + long earliestBucketTimeMs = earliestQuantiles.getTimestamp().getTime(); + long latestBucketTimeMs = latestQuantiles.getTimestamp().getTime(); + // If we're going to skip quantiles, renormalize using the latest quantiles + // over the time ranges implied by all quantiles that were provided. + long windowExtensionMs = latestBucketTimeMs - earliestBucketTimeMs; + if (windowExtensionMs < 0) { + LOGGER.warn("[{}] Quantiles not supplied in time order - {} after {}", + jobId, latestBucketTimeMs, earliestBucketTimeMs); + windowExtensionMs = 0; + } + scoresUpdater.update(latestQuantiles.getQuantileState(), latestBucketTimeMs, windowExtensionMs, + isPerPartitionNormalization); + latch.countDown(); + latch = null; + } + // Loop if more work has become available while we were working, because the + // tasks originally submitted to do that work will have exited early. + } while (tryFinishWork() == false); + } catch (Exception e) { + LOGGER.error("[" + jobId + "] Normalization failed", e); + if (latch != null) { + latch.countDown(); + } + forceFinishWork(); + } + } + + /** + * Simple grouping of a {@linkplain Quantiles} object with its corresponding {@linkplain CountDownLatch} object. + */ + private static class QuantilesWithLatch { + private final Quantiles quantiles; + private final CountDownLatch latch; + + QuantilesWithLatch(Quantiles quantiles, CountDownLatch latch) { + this.quantiles = Objects.requireNonNull(quantiles); + this.latch = Objects.requireNonNull(latch); + } + + Quantiles getQuantiles() { + return quantiles; + } + + CountDownLatch getLatch() { + return latch; + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/noop/NoOpRenormalizer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/noop/NoOpRenormalizer.java new file mode 100644 index 00000000000..bfbf6d60dd8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/noop/NoOpRenormalizer.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer.noop; + +import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; + +/** + * A {@link Renormalizer} implementation that does absolutely nothing + * This should be removed when the normalizer code is ported + */ +public class NoOpRenormalizer implements Renormalizer { + + @Override + public void renormalize(Quantiles quantiles) { + } + + @Override + public void waitUntilIdle() { + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java new file mode 100644 index 00000000000..c5d91d372e8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/output/NormalizerResultHandler.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.process.normalizer.output; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.bytes.CompositeBytesReference; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.ml.job.process.normalizer.NormalizerResult; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +/** + * Reads normalizer output. + */ +public class NormalizerResultHandler extends AbstractComponent { + + private static final int READ_BUF_SIZE = 1024; + + private final InputStream inputStream; + private final List normalizedResults; + + public NormalizerResultHandler(Settings settings, InputStream inputStream) { + super(settings); + this.inputStream = inputStream; + normalizedResults = new ArrayList<>(); + } + + public List getNormalizedResults() { + return normalizedResults; + } + + public void process() throws IOException { + XContent xContent = XContentFactory.xContent(XContentType.JSON); + BytesReference bytesRef = null; + byte[] readBuf = new byte[READ_BUF_SIZE]; + for (int bytesRead = inputStream.read(readBuf); bytesRead != -1; bytesRead = inputStream.read(readBuf)) { + if (bytesRef == null) { + bytesRef = new BytesArray(readBuf, 0, bytesRead); + } else { + bytesRef = new CompositeBytesReference(bytesRef, new BytesArray(readBuf, 0, bytesRead)); + } + bytesRef = parseResults(xContent, bytesRef); + readBuf = new byte[READ_BUF_SIZE]; + } + } + + private BytesReference parseResults(XContent xContent, BytesReference bytesRef) throws IOException { + byte marker = xContent.streamSeparator(); + int from = 0; + while (true) { + int nextMarker = findNextMarker(marker, bytesRef, from); + if (nextMarker == -1) { + // No more markers in this block + break; + } + // Ignore blank lines + if (nextMarker > from) { + parseResult(xContent, bytesRef.slice(from, nextMarker - from)); + } + from = nextMarker + 1; + } + if (from >= bytesRef.length()) { + return null; + } + return bytesRef.slice(from, bytesRef.length() - from); + } + + private void parseResult(XContent xContent, BytesReference bytesRef) throws IOException { + XContentParser parser = xContent.createParser(NamedXContentRegistry.EMPTY, bytesRef); + NormalizerResult result = NormalizerResult.PARSER.apply(parser, null); + normalizedResults.add(result); + } + + private static int findNextMarker(byte marker, BytesReference bytesRef, int from) { + for (int i = from; i < bytesRef.length(); ++i) { + if (bytesRef.get(i) == marker) { + return i; + } + } + return -1; + } +} + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyCause.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyCause.java new file mode 100644 index 00000000000..71161c66b5e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyCause.java @@ -0,0 +1,350 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * Anomaly Cause POJO. + * Used as a nested level inside population anomaly records. + */ +public class AnomalyCause extends ToXContentToBytes implements Writeable { + public static final ParseField ANOMALY_CAUSE = new ParseField("anomaly_cause"); + /** + * Result fields + */ + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value"); + public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name"); + public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value"); + public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlated_by_field_value"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value"); + public static final ParseField FUNCTION = new ParseField("function"); + public static final ParseField FUNCTION_DESCRIPTION = new ParseField("function_description"); + public static final ParseField TYPICAL = new ParseField("typical"); + public static final ParseField ACTUAL = new ParseField("actual"); + public static final ParseField INFLUENCERS = new ParseField("influencers"); + + /** + * Metric Results + */ + public static final ParseField FIELD_NAME = new ParseField("field_name"); + + public static final ObjectParser PARSER = new ObjectParser<>(ANOMALY_CAUSE.getPreferredName(), + AnomalyCause::new); + static { + PARSER.declareDouble(AnomalyCause::setProbability, PROBABILITY); + PARSER.declareString(AnomalyCause::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(AnomalyCause::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(AnomalyCause::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); + PARSER.declareString(AnomalyCause::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(AnomalyCause::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(AnomalyCause::setFunction, FUNCTION); + PARSER.declareString(AnomalyCause::setFunctionDescription, FUNCTION_DESCRIPTION); + PARSER.declareDoubleArray(AnomalyCause::setTypical, TYPICAL); + PARSER.declareDoubleArray(AnomalyCause::setActual, ACTUAL); + PARSER.declareString(AnomalyCause::setFieldName, FIELD_NAME); + PARSER.declareString(AnomalyCause::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(AnomalyCause::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareObjectArray(AnomalyCause::setInfluencers, Influence.PARSER, INFLUENCERS); + } + + private double probability; + private String byFieldName; + private String byFieldValue; + private String correlatedByFieldValue; + private String partitionFieldName; + private String partitionFieldValue; + private String function; + private String functionDescription; + private List typical; + private List actual; + + private String fieldName; + + private String overFieldName; + private String overFieldValue; + + private List influencers; + + public AnomalyCause() { + } + + @SuppressWarnings("unchecked") + public AnomalyCause(StreamInput in) throws IOException { + probability = in.readDouble(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + correlatedByFieldValue = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + function = in.readOptionalString(); + functionDescription = in.readOptionalString(); + if (in.readBoolean()) { + typical = (List) in.readGenericValue(); + } + if (in.readBoolean()) { + actual = (List) in.readGenericValue(); + } + fieldName = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + if (in.readBoolean()) { + influencers = in.readList(Influence::new); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeDouble(probability); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(correlatedByFieldValue); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(function); + out.writeOptionalString(functionDescription); + boolean hasTypical = typical != null; + out.writeBoolean(hasTypical); + if (hasTypical) { + out.writeGenericValue(typical); + } + boolean hasActual = actual != null; + out.writeBoolean(hasActual); + if (hasActual) { + out.writeGenericValue(actual); + } + out.writeOptionalString(fieldName); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + boolean hasInfluencers = influencers != null; + out.writeBoolean(hasInfluencers); + if (hasInfluencers) { + out.writeList(influencers); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(PROBABILITY.getPreferredName(), probability); + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (correlatedByFieldValue != null) { + builder.field(CORRELATED_BY_FIELD_VALUE.getPreferredName(), correlatedByFieldValue); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (function != null) { + builder.field(FUNCTION.getPreferredName(), function); + } + if (functionDescription != null) { + builder.field(FUNCTION_DESCRIPTION.getPreferredName(), functionDescription); + } + if (typical != null) { + builder.field(TYPICAL.getPreferredName(), typical); + } + if (actual != null) { + builder.field(ACTUAL.getPreferredName(), actual); + } + if (fieldName != null) { + builder.field(FIELD_NAME.getPreferredName(), fieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (influencers != null) { + builder.field(INFLUENCERS.getPreferredName(), influencers); + } + builder.endObject(); + return builder; + } + + + public double getProbability() { + return probability; + } + + public void setProbability(double value) { + probability = value; + } + + + public String getByFieldName() { + return byFieldName; + } + + public void setByFieldName(String value) { + byFieldName = value.intern(); + } + + public String getByFieldValue() { + return byFieldValue; + } + + public void setByFieldValue(String value) { + byFieldValue = value.intern(); + } + + public String getCorrelatedByFieldValue() { + return correlatedByFieldValue; + } + + public void setCorrelatedByFieldValue(String value) { + correlatedByFieldValue = value.intern(); + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String field) { + partitionFieldName = field.intern(); + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String value) { + partitionFieldValue = value.intern(); + } + + public String getFunction() { + return function; + } + + public void setFunction(String name) { + function = name.intern(); + } + + public String getFunctionDescription() { + return functionDescription; + } + + public void setFunctionDescription(String functionDescription) { + this.functionDescription = functionDescription.intern(); + } + + public List getTypical() { + return typical; + } + + public void setTypical(List typical) { + this.typical = typical; + } + + public List getActual() { + return actual; + } + + public void setActual(List actual) { + this.actual = actual; + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String field) { + fieldName = field.intern(); + } + + public String getOverFieldName() { + return overFieldName; + } + + public void setOverFieldName(String name) { + overFieldName = name.intern(); + } + + public String getOverFieldValue() { + return overFieldValue; + } + + public void setOverFieldValue(String value) { + overFieldValue = value.intern(); + } + + public List getInfluencers() { + return influencers; + } + + public void setInfluencers(List influencers) { + this.influencers = influencers; + } + + @Override + public int hashCode() { + return Objects.hash(probability, + actual, + typical, + byFieldName, + byFieldValue, + correlatedByFieldValue, + fieldName, + function, + functionDescription, + overFieldName, + overFieldValue, + partitionFieldName, + partitionFieldValue, + influencers); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof AnomalyCause == false) { + return false; + } + + AnomalyCause that = (AnomalyCause)other; + + return this.probability == that.probability && + Objects.deepEquals(this.typical, that.typical) && + Objects.deepEquals(this.actual, that.actual) && + Objects.equals(this.function, that.function) && + Objects.equals(this.functionDescription, that.functionDescription) && + Objects.equals(this.fieldName, that.fieldName) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.byFieldValue, that.byFieldValue) && + Objects.equals(this.correlatedByFieldValue, that.correlatedByFieldValue) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.partitionFieldValue, that.partitionFieldValue) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.overFieldValue, that.overFieldValue) && + Objects.equals(this.influencers, that.influencers); + } + + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java new file mode 100644 index 00000000000..7b3fead55b5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java @@ -0,0 +1,571 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Anomaly Record POJO. + * Uses the object wrappers Boolean and Double so null values + * can be returned if the members have not been set. + */ +public class AnomalyRecord extends ToXContentToBytes implements Writeable { + + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "record"; + /** + * Result fields (all detector types) + */ + public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); + public static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name"); + public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value"); + public static final ParseField CORRELATED_BY_FIELD_VALUE = new ParseField("correlated_by_field_value"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value"); + public static final ParseField FUNCTION = new ParseField("function"); + public static final ParseField FUNCTION_DESCRIPTION = new ParseField("function_description"); + public static final ParseField TYPICAL = new ParseField("typical"); + public static final ParseField ACTUAL = new ParseField("actual"); + public static final ParseField IS_INTERIM = new ParseField("is_interim"); + public static final ParseField INFLUENCERS = new ParseField("influencers"); + public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("records"); + + /** + * Metric Results (including population metrics) + */ + public static final ParseField FIELD_NAME = new ParseField("field_name"); + + /** + * Population results + */ + public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value"); + public static final ParseField CAUSES = new ParseField("causes"); + + /** + * Normalization + */ + public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + public static final ParseField NORMALIZED_PROBABILITY = new ParseField("normalized_probability"); + public static final ParseField INITIAL_NORMALIZED_PROBABILITY = new ParseField("initial_normalized_probability"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, + a -> new AnomalyRecord((String) a[0], (Date) a[1], (long) a[2], (int) a[3])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SEQUENCE_NUM); + PARSER.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE); + PARSER.declareDouble(AnomalyRecord::setProbability, PROBABILITY); + PARSER.declareDouble(AnomalyRecord::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(AnomalyRecord::setNormalizedProbability, NORMALIZED_PROBABILITY); + PARSER.declareDouble(AnomalyRecord::setInitialNormalizedProbability, INITIAL_NORMALIZED_PROBABILITY); + PARSER.declareInt(AnomalyRecord::setDetectorIndex, DETECTOR_INDEX); + PARSER.declareBoolean(AnomalyRecord::setInterim, IS_INTERIM); + PARSER.declareString(AnomalyRecord::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(AnomalyRecord::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(AnomalyRecord::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); + PARSER.declareString(AnomalyRecord::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(AnomalyRecord::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(AnomalyRecord::setFunction, FUNCTION); + PARSER.declareString(AnomalyRecord::setFunctionDescription, FUNCTION_DESCRIPTION); + PARSER.declareDoubleArray(AnomalyRecord::setTypical, TYPICAL); + PARSER.declareDoubleArray(AnomalyRecord::setActual, ACTUAL); + PARSER.declareString(AnomalyRecord::setFieldName, FIELD_NAME); + PARSER.declareString(AnomalyRecord::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(AnomalyRecord::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareObjectArray(AnomalyRecord::setCauses, AnomalyCause.PARSER, CAUSES); + PARSER.declareObjectArray(AnomalyRecord::setInfluencers, Influence.PARSER, INFLUENCERS); + } + + private final String jobId; + private final int sequenceNum; + private int detectorIndex; + private double probability; + private String byFieldName; + private String byFieldValue; + private String correlatedByFieldValue; + private String partitionFieldName; + private String partitionFieldValue; + private String function; + private String functionDescription; + private List typical; + private List actual; + private boolean isInterim; + + private String fieldName; + + private String overFieldName; + private String overFieldValue; + private List causes; + + private double anomalyScore; + private double normalizedProbability; + + private double initialNormalizedProbability; + + private final Date timestamp; + private final long bucketSpan; + + private List influences; + + public AnomalyRecord(String jobId, Date timestamp, long bucketSpan, int sequenceNum) { + this.jobId = jobId; + this.timestamp = ExceptionsHelper.requireNonNull(timestamp, TIMESTAMP.getPreferredName()); + this.bucketSpan = bucketSpan; + this.sequenceNum = sequenceNum; + } + + @SuppressWarnings("unchecked") + public AnomalyRecord(StreamInput in) throws IOException { + jobId = in.readString(); + sequenceNum = in.readInt(); + detectorIndex = in.readInt(); + probability = in.readDouble(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + correlatedByFieldValue = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + function = in.readOptionalString(); + functionDescription = in.readOptionalString(); + fieldName = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + if (in.readBoolean()) { + typical = (List) in.readGenericValue(); + } + if (in.readBoolean()) { + actual = (List) in.readGenericValue(); + } + isInterim = in.readBoolean(); + if (in.readBoolean()) { + causes = in.readList(AnomalyCause::new); + } + anomalyScore = in.readDouble(); + normalizedProbability = in.readDouble(); + initialNormalizedProbability = in.readDouble(); + timestamp = new Date(in.readLong()); + bucketSpan = in.readLong(); + if (in.readBoolean()) { + influences = in.readList(Influence::new); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeInt(sequenceNum); + out.writeInt(detectorIndex); + out.writeDouble(probability); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(correlatedByFieldValue); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(function); + out.writeOptionalString(functionDescription); + out.writeOptionalString(fieldName); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + boolean hasTypical = typical != null; + out.writeBoolean(hasTypical); + if (hasTypical) { + out.writeGenericValue(typical); + } + boolean hasActual = actual != null; + out.writeBoolean(hasActual); + if (hasActual) { + out.writeGenericValue(actual); + } + out.writeBoolean(isInterim); + boolean hasCauses = causes != null; + out.writeBoolean(hasCauses); + if (hasCauses) { + out.writeList(causes); + } + out.writeDouble(anomalyScore); + out.writeDouble(normalizedProbability); + out.writeDouble(initialNormalizedProbability); + out.writeLong(timestamp.getTime()); + out.writeLong(bucketSpan); + boolean hasInfluencers = influences != null; + out.writeBoolean(hasInfluencers); + if (hasInfluencers) { + out.writeList(influences); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(PROBABILITY.getPreferredName(), probability); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(NORMALIZED_PROBABILITY.getPreferredName(), normalizedProbability); + builder.field(INITIAL_NORMALIZED_PROBABILITY.getPreferredName(), initialNormalizedProbability); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); + builder.field(SEQUENCE_NUM.getPreferredName(), sequenceNum); + builder.field(IS_INTERIM.getPreferredName(), isInterim); + builder.dateField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (correlatedByFieldValue != null) { + builder.field(CORRELATED_BY_FIELD_VALUE.getPreferredName(), correlatedByFieldValue); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (function != null) { + builder.field(FUNCTION.getPreferredName(), function); + } + if (functionDescription != null) { + builder.field(FUNCTION_DESCRIPTION.getPreferredName(), functionDescription); + } + if (typical != null) { + builder.field(TYPICAL.getPreferredName(), typical); + } + if (actual != null) { + builder.field(ACTUAL.getPreferredName(), actual); + } + if (fieldName != null) { + builder.field(FIELD_NAME.getPreferredName(), fieldName); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (causes != null) { + builder.field(CAUSES.getPreferredName(), causes); + } + if (influences != null) { + builder.field(INFLUENCERS.getPreferredName(), influences); + } + + Map> inputFields = inputFieldMap(); + for (String fieldName : inputFields.keySet()) { + builder.field(fieldName, inputFields.get(fieldName)); + } + + builder.endObject(); + return builder; + } + + private Map> inputFieldMap() { + Map> result = new HashMap<>(); + + addInputFieldsToMap(result, byFieldName, byFieldValue); + addInputFieldsToMap(result, overFieldName, overFieldValue); + addInputFieldsToMap(result, partitionFieldName, partitionFieldValue); + + if (influences != null) { + for (Influence inf : influences) { + String fieldName = inf.getInfluencerFieldName(); + for (String fieldValue : inf.getInfluencerFieldValues()) { + addInputFieldsToMap(result, fieldName, fieldValue); + } + } + } + return result; + } + + private void addInputFieldsToMap(Map> inputFields, String fieldName, String fieldValue) { + if (!Strings.isNullOrEmpty(fieldName) && fieldValue != null) { + if (ReservedFieldNames.isValidFieldName(fieldName)) { + inputFields.computeIfAbsent(fieldName, k -> new HashSet()).add(fieldValue); + } + } + } + + public String getJobId() { + return this.jobId; + } + + /** + * Data store ID of this record. + */ + public String getId() { + return jobId + "_" + timestamp.getTime() + "_" + bucketSpan + "_" + sequenceNum; + } + + public int getDetectorIndex() { + return detectorIndex; + } + + public void setDetectorIndex(int detectorIndex) { + this.detectorIndex = detectorIndex; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public double getNormalizedProbability() { + return normalizedProbability; + } + + public void setNormalizedProbability(double normalizedProbability) { + this.normalizedProbability = normalizedProbability; + } + + public double getInitialNormalizedProbability() { + return initialNormalizedProbability; + } + + public void setInitialNormalizedProbability(double initialNormalizedProbability) { + this.initialNormalizedProbability = initialNormalizedProbability; + } + + public Date getTimestamp() { + return timestamp; + } + + /** + * Bucketspan expressed in seconds + */ + public long getBucketSpan() { + return bucketSpan; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double value) { + probability = value; + } + + public String getByFieldName() { + return byFieldName; + } + + public void setByFieldName(String value) { + byFieldName = value.intern(); + } + + public String getByFieldValue() { + return byFieldValue; + } + + public void setByFieldValue(String value) { + byFieldValue = value.intern(); + } + + public String getCorrelatedByFieldValue() { + return correlatedByFieldValue; + } + + public void setCorrelatedByFieldValue(String value) { + correlatedByFieldValue = value.intern(); + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String field) { + partitionFieldName = field.intern(); + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String value) { + partitionFieldValue = value.intern(); + } + + public String getFunction() { + return function; + } + + public void setFunction(String name) { + function = name.intern(); + } + + public String getFunctionDescription() { + return functionDescription; + } + + public void setFunctionDescription(String functionDescription) { + this.functionDescription = functionDescription.intern(); + } + + public List getTypical() { + return typical; + } + + public void setTypical(List typical) { + this.typical = typical; + } + + public List getActual() { + return actual; + } + + public void setActual(List actual) { + this.actual = actual; + } + + public boolean isInterim() { + return isInterim; + } + + public void setInterim(boolean isInterim) { + this.isInterim = isInterim; + } + + public String getFieldName() { + return fieldName; + } + + public void setFieldName(String field) { + fieldName = field.intern(); + } + + public String getOverFieldName() { + return overFieldName; + } + + public void setOverFieldName(String name) { + overFieldName = name.intern(); + } + + public String getOverFieldValue() { + return overFieldValue; + } + + public void setOverFieldValue(String value) { + overFieldValue = value.intern(); + } + + public List getCauses() { + return causes; + } + + public void setCauses(List causes) { + this.causes = causes; + } + + public void addCause(AnomalyCause cause) { + if (causes == null) { + causes = new ArrayList<>(); + } + causes.add(cause); + } + + public List getInfluencers() { + return influences; + } + + public void setInfluencers(List influencers) { + this.influences = influencers; + } + + + @Override + public int hashCode() { + return Objects.hash(jobId, detectorIndex, sequenceNum, bucketSpan, probability, anomalyScore, + normalizedProbability, initialNormalizedProbability, typical, actual, + function, functionDescription, fieldName, byFieldName, byFieldValue, correlatedByFieldValue, + partitionFieldName, partitionFieldValue, overFieldName, overFieldValue, + timestamp, isInterim, causes, influences, jobId); + } + + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof AnomalyRecord == false) { + return false; + } + + AnomalyRecord that = (AnomalyRecord) other; + + return Objects.equals(this.jobId, that.jobId) + && this.detectorIndex == that.detectorIndex + && this.sequenceNum == that.sequenceNum + && this.bucketSpan == that.bucketSpan + && this.probability == that.probability + && this.anomalyScore == that.anomalyScore + && this.normalizedProbability == that.normalizedProbability + && this.initialNormalizedProbability == that.initialNormalizedProbability + && Objects.deepEquals(this.typical, that.typical) + && Objects.deepEquals(this.actual, that.actual) + && Objects.equals(this.function, that.function) + && Objects.equals(this.functionDescription, that.functionDescription) + && Objects.equals(this.fieldName, that.fieldName) + && Objects.equals(this.byFieldName, that.byFieldName) + && Objects.equals(this.byFieldValue, that.byFieldValue) + && Objects.equals(this.correlatedByFieldValue, that.correlatedByFieldValue) + && Objects.equals(this.partitionFieldName, that.partitionFieldName) + && Objects.equals(this.partitionFieldValue, that.partitionFieldValue) + && Objects.equals(this.overFieldName, that.overFieldName) + && Objects.equals(this.overFieldValue, that.overFieldValue) + && Objects.equals(this.timestamp, that.timestamp) + && Objects.equals(this.isInterim, that.isInterim) + && Objects.equals(this.causes, that.causes) + && Objects.equals(this.influences, that.influences); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java new file mode 100644 index 00000000000..6c2f9dc06a7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +public class AutodetectResult extends ToXContentToBytes implements Writeable { + + public static final ParseField TYPE = new ParseField("autodetect_result"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TYPE.getPreferredName(), a -> new AutodetectResult((Bucket) a[0], (List) a[1], (List) a[2], + (Quantiles) a[3], (ModelSnapshot) a[4], a[5] == null ? null : ((ModelSizeStats.Builder) a[5]).build(), + (ModelDebugOutput) a[6], (CategoryDefinition) a[7], (FlushAcknowledgement) a[8])); + + static { + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Bucket.PARSER, Bucket.RESULT_TYPE_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), AnomalyRecord.PARSER, AnomalyRecord.RESULTS_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), Influencer.PARSER, Influencer.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Quantiles.PARSER, Quantiles.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSnapshot.PARSER, ModelSnapshot.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSizeStats.PARSER, + ModelSizeStats.RESULT_TYPE_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelDebugOutput.PARSER, ModelDebugOutput.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), CategoryDefinition.PARSER, CategoryDefinition.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), FlushAcknowledgement.PARSER, FlushAcknowledgement.TYPE); + } + + private final Bucket bucket; + private final List records; + private final List influencers; + private final Quantiles quantiles; + private final ModelSnapshot modelSnapshot; + private final ModelSizeStats modelSizeStats; + private final ModelDebugOutput modelDebugOutput; + private final CategoryDefinition categoryDefinition; + private final FlushAcknowledgement flushAcknowledgement; + + public AutodetectResult(Bucket bucket, List records, List influencers, Quantiles quantiles, + ModelSnapshot modelSnapshot, ModelSizeStats modelSizeStats, ModelDebugOutput modelDebugOutput, + CategoryDefinition categoryDefinition, FlushAcknowledgement flushAcknowledgement) { + this.bucket = bucket; + this.records = records; + this.influencers = influencers; + this.quantiles = quantiles; + this.modelSnapshot = modelSnapshot; + this.modelSizeStats = modelSizeStats; + this.modelDebugOutput = modelDebugOutput; + this.categoryDefinition = categoryDefinition; + this.flushAcknowledgement = flushAcknowledgement; + } + + public AutodetectResult(StreamInput in) throws IOException { + if (in.readBoolean()) { + this.bucket = new Bucket(in); + } else { + this.bucket = null; + } + if (in.readBoolean()) { + this.records = in.readList(AnomalyRecord::new); + } else { + this.records = null; + } + if (in.readBoolean()) { + this.influencers = in.readList(Influencer::new); + } else { + this.influencers = null; + } + if (in.readBoolean()) { + this.quantiles = new Quantiles(in); + } else { + this.quantiles = null; + } + if (in.readBoolean()) { + this.modelSnapshot = new ModelSnapshot(in); + } else { + this.modelSnapshot = null; + } + if (in.readBoolean()) { + this.modelSizeStats = new ModelSizeStats(in); + } else { + this.modelSizeStats = null; + } + if (in.readBoolean()) { + this.modelDebugOutput = new ModelDebugOutput(in); + } else { + this.modelDebugOutput = null; + } + if (in.readBoolean()) { + this.categoryDefinition = new CategoryDefinition(in); + } else { + this.categoryDefinition = null; + } + if (in.readBoolean()) { + this.flushAcknowledgement = new FlushAcknowledgement(in); + } else { + this.flushAcknowledgement = null; + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeNullable(bucket, out); + writeNullable(records, out); + writeNullable(influencers, out); + writeNullable(quantiles, out); + writeNullable(modelSnapshot, out); + writeNullable(modelSizeStats, out); + writeNullable(modelDebugOutput, out); + writeNullable(categoryDefinition, out); + writeNullable(flushAcknowledgement, out); + } + + private static void writeNullable(Writeable writeable, StreamOutput out) throws IOException { + boolean isPresent = writeable != null; + out.writeBoolean(isPresent); + if (isPresent) { + writeable.writeTo(out); + } + } + + private static void writeNullable(List writeables, StreamOutput out) throws IOException { + boolean isPresent = writeables != null; + out.writeBoolean(isPresent); + if (isPresent) { + out.writeList(writeables); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + addNullableField(Bucket.RESULT_TYPE_FIELD, bucket, builder); + addNullableField(AnomalyRecord.RESULTS_FIELD, records, builder); + addNullableField(Influencer.RESULTS_FIELD, influencers, builder); + addNullableField(Quantiles.TYPE, quantiles, builder); + addNullableField(ModelSnapshot.TYPE, modelSnapshot, builder); + addNullableField(ModelSizeStats.RESULT_TYPE_FIELD, modelSizeStats, builder); + addNullableField(ModelDebugOutput.RESULTS_FIELD, modelDebugOutput, builder); + addNullableField(CategoryDefinition.TYPE, categoryDefinition, builder); + addNullableField(FlushAcknowledgement.TYPE, flushAcknowledgement, builder); + builder.endObject(); + return builder; + } + + private static void addNullableField(ParseField field, ToXContent value, XContentBuilder builder) throws IOException { + if (value != null) { + builder.field(field.getPreferredName(), value); + } + } + + private static void addNullableField(ParseField field, List values, XContentBuilder builder) throws IOException { + if (values != null) { + builder.field(field.getPreferredName(), values); + } + } + + public Bucket getBucket() { + return bucket; + } + + public List getRecords() { + return records; + } + + public List getInfluencers() { + return influencers; + } + + public Quantiles getQuantiles() { + return quantiles; + } + + public ModelSnapshot getModelSnapshot() { + return modelSnapshot; + } + + public ModelSizeStats getModelSizeStats() { + return modelSizeStats; + } + + public ModelDebugOutput getModelDebugOutput() { + return modelDebugOutput; + } + + public CategoryDefinition getCategoryDefinition() { + return categoryDefinition; + } + + public FlushAcknowledgement getFlushAcknowledgement() { + return flushAcknowledgement; + } + + @Override + public int hashCode() { + return Objects.hash(bucket, records, influencers, categoryDefinition, flushAcknowledgement, modelDebugOutput, modelSizeStats, + modelSnapshot, quantiles); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AutodetectResult other = (AutodetectResult) obj; + return Objects.equals(bucket, other.bucket) && + Objects.equals(records, other.records) && + Objects.equals(influencers, other.influencers) && + Objects.equals(categoryDefinition, other.categoryDefinition) && + Objects.equals(flushAcknowledgement, other.flushAcknowledgement) && + Objects.equals(modelDebugOutput, other.modelDebugOutput) && + Objects.equals(modelSizeStats, other.modelSizeStats) && + Objects.equals(modelSnapshot, other.modelSnapshot) && + Objects.equals(quantiles, other.quantiles); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java new file mode 100644 index 00000000000..d5fa9ed84f5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java @@ -0,0 +1,369 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Bucket Result POJO + */ +public class Bucket extends ToXContentToBytes implements Writeable { + /* + * Field Names + */ + private static final ParseField JOB_ID = Job.ID; + + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score"); + public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("max_normalized_probability"); + public static final ParseField IS_INTERIM = new ParseField("is_interim"); + public static final ParseField RECORD_COUNT = new ParseField("record_count"); + public static final ParseField EVENT_COUNT = new ParseField("event_count"); + public static final ParseField RECORDS = new ParseField("records"); + public static final ParseField BUCKET_INFLUENCERS = new ParseField("bucket_influencers"); + public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + public static final ParseField PROCESSING_TIME_MS = new ParseField("processing_time_ms"); + public static final ParseField PARTITION_SCORES = new ParseField("partition_scores"); + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("buckets"); + + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "bucket"; + public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new Bucket((String) a[0], (Date) a[1], (long) a[2])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + PARSER.declareDouble(Bucket::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(Bucket::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareDouble(Bucket::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); + PARSER.declareBoolean(Bucket::setInterim, IS_INTERIM); + PARSER.declareInt(Bucket::setRecordCount, RECORD_COUNT); + PARSER.declareLong(Bucket::setEventCount, EVENT_COUNT); + PARSER.declareObjectArray(Bucket::setRecords, AnomalyRecord.PARSER, RECORDS); + PARSER.declareObjectArray(Bucket::setBucketInfluencers, BucketInfluencer.PARSER, BUCKET_INFLUENCERS); + PARSER.declareLong(Bucket::setProcessingTimeMs, PROCESSING_TIME_MS); + PARSER.declareObjectArray(Bucket::setPartitionScores, PartitionScore.PARSER, PARTITION_SCORES); + PARSER.declareString((bucket, s) -> {}, Result.RESULT_TYPE); + } + + private final String jobId; + private final Date timestamp; + private final long bucketSpan; + private double anomalyScore; + private double initialAnomalyScore; + private double maxNormalizedProbability; + private int recordCount; + private List records = new ArrayList<>(); + private long eventCount; + private boolean isInterim; + private List bucketInfluencers = new ArrayList<>(); // Can't use emptyList as might be appended to + private long processingTimeMs; + private Map perPartitionMaxProbability = Collections.emptyMap(); + private List partitionScores = Collections.emptyList(); + + public Bucket(String jobId, Date timestamp, long bucketSpan) { + this.jobId = jobId; + this.timestamp = ExceptionsHelper.requireNonNull(timestamp, TIMESTAMP.getPreferredName()); + this.bucketSpan = bucketSpan; + } + + public Bucket(Bucket other) { + this.jobId = other.jobId; + this.timestamp = other.timestamp; + this.bucketSpan = other.bucketSpan; + this.anomalyScore = other.anomalyScore; + this.initialAnomalyScore = other.initialAnomalyScore; + this.maxNormalizedProbability = other.maxNormalizedProbability; + this.recordCount = other.recordCount; + this.records = new ArrayList<>(other.records); + this.eventCount = other.eventCount; + this.isInterim = other.isInterim; + this.bucketInfluencers = new ArrayList<>(other.bucketInfluencers); + this.processingTimeMs = other.processingTimeMs; + this.perPartitionMaxProbability = other.perPartitionMaxProbability; + this.partitionScores = new ArrayList<>(other.partitionScores); + } + + @SuppressWarnings("unchecked") + public Bucket(StreamInput in) throws IOException { + jobId = in.readString(); + timestamp = new Date(in.readLong()); + anomalyScore = in.readDouble(); + bucketSpan = in.readLong(); + initialAnomalyScore = in.readDouble(); + maxNormalizedProbability = in.readDouble(); + recordCount = in.readInt(); + records = in.readList(AnomalyRecord::new); + eventCount = in.readLong(); + isInterim = in.readBoolean(); + bucketInfluencers = in.readList(BucketInfluencer::new); + processingTimeMs = in.readLong(); + perPartitionMaxProbability = (Map) in.readGenericValue(); + partitionScores = in.readList(PartitionScore::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(timestamp.getTime()); + out.writeDouble(anomalyScore); + out.writeLong(bucketSpan); + out.writeDouble(initialAnomalyScore); + out.writeDouble(maxNormalizedProbability); + out.writeInt(recordCount); + out.writeList(records); + out.writeLong(eventCount); + out.writeBoolean(isInterim); + out.writeList(bucketInfluencers); + out.writeLong(processingTimeMs); + out.writeGenericValue(perPartitionMaxProbability); + out.writeList(partitionScores); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(JOB_ID.getPreferredName(), jobId); + builder.dateField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); + builder.field(RECORD_COUNT.getPreferredName(), recordCount); + if (!records.isEmpty()) { + builder.field(RECORDS.getPreferredName(), records); + } + builder.field(EVENT_COUNT.getPreferredName(), eventCount); + builder.field(IS_INTERIM.getPreferredName(), isInterim); + builder.field(BUCKET_INFLUENCERS.getPreferredName(), bucketInfluencers); + builder.field(PROCESSING_TIME_MS.getPreferredName(), processingTimeMs); + builder.field(PARTITION_SCORES.getPreferredName(), partitionScores); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public String getId() { + return jobId + "_" + timestamp.getTime() + "_" + bucketSpan; + } + + /** + * Timestamp expressed in seconds since the epoch (rather than Java's + * convention of milliseconds). + */ + public long getEpoch() { + return timestamp.getTime() / 1000; + } + + public Date getTimestamp() { + return timestamp; + } + + /** + * Bucketspan expressed in seconds + */ + public long getBucketSpan() { + return bucketSpan; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public void setInitialAnomalyScore(double initialAnomalyScore) { + this.initialAnomalyScore = initialAnomalyScore; + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + public void setMaxNormalizedProbability(double maxNormalizedProbability) { + this.maxNormalizedProbability = maxNormalizedProbability; + } + + public int getRecordCount() { + return recordCount; + } + + public void setRecordCount(int recordCount) { + this.recordCount = recordCount; + } + + /** + * Get all the anomaly records associated with this bucket. + * The records are not part of the bucket document. They will + * only be present when the bucket was retrieved and expanded + * to contain the associated records. + * + * @return the anomaly records for the bucket IF the bucket was expanded. + */ + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = Objects.requireNonNull(records); + } + + /** + * The number of records (events) actually processed in this bucket. + */ + public long getEventCount() { + return eventCount; + } + + public void setEventCount(long value) { + eventCount = value; + } + + public boolean isInterim() { + return isInterim; + } + + public void setInterim(boolean isInterim) { + this.isInterim = isInterim; + } + + public long getProcessingTimeMs() { + return processingTimeMs; + } + + public void setProcessingTimeMs(long timeMs) { + processingTimeMs = timeMs; + } + + public List getBucketInfluencers() { + return bucketInfluencers; + } + + public void setBucketInfluencers(List bucketInfluencers) { + this.bucketInfluencers = Objects.requireNonNull(bucketInfluencers); + } + + public void addBucketInfluencer(BucketInfluencer bucketInfluencer) { + bucketInfluencers.add(bucketInfluencer); + } + + public List getPartitionScores() { + return partitionScores; + } + + public void setPartitionScores(List scores) { + partitionScores = Objects.requireNonNull(scores); + } + + public Map getPerPartitionMaxProbability() { + return perPartitionMaxProbability; + } + + public void setPerPartitionMaxProbability(Map perPartitionMaxProbability) { + this.perPartitionMaxProbability = Objects.requireNonNull(perPartitionMaxProbability); + } + + public double partitionInitialAnomalyScore(String partitionValue) { + Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) + .findFirst(); + + return first.isPresent() ? first.get().getInitialAnomalyScore() : 0.0; + } + + public double partitionAnomalyScore(String partitionValue) { + Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) + .findFirst(); + + return first.isPresent() ? first.get().getAnomalyScore() : 0.0; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, eventCount, initialAnomalyScore, anomalyScore, maxNormalizedProbability, recordCount, records, + isInterim, bucketSpan, bucketInfluencers); + } + + /** + * Compare all the fields and embedded anomaly records (if any) + */ + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof Bucket == false) { + return false; + } + + Bucket that = (Bucket) other; + + return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp) + && (this.eventCount == that.eventCount) && (this.bucketSpan == that.bucketSpan) + && (this.anomalyScore == that.anomalyScore) && (this.initialAnomalyScore == that.initialAnomalyScore) + && (this.maxNormalizedProbability == that.maxNormalizedProbability) && (this.recordCount == that.recordCount) + && Objects.equals(this.records, that.records) && Objects.equals(this.isInterim, that.isInterim) + && Objects.equals(this.bucketInfluencers, that.bucketInfluencers); + } + + /** + * This method encapsulated the logic for whether a bucket should be + * normalized. Buckets that have no records and a score of + * zero should not be normalized as their score will not change and they + * will just add overhead. + * + * @return true if the bucket should be normalized or false otherwise + */ + public boolean isNormalizable() { + return anomalyScore > 0.0 || recordCount > 0; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/BucketInfluencer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/BucketInfluencer.java new file mode 100644 index 00000000000..c16dad00f43 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/BucketInfluencer.java @@ -0,0 +1,231 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class BucketInfluencer extends ToXContentToBytes implements Writeable { + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "bucket_influencer"; + public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE); + + /** + * Field names + */ + public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name"); + public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + public static final ParseField RAW_ANOMALY_SCORE = new ParseField("raw_anomaly_score"); + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField IS_INTERIM = new ParseField("is_interim"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + public static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); + + /** + * The influencer field name used for time influencers + */ + public static final String BUCKET_TIME = "bucket_time"; + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_FIELD.getPreferredName(), a -> new BucketInfluencer((String) a[0], + (Date) a[1], (long) a[2], (int) a[3])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SEQUENCE_NUM); + PARSER.declareString((bucketInfluencer, s) -> {}, Result.RESULT_TYPE); + PARSER.declareString(BucketInfluencer::setInfluencerFieldName, INFLUENCER_FIELD_NAME); + PARSER.declareDouble(BucketInfluencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareDouble(BucketInfluencer::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(BucketInfluencer::setRawAnomalyScore, RAW_ANOMALY_SCORE); + PARSER.declareDouble(BucketInfluencer::setProbability, PROBABILITY); + PARSER.declareBoolean(BucketInfluencer::setIsInterim, IS_INTERIM); + } + + private final String jobId; + private String influenceField; + private double initialAnomalyScore; + private double anomalyScore; + private double rawAnomalyScore; + private double probability; + private boolean isInterim; + private final Date timestamp; + private final long bucketSpan; + private final int sequenceNum; + + public BucketInfluencer(String jobId, Date timestamp, long bucketSpan, int sequenceNum) { + this.jobId = jobId; + this.timestamp = ExceptionsHelper.requireNonNull(timestamp, TIMESTAMP.getPreferredName()); + this.bucketSpan = bucketSpan; + this.sequenceNum = sequenceNum; + } + + public BucketInfluencer(StreamInput in) throws IOException { + jobId = in.readString(); + influenceField = in.readOptionalString(); + initialAnomalyScore = in.readDouble(); + anomalyScore = in.readDouble(); + rawAnomalyScore = in.readDouble(); + probability = in.readDouble(); + isInterim = in.readBoolean(); + timestamp = new Date(in.readLong()); + bucketSpan = in.readLong(); + sequenceNum = in.readInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeOptionalString(influenceField); + out.writeDouble(initialAnomalyScore); + out.writeDouble(anomalyScore); + out.writeDouble(rawAnomalyScore); + out.writeDouble(probability); + out.writeBoolean(isInterim); + out.writeLong(timestamp.getTime()); + out.writeLong(bucketSpan); + out.writeInt(sequenceNum); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + if (influenceField != null) { + builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField); + } + builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(RAW_ANOMALY_SCORE.getPreferredName(), rawAnomalyScore); + builder.field(PROBABILITY.getPreferredName(), probability); + builder.dateField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(SEQUENCE_NUM.getPreferredName(), sequenceNum); + builder.field(IS_INTERIM.getPreferredName(), isInterim); + builder.endObject(); + return builder; + } + + /** + * Data store ID of this bucket influencer. + */ + public String getId() { + return jobId + "_" + timestamp.getTime() + "_" + bucketSpan + "_" + sequenceNum; + } + + public String getJobId() { + return jobId; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + public String getInfluencerFieldName() { + return influenceField; + } + + public void setInfluencerFieldName(String fieldName) { + this.influenceField = fieldName; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public void setInitialAnomalyScore(double influenceScore) { + this.initialAnomalyScore = influenceScore; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double score) { + anomalyScore = score; + } + + public double getRawAnomalyScore() { + return rawAnomalyScore; + } + + public void setRawAnomalyScore(double score) { + rawAnomalyScore = score; + } + + public void setIsInterim(boolean isInterim) { + this.isInterim = isInterim; + } + + public boolean isInterim() { + return isInterim; + } + + public Date getTimestamp() { + return timestamp; + } + + @Override + public int hashCode() { + return Objects.hash(influenceField, initialAnomalyScore, anomalyScore, rawAnomalyScore, probability, isInterim, timestamp, jobId, + bucketSpan, sequenceNum); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + BucketInfluencer other = (BucketInfluencer) obj; + + return Objects.equals(influenceField, other.influenceField) && Double.compare(initialAnomalyScore, other.initialAnomalyScore) == 0 + && Double.compare(anomalyScore, other.anomalyScore) == 0 && Double.compare(rawAnomalyScore, other.rawAnomalyScore) == 0 + && Double.compare(probability, other.probability) == 0 && Objects.equals(isInterim, other.isInterim) + && Objects.equals(timestamp, other.timestamp) && Objects.equals(jobId, other.jobId) && bucketSpan == other.bucketSpan + && sequenceNum == other.sequenceNum; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinition.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinition.java new file mode 100644 index 00000000000..55b9644a3d7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinition.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + +public class CategoryDefinition extends ToXContentToBytes implements Writeable { + + public static final ParseField TYPE = new ParseField("category_definition"); + public static final ParseField CATEGORY_ID = new ParseField("category_id"); + public static final ParseField TERMS = new ParseField("terms"); + public static final ParseField REGEX = new ParseField("regex"); + public static final ParseField MAX_MATCHING_LENGTH = new ParseField("max_matching_length"); + public static final ParseField EXAMPLES = new ParseField("examples"); + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("categories"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new CategoryDefinition((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareLong(CategoryDefinition::setCategoryId, CATEGORY_ID); + PARSER.declareString(CategoryDefinition::setTerms, TERMS); + PARSER.declareString(CategoryDefinition::setRegex, REGEX); + PARSER.declareLong(CategoryDefinition::setMaxMatchingLength, MAX_MATCHING_LENGTH); + PARSER.declareStringArray(CategoryDefinition::setExamples, EXAMPLES); + } + + public static String documentId(String jobId, String categoryId) { + return jobId + "-" + categoryId; + } + + private final String jobId; + private long id = 0L; + private String terms = ""; + private String regex = ""; + private long maxMatchingLength = 0L; + private final Set examples; + + public CategoryDefinition(String jobId) { + this.jobId = jobId; + examples = new TreeSet<>(); + } + + public CategoryDefinition(StreamInput in) throws IOException { + jobId = in.readString(); + id = in.readLong(); + terms = in.readString(); + regex = in.readString(); + maxMatchingLength = in.readLong(); + examples = new TreeSet<>(in.readList(StreamInput::readString)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(id); + out.writeString(terms); + out.writeString(regex); + out.writeLong(maxMatchingLength); + out.writeStringList(new ArrayList<>(examples)); + } + + public String getJobId() { + return jobId; + } + + public long getCategoryId() { + return id; + } + + public void setCategoryId(long categoryId) { + id = categoryId; + } + + public String getTerms() { + return terms; + } + + public void setTerms(String terms) { + this.terms = terms; + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + + public long getMaxMatchingLength() { + return maxMatchingLength; + } + + public void setMaxMatchingLength(long maxMatchingLength) { + this.maxMatchingLength = maxMatchingLength; + } + + public List getExamples() { + return new ArrayList<>(examples); + } + + public void setExamples(Collection examples) { + this.examples.clear(); + this.examples.addAll(examples); + } + + public void addExample(String example) { + examples.add(example); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(CATEGORY_ID.getPreferredName(), id); + builder.field(TERMS.getPreferredName(), terms); + builder.field(REGEX.getPreferredName(), regex); + builder.field(MAX_MATCHING_LENGTH.getPreferredName(), maxMatchingLength); + builder.field(EXAMPLES.getPreferredName(), examples); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof CategoryDefinition == false) { + return false; + } + CategoryDefinition that = (CategoryDefinition) other; + return Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.id, that.id) + && Objects.equals(this.terms, that.terms) + && Objects.equals(this.regex, that.regex) + && Objects.equals(this.maxMatchingLength, that.maxMatchingLength) + && Objects.equals(this.examples, that.examples); + } + + @Override + public int hashCode() { + return Objects.hash(jobId, id, terms, regex, maxMatchingLength, examples); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Influence.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Influence.java new file mode 100644 index 00000000000..c9d37f8b8b1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Influence.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Influence field name and list of influence field values/score pairs + */ +public class Influence extends ToXContentToBytes implements Writeable { + + /** + * Note all publicly exposed field names are "influencer" not "influence" + */ + public static final ParseField INFLUENCER = new ParseField("influencer"); + public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name"); + public static final ParseField INFLUENCER_FIELD_VALUES = new ParseField("influencer_field_values"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + INFLUENCER.getPreferredName(), a -> new Influence((String) a[0], (List) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); + PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUES); + } + + private String field; + private List fieldValues; + + public Influence(String field, List fieldValues) { + this.field = field; + this.fieldValues = fieldValues; + } + + public Influence(StreamInput in) throws IOException { + this.field = in.readString(); + this.fieldValues = Arrays.asList(in.readStringArray()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(field); + out.writeStringArray(fieldValues.toArray(new String[fieldValues.size()])); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), field); + builder.field(INFLUENCER_FIELD_VALUES.getPreferredName(), fieldValues); + builder.endObject(); + return builder; + } + + public String getInfluencerFieldName() { + return field; + } + + public List getInfluencerFieldValues() { + return fieldValues; + } + + @Override + public int hashCode() { + return Objects.hash(field, fieldValues); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + Influence other = (Influence) obj; + return Objects.equals(field, other.field) && Objects.equals(fieldValues, other.fieldValues); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java new file mode 100644 index 00000000000..b7e0c4d99c6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class Influencer extends ToXContentToBytes implements Writeable { + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "influencer"; + public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE); + + /* + * Field names + */ + public static final ParseField PROBABILITY = new ParseField("probability"); + public static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); + public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name"); + public static final ParseField INFLUENCER_FIELD_VALUE = new ParseField("influencer_field_value"); + public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score"); + public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + + // Used for QueryPage + public static final ParseField RESULTS_FIELD = new ParseField("influencers"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + RESULT_TYPE_FIELD.getPreferredName(), true, a -> new Influencer((String) a[0], (String) a[1], (String) a[2], + (Date) a[3], (long) a[4], (int) a[5])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUE); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), SEQUENCE_NUM); + PARSER.declareString((influencer, s) -> {}, Result.RESULT_TYPE); + PARSER.declareDouble(Influencer::setProbability, PROBABILITY); + PARSER.declareDouble(Influencer::setAnomalyScore, ANOMALY_SCORE); + PARSER.declareDouble(Influencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareBoolean(Influencer::setInterim, Bucket.IS_INTERIM); + } + + private final String jobId; + private final Date timestamp; + private final long bucketSpan; + private final int sequenceNum; + private String influenceField; + private String influenceValue; + private double probability; + private double initialAnomalyScore; + private double anomalyScore; + private boolean isInterim; + + public Influencer(String jobId, String fieldName, String fieldValue, Date timestamp, long bucketSpan, int sequenceNum) { + this.jobId = jobId; + influenceField = fieldName; + influenceValue = fieldValue; + this.timestamp = ExceptionsHelper.requireNonNull(timestamp, TIMESTAMP.getPreferredName()); + this.bucketSpan = bucketSpan; + this.sequenceNum = sequenceNum; + } + + public Influencer(StreamInput in) throws IOException { + jobId = in.readString(); + timestamp = new Date(in.readLong()); + influenceField = in.readString(); + influenceValue = in.readString(); + probability = in.readDouble(); + initialAnomalyScore = in.readDouble(); + anomalyScore = in.readDouble(); + isInterim = in.readBoolean(); + bucketSpan = in.readLong(); + sequenceNum = in.readInt(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(timestamp.getTime()); + out.writeString(influenceField); + out.writeString(influenceValue); + out.writeDouble(probability); + out.writeDouble(initialAnomalyScore); + out.writeDouble(anomalyScore); + out.writeBoolean(isInterim); + out.writeLong(bucketSpan); + out.writeInt(sequenceNum); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.field(INFLUENCER_FIELD_NAME.getPreferredName(), influenceField); + builder.field(INFLUENCER_FIELD_VALUE.getPreferredName(), influenceValue); + if (ReservedFieldNames.isValidFieldName(influenceField)) { + builder.field(influenceField, influenceValue); + } + builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(PROBABILITY.getPreferredName(), probability); + builder.field(SEQUENCE_NUM.getPreferredName(), sequenceNum); + builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(Bucket.IS_INTERIM.getPreferredName(), isInterim); + builder.dateField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public String getId() { + return jobId + "_" + timestamp.getTime() + "_" + bucketSpan + "_" + sequenceNum; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + public Date getTimestamp() { + return timestamp; + } + + public String getInfluencerFieldName() { + return influenceField; + } + + public String getInfluencerFieldValue() { + return influenceValue; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public void setInitialAnomalyScore(double influenceScore) { + initialAnomalyScore = influenceScore; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double score) { + anomalyScore = score; + } + + public boolean isInterim() { + return isInterim; + } + + public void setInterim(boolean value) { + isInterim = value; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, influenceField, influenceValue, initialAnomalyScore, anomalyScore, probability, isInterim, + bucketSpan, sequenceNum); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj == null) { + return false; + } + + if (getClass() != obj.getClass()) { + return false; + } + + Influencer other = (Influencer) obj; + return Objects.equals(jobId, other.jobId) && Objects.equals(timestamp, other.timestamp) + && Objects.equals(influenceField, other.influenceField) + && Objects.equals(influenceValue, other.influenceValue) + && Double.compare(initialAnomalyScore, other.initialAnomalyScore) == 0 + && Double.compare(anomalyScore, other.anomalyScore) == 0 && Double.compare(probability, other.probability) == 0 + && (isInterim == other.isInterim) && (bucketSpan == other.bucketSpan) && (sequenceNum == other.sequenceNum); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ModelDebugOutput.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ModelDebugOutput.java new file mode 100644 index 00000000000..f589b91f93e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ModelDebugOutput.java @@ -0,0 +1,314 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +/** + * Model Debug POJO. + * Some of the fields being with the word "debug". This avoids creation of + * reserved words that are likely to clash with fields in the input data (due to + * the restrictions on Elasticsearch mappings). + */ +public class ModelDebugOutput extends ToXContentToBytes implements Writeable { + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "model_debug_output"; + public static final ParseField RESULTS_FIELD = new ParseField(RESULT_TYPE_VALUE); + + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + public static final ParseField PARTITION_FIELD_NAME = new ParseField("partition_field_name"); + public static final ParseField PARTITION_FIELD_VALUE = new ParseField("partition_field_value"); + public static final ParseField OVER_FIELD_NAME = new ParseField("over_field_name"); + public static final ParseField OVER_FIELD_VALUE = new ParseField("over_field_value"); + public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name"); + public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value"); + public static final ParseField DEBUG_FEATURE = new ParseField("debug_feature"); + public static final ParseField DEBUG_LOWER = new ParseField("debug_lower"); + public static final ParseField DEBUG_UPPER = new ParseField("debug_upper"); + public static final ParseField DEBUG_MEDIAN = new ParseField("debug_median"); + public static final ParseField ACTUAL = new ParseField("actual"); + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new ModelDebugOutput((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareString((modelDebugOutput, s) -> {}, Result.RESULT_TYPE); + PARSER.declareField(ModelDebugOutput::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + PARSER.declareString(ModelDebugOutput::setPartitionFieldName, PARTITION_FIELD_NAME); + PARSER.declareString(ModelDebugOutput::setPartitionFieldValue, PARTITION_FIELD_VALUE); + PARSER.declareString(ModelDebugOutput::setOverFieldName, OVER_FIELD_NAME); + PARSER.declareString(ModelDebugOutput::setOverFieldValue, OVER_FIELD_VALUE); + PARSER.declareString(ModelDebugOutput::setByFieldName, BY_FIELD_NAME); + PARSER.declareString(ModelDebugOutput::setByFieldValue, BY_FIELD_VALUE); + PARSER.declareString(ModelDebugOutput::setDebugFeature, DEBUG_FEATURE); + PARSER.declareDouble(ModelDebugOutput::setDebugLower, DEBUG_LOWER); + PARSER.declareDouble(ModelDebugOutput::setDebugUpper, DEBUG_UPPER); + PARSER.declareDouble(ModelDebugOutput::setDebugMedian, DEBUG_MEDIAN); + PARSER.declareDouble(ModelDebugOutput::setActual, ACTUAL); + } + + private final String jobId; + private Date timestamp; + private String id; + private String partitionFieldName; + private String partitionFieldValue; + private String overFieldName; + private String overFieldValue; + private String byFieldName; + private String byFieldValue; + private String debugFeature; + private double debugLower; + private double debugUpper; + private double debugMedian; + private double actual; + + public ModelDebugOutput(String jobId) { + this.jobId = jobId; + } + + public ModelDebugOutput(StreamInput in) throws IOException { + jobId = in.readString(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + id = in.readOptionalString(); + partitionFieldName = in.readOptionalString(); + partitionFieldValue = in.readOptionalString(); + overFieldName = in.readOptionalString(); + overFieldValue = in.readOptionalString(); + byFieldName = in.readOptionalString(); + byFieldValue = in.readOptionalString(); + debugFeature = in.readOptionalString(); + debugLower = in.readDouble(); + debugUpper = in.readDouble(); + debugMedian = in.readDouble(); + actual = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + out.writeOptionalString(id); + out.writeOptionalString(partitionFieldName); + out.writeOptionalString(partitionFieldValue); + out.writeOptionalString(overFieldName); + out.writeOptionalString(overFieldValue); + out.writeOptionalString(byFieldName); + out.writeOptionalString(byFieldValue); + out.writeOptionalString(debugFeature); + out.writeDouble(debugLower); + out.writeDouble(debugUpper); + out.writeDouble(debugMedian); + out.writeDouble(actual); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + if (timestamp != null) { + builder.dateField(TIMESTAMP.getPreferredName(), TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + } + if (partitionFieldName != null) { + builder.field(PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + } + if (partitionFieldValue != null) { + builder.field(PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + } + if (overFieldName != null) { + builder.field(OVER_FIELD_NAME.getPreferredName(), overFieldName); + } + if (overFieldValue != null) { + builder.field(OVER_FIELD_VALUE.getPreferredName(), overFieldValue); + } + if (byFieldName != null) { + builder.field(BY_FIELD_NAME.getPreferredName(), byFieldName); + } + if (byFieldValue != null) { + builder.field(BY_FIELD_VALUE.getPreferredName(), byFieldValue); + } + if (debugFeature != null) { + builder.field(DEBUG_FEATURE.getPreferredName(), debugFeature); + } + builder.field(DEBUG_LOWER.getPreferredName(), debugLower); + builder.field(DEBUG_UPPER.getPreferredName(), debugUpper); + builder.field(DEBUG_MEDIAN.getPreferredName(), debugMedian); + builder.field(ACTUAL.getPreferredName(), actual); + builder.endObject(); + return builder; + } + + public String getJobId() { + return jobId; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public void setPartitionFieldName(String partitionFieldName) { + this.partitionFieldName = partitionFieldName; + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public void setPartitionFieldValue(String partitionFieldValue) { + this.partitionFieldValue = partitionFieldValue; + } + + public String getOverFieldName() { + return overFieldName; + } + + public void setOverFieldName(String overFieldName) { + this.overFieldName = overFieldName; + } + + public String getOverFieldValue() { + return overFieldValue; + } + + public void setOverFieldValue(String overFieldValue) { + this.overFieldValue = overFieldValue; + } + + public String getByFieldName() { + return byFieldName; + } + + public void setByFieldName(String byFieldName) { + this.byFieldName = byFieldName; + } + + public String getByFieldValue() { + return byFieldValue; + } + + public void setByFieldValue(String byFieldValue) { + this.byFieldValue = byFieldValue; + } + + public String getDebugFeature() { + return debugFeature; + } + + public void setDebugFeature(String debugFeature) { + this.debugFeature = debugFeature; + } + + public double getDebugLower() { + return debugLower; + } + + public void setDebugLower(double debugLower) { + this.debugLower = debugLower; + } + + public double getDebugUpper() { + return debugUpper; + } + + public void setDebugUpper(double debugUpper) { + this.debugUpper = debugUpper; + } + + public double getDebugMedian() { + return debugMedian; + } + + public void setDebugMedian(double debugMedian) { + this.debugMedian = debugMedian; + } + + public double getActual() { + return actual; + } + + public void setActual(double actual) { + this.actual = actual; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other instanceof ModelDebugOutput == false) { + return false; + } + // id excluded here as it is generated by the datastore + ModelDebugOutput that = (ModelDebugOutput) other; + return Objects.equals(this.jobId, that.jobId) && + Objects.equals(this.timestamp, that.timestamp) && + Objects.equals(this.partitionFieldValue, that.partitionFieldValue) && + Objects.equals(this.partitionFieldName, that.partitionFieldName) && + Objects.equals(this.overFieldValue, that.overFieldValue) && + Objects.equals(this.overFieldName, that.overFieldName) && + Objects.equals(this.byFieldValue, that.byFieldValue) && + Objects.equals(this.byFieldName, that.byFieldName) && + Objects.equals(this.debugFeature, that.debugFeature) && + this.debugLower == that.debugLower && + this.debugUpper == that.debugUpper && + this.debugMedian == that.debugMedian && + this.actual == that.actual; + } + + @Override + public int hashCode() { + // id excluded here as it is generated by the datastore + return Objects.hash(jobId, timestamp, partitionFieldName, partitionFieldValue, + overFieldName, overFieldValue, byFieldName, byFieldValue, + debugFeature, debugLower, debugUpper, debugMedian, actual); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java new file mode 100644 index 00000000000..097ed71aa24 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +public class PartitionScore extends ToXContentToBytes implements Writeable { + public static final ParseField PARTITION_SCORE = new ParseField("partition_score"); + + private final String partitionFieldValue; + private final String partitionFieldName; + private final double initialAnomalyScore; + private double anomalyScore; + private double probability; + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + PARTITION_SCORE.getPreferredName(), a -> new PartitionScore((String) a[0], (String) a[1], (Double) a[2], (Double) a[3], + (Double) a[4])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_NAME); + PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), Bucket.INITIAL_ANOMALY_SCORE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.ANOMALY_SCORE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.PROBABILITY); + } + + public PartitionScore(String fieldName, String fieldValue, double initialAnomalyScore, double anomalyScore, double probability) { + partitionFieldName = fieldName; + partitionFieldValue = fieldValue; + this.initialAnomalyScore = initialAnomalyScore; + this.anomalyScore = anomalyScore; + this.probability = probability; + } + + public PartitionScore(StreamInput in) throws IOException { + partitionFieldName = in.readString(); + partitionFieldValue = in.readString(); + initialAnomalyScore = in.readDouble(); + anomalyScore = in.readDouble(); + probability = in.readDouble(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(partitionFieldName); + out.writeString(partitionFieldValue); + out.writeDouble(initialAnomalyScore); + out.writeDouble(anomalyScore); + out.writeDouble(probability); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); + builder.field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); + builder.field(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(AnomalyRecord.ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(AnomalyRecord.PROBABILITY.getPreferredName(), probability); + builder.endObject(); + return builder; + } + + public double getInitialAnomalyScore() { + return initialAnomalyScore; + } + + public double getAnomalyScore() { + return anomalyScore; + } + + public void setAnomalyScore(double anomalyScore) { + this.anomalyScore = anomalyScore; + } + + public String getPartitionFieldName() { + return partitionFieldName; + } + + public String getPartitionFieldValue() { + return partitionFieldValue; + } + + public double getProbability() { + return probability; + } + + public void setProbability(double probability) { + this.probability = probability; + } + + @Override + public int hashCode() { + return Objects.hash(partitionFieldName, partitionFieldValue, probability, initialAnomalyScore, anomalyScore); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof PartitionScore == false) { + return false; + } + + PartitionScore that = (PartitionScore) other; + + // id is excluded from the test as it is generated by the datastore + return Objects.equals(this.partitionFieldValue, that.partitionFieldValue) + && Objects.equals(this.partitionFieldName, that.partitionFieldName) && (this.probability == that.probability) + && (this.initialAnomalyScore == that.initialAnomalyScore) && (this.anomalyScore == that.anomalyScore); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java new file mode 100644 index 00000000000..895b48139c0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * When per-partition normalization is enabled this class represents + * the max anomalous probabilities of each partition per bucket. These values + * calculated from the bucket's anomaly records. + */ +public class PerPartitionMaxProbabilities extends ToXContentToBytes implements Writeable { + + /** + * Result type + */ + public static final String RESULT_TYPE_VALUE = "partition_normalized_probs"; + + /* + * Field Names + */ + public static final ParseField PER_PARTITION_MAX_PROBABILITIES = new ParseField("per_partition_max_probabilities"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> + new PerPartitionMaxProbabilities((String) a[0], (Date) a[1], (long) a[2], (List) a[3])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException( + "unexpected token [" + p.currentToken() + "] for [" + Bucket.TIMESTAMP.getPreferredName() + "]"); + }, Bucket.TIMESTAMP, ObjectParser.ValueType.VALUE); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), Bucket.BUCKET_SPAN); + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), PartitionProbability.PARSER, PER_PARTITION_MAX_PROBABILITIES); + PARSER.declareString((p, s) -> {}, Result.RESULT_TYPE); + } + + private final String jobId; + private final Date timestamp; + private final long bucketSpan; + private final List perPartitionMaxProbabilities; + + public PerPartitionMaxProbabilities(String jobId, Date timestamp, long bucketSpan, + List partitionProbabilities) { + this.jobId = jobId; + this.timestamp = timestamp; + this.bucketSpan = bucketSpan; + this.perPartitionMaxProbabilities = partitionProbabilities; + } + + public PerPartitionMaxProbabilities(List records) { + if (records.isEmpty()) { + throw new IllegalArgumentException("PerPartitionMaxProbabilities cannot be created from an empty list of records"); + } + this.jobId = records.get(0).getJobId(); + this.timestamp = records.get(0).getTimestamp(); + this.bucketSpan = records.get(0).getBucketSpan(); + this.perPartitionMaxProbabilities = calcMaxNormalizedProbabilityPerPartition(records); + } + + public PerPartitionMaxProbabilities(StreamInput in) throws IOException { + jobId = in.readString(); + timestamp = new Date(in.readLong()); + bucketSpan = in.readLong(); + perPartitionMaxProbabilities = in.readList(PartitionProbability::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(jobId); + out.writeLong(timestamp.getTime()); + out.writeLong(bucketSpan); + out.writeList(perPartitionMaxProbabilities); + } + + public String getJobId() { + return jobId; + } + + public String getId() { + return jobId + "_" + timestamp.getTime() + "_" + bucketSpan + "_" + RESULT_TYPE_VALUE; + } + + public Date getTimestamp() { + return timestamp; + } + + public List getPerPartitionMaxProbabilities() { + return perPartitionMaxProbabilities; + } + + public double getMaxProbabilityForPartition(String partitionValue) { + Optional first = + perPartitionMaxProbabilities.stream().filter(pp -> partitionValue.equals(pp.getPartitionValue())).findFirst(); + + return first.isPresent() ? first.get().getMaxNormalizedProbability() : 0.0; + } + + /** + * Box class for the stream collector function below + */ + private final class DoubleMaxBox { + private double value = 0.0; + + DoubleMaxBox() { + } + + public void accept(double d) { + if (d > value) { + value = d; + } + } + + public DoubleMaxBox combine(DoubleMaxBox other) { + return (this.value > other.value) ? this : other; + } + + public Double value() { + return this.value; + } + } + + private List calcMaxNormalizedProbabilityPerPartition(List anomalyRecords) { + Map maxValueByPartition = anomalyRecords.stream().collect( + Collectors.groupingBy(AnomalyRecord::getPartitionFieldValue, + Collector.of(DoubleMaxBox::new, (m, ar) -> m.accept(ar.getNormalizedProbability()), + DoubleMaxBox::combine, DoubleMaxBox::value))); + + List pProbs = new ArrayList<>(); + for (Map.Entry entry : maxValueByPartition.entrySet()) { + pProbs.add(new PartitionProbability(entry.getKey(), entry.getValue())); + } + + return pProbs; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + builder.dateField(Bucket.TIMESTAMP.getPreferredName(), Bucket.TIMESTAMP.getPreferredName() + "_string", timestamp.getTime()); + builder.field(Bucket.BUCKET_SPAN.getPreferredName(), bucketSpan); + builder.field(PER_PARTITION_MAX_PROBABILITIES.getPreferredName(), perPartitionMaxProbabilities); + builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, timestamp, perPartitionMaxProbabilities, bucketSpan); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof PerPartitionMaxProbabilities == false) { + return false; + } + + PerPartitionMaxProbabilities that = (PerPartitionMaxProbabilities) other; + + return Objects.equals(this.jobId, that.jobId) + && Objects.equals(this.timestamp, that.timestamp) + && this.bucketSpan == that.bucketSpan + && Objects.equals(this.perPartitionMaxProbabilities, that.perPartitionMaxProbabilities); + } + + /** + * Class for partitionValue, maxNormalizedProb pairs + */ + public static class PartitionProbability extends ToXContentToBytes implements Writeable { + + public static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("partitionProbability", + a -> new PartitionProbability((String) a[0], (double) a[1])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), Bucket.MAX_NORMALIZED_PROBABILITY); + } + + private final String partitionValue; + private final double maxNormalizedProbability; + + PartitionProbability(String partitionName, double maxNormalizedProbability) { + this.partitionValue = partitionName; + this.maxNormalizedProbability = maxNormalizedProbability; + } + + public PartitionProbability(StreamInput in) throws IOException { + partitionValue = in.readString(); + maxNormalizedProbability = in.readDouble(); + } + + public String getPartitionValue() { + return partitionValue; + } + + public double getMaxNormalizedProbability() { + return maxNormalizedProbability; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(partitionValue); + out.writeDouble(maxNormalizedProbability); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionValue) + .field(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability) + .endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(partitionValue, maxNormalizedProbability); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other instanceof PartitionProbability == false) { + return false; + } + + PartitionProbability that = (PartitionProbability) other; + + return Objects.equals(this.partitionValue, that.partitionValue) + && this.maxNormalizedProbability == that.maxNormalizedProbability; + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java new file mode 100644 index 00000000000..539e07d6948 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Pattern; + + +/** + * Defines the field names that we use for our results. + * Fields from the raw data with these names are not added to any result. Even + * different types of results will not have raw data fields with reserved names + * added to them, as it could create confusion if in some results a given field + * contains raw data and in others it contains some aspect of our output. + */ +public final class ReservedFieldNames { + private static final Pattern DOT_PATTERN = Pattern.compile("\\."); + + /** + * This array should be updated to contain all the field names that appear + * in any documents we store in our results index. (The reason it's any + * documents we store and not just results documents is that Elasticsearch + * 2.x requires mappings for given fields be consistent across all types + * in a given index.) + */ + private static final String[] RESERVED_FIELD_NAME_ARRAY = { + ElasticsearchMappings.ALL_FIELD_VALUES, + + Job.ID.getPreferredName(), + + AnomalyCause.PROBABILITY.getPreferredName(), + AnomalyCause.OVER_FIELD_NAME.getPreferredName(), + AnomalyCause.OVER_FIELD_VALUE.getPreferredName(), + AnomalyCause.BY_FIELD_NAME.getPreferredName(), + AnomalyCause.BY_FIELD_VALUE.getPreferredName(), + AnomalyCause.CORRELATED_BY_FIELD_VALUE.getPreferredName(), + AnomalyCause.PARTITION_FIELD_NAME.getPreferredName(), + AnomalyCause.PARTITION_FIELD_VALUE.getPreferredName(), + AnomalyCause.FUNCTION.getPreferredName(), + AnomalyCause.FUNCTION_DESCRIPTION.getPreferredName(), + AnomalyCause.TYPICAL.getPreferredName(), + AnomalyCause.ACTUAL.getPreferredName(), + AnomalyCause.INFLUENCERS.getPreferredName(), + AnomalyCause.FIELD_NAME.getPreferredName(), + + AnomalyRecord.DETECTOR_INDEX.getPreferredName(), + AnomalyRecord.PROBABILITY.getPreferredName(), + AnomalyRecord.BY_FIELD_NAME.getPreferredName(), + AnomalyRecord.BY_FIELD_VALUE.getPreferredName(), + AnomalyRecord.CORRELATED_BY_FIELD_VALUE.getPreferredName(), + AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName(), + AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), + AnomalyRecord.FUNCTION.getPreferredName(), + AnomalyRecord.FUNCTION_DESCRIPTION.getPreferredName(), + AnomalyRecord.TYPICAL.getPreferredName(), + AnomalyRecord.ACTUAL.getPreferredName(), + AnomalyRecord.IS_INTERIM.getPreferredName(), + AnomalyRecord.INFLUENCERS.getPreferredName(), + AnomalyRecord.FIELD_NAME.getPreferredName(), + AnomalyRecord.OVER_FIELD_NAME.getPreferredName(), + AnomalyRecord.OVER_FIELD_VALUE.getPreferredName(), + AnomalyRecord.CAUSES.getPreferredName(), + AnomalyRecord.ANOMALY_SCORE.getPreferredName(), + AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName(), + AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY.getPreferredName(), + AnomalyRecord.BUCKET_SPAN.getPreferredName(), + AnomalyRecord.SEQUENCE_NUM.getPreferredName(), + + Bucket.ANOMALY_SCORE.getPreferredName(), + Bucket.BUCKET_INFLUENCERS.getPreferredName(), + Bucket.BUCKET_SPAN.getPreferredName(), + Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), + Bucket.IS_INTERIM.getPreferredName(), + Bucket.RECORD_COUNT.getPreferredName(), + Bucket.EVENT_COUNT.getPreferredName(), + Bucket.INITIAL_ANOMALY_SCORE.getPreferredName(), + Bucket.PROCESSING_TIME_MS.getPreferredName(), + Bucket.PARTITION_SCORES.getPreferredName(), + Bucket.TIMESTAMP.getPreferredName(), + + BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName(), BucketInfluencer.ANOMALY_SCORE.getPreferredName(), + BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName(), BucketInfluencer.PROBABILITY.getPreferredName(), + + CategoryDefinition.CATEGORY_ID.getPreferredName(), + CategoryDefinition.TERMS.getPreferredName(), + CategoryDefinition.REGEX.getPreferredName(), + CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName(), + CategoryDefinition.EXAMPLES.getPreferredName(), + + DataCounts.PROCESSED_RECORD_COUNT.getPreferredName(), + DataCounts.PROCESSED_FIELD_COUNT.getPreferredName(), + DataCounts.INPUT_BYTES.getPreferredName(), + DataCounts.INPUT_RECORD_COUNT.getPreferredName(), + DataCounts.INPUT_FIELD_COUNT.getPreferredName(), + DataCounts.INVALID_DATE_COUNT.getPreferredName(), + DataCounts.MISSING_FIELD_COUNT.getPreferredName(), + DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName(), + DataCounts.LATEST_RECORD_TIME.getPreferredName(), + DataCounts.EARLIEST_RECORD_TIME.getPreferredName(), + + Influence.INFLUENCER_FIELD_NAME.getPreferredName(), + Influence.INFLUENCER_FIELD_VALUES.getPreferredName(), + + Influencer.PROBABILITY.getPreferredName(), + Influencer.INFLUENCER_FIELD_NAME.getPreferredName(), + Influencer.INFLUENCER_FIELD_VALUE.getPreferredName(), + Influencer.INITIAL_ANOMALY_SCORE.getPreferredName(), + Influencer.ANOMALY_SCORE.getPreferredName(), + Influencer.BUCKET_SPAN.getPreferredName(), + Influencer.SEQUENCE_NUM.getPreferredName(), + + ModelDebugOutput.PARTITION_FIELD_NAME.getPreferredName(), ModelDebugOutput.PARTITION_FIELD_VALUE.getPreferredName(), + ModelDebugOutput.OVER_FIELD_NAME.getPreferredName(), ModelDebugOutput.OVER_FIELD_VALUE.getPreferredName(), + ModelDebugOutput.BY_FIELD_NAME.getPreferredName(), ModelDebugOutput.BY_FIELD_VALUE.getPreferredName(), + ModelDebugOutput.DEBUG_FEATURE.getPreferredName(), ModelDebugOutput.DEBUG_LOWER.getPreferredName(), + ModelDebugOutput.DEBUG_UPPER.getPreferredName(), ModelDebugOutput.DEBUG_MEDIAN.getPreferredName(), + ModelDebugOutput.ACTUAL.getPreferredName(), + + ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName(), + ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), + ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), + ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), + ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName(), + ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName(), + ModelSizeStats.LOG_TIME_FIELD.getPreferredName(), + + ModelSnapshot.DESCRIPTION.getPreferredName(), + ModelSnapshot.RESTORE_PRIORITY.getPreferredName(), + ModelSnapshot.SNAPSHOT_ID.getPreferredName(), + ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName(), + ModelSnapshot.LATEST_RECORD_TIME.getPreferredName(), + ModelSnapshot.LATEST_RESULT_TIME.getPreferredName(), + + PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName(), + + Result.RESULT_TYPE.getPreferredName() + }; + + /** + * Test if fieldName is one of the reserved names or if it contains dots then + * that the segment before the first dot is not a reserved name. A fieldName + * containing dots represents nested fields in which case we only care about + * the top level. + * + * @param fieldName Document field name. This may contain dots '.' + * @return True if fieldName is not a reserved name or the top level segment + * is not a reserved name. + */ + public static boolean isValidFieldName(String fieldName) { + String[] segments = DOT_PATTERN.split(fieldName); + return !RESERVED_FIELD_NAMES.contains(segments[0]); + } + + /** + * A set of all reserved field names in our results. Fields from the raw + * data with these names are not added to any result. + */ + public static final Set RESERVED_FIELD_NAMES = new HashSet<>(Arrays.asList(RESERVED_FIELD_NAME_ARRAY)); + + private ReservedFieldNames() { + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Result.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Result.java new file mode 100644 index 00000000000..8dc25a32990 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/Result.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.results; + +import org.elasticsearch.common.ParseField; + +/** + * Common attributes of the result types + */ +public class Result { + + /** + * Serialisation fields + */ + public static final ParseField TYPE = new ParseField("result"); + public static final ParseField RESULT_TYPE = new ParseField("result_type"); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/AuditActivity.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/AuditActivity.java new file mode 100644 index 00000000000..159dcc909fb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/AuditActivity.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.notifications; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class AuditActivity extends ToXContentToBytes implements Writeable { + public static final ParseField TYPE = new ParseField("audit_activity"); + + public static final ParseField TOTAL_JOBS = new ParseField("total_jobs"); + public static final ParseField TOTAL_DETECTORS = new ParseField("total_detectors"); + public static final ParseField RUNNING_JOBS = new ParseField("running_jobs"); + public static final ParseField RUNNING_DETECTORS = new ParseField("running_detectors"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), + AuditActivity::new); + + static { + PARSER.declareInt(AuditActivity::setTotalJobs, TOTAL_JOBS); + PARSER.declareInt(AuditActivity::setTotalDetectors, TOTAL_DETECTORS); + PARSER.declareInt(AuditActivity::setRunningJobs, RUNNING_JOBS); + PARSER.declareInt(AuditActivity::setRunningDetectors, RUNNING_DETECTORS); + PARSER.declareField(AuditActivity::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + } + + private int totalJobs; + private int totalDetectors; + private int runningJobs; + private int runningDetectors; + private Date timestamp; + + public AuditActivity() { + } + + private AuditActivity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) { + this.totalJobs = totalJobs; + this.totalDetectors = totalDetectors; + this.runningJobs = runningJobs; + this.runningDetectors = runningDetectors; + timestamp = new Date(); + } + + public AuditActivity(StreamInput in) throws IOException { + totalJobs = in.readInt(); + totalDetectors = in.readInt(); + runningJobs = in.readInt(); + runningDetectors = in.readInt(); + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(totalJobs); + out.writeInt(totalDetectors); + out.writeInt(runningJobs); + out.writeInt(runningDetectors); + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + public int getTotalJobs() { + return totalJobs; + } + + public void setTotalJobs(int totalJobs) { + this.totalJobs = totalJobs; + } + + public int getTotalDetectors() { + return totalDetectors; + } + + public void setTotalDetectors(int totalDetectors) { + this.totalDetectors = totalDetectors; + } + + public int getRunningJobs() { + return runningJobs; + } + + public void setRunningJobs(int runningJobs) { + this.runningJobs = runningJobs; + } + + public int getRunningDetectors() { + return runningDetectors; + } + + public void setRunningDetectors(int runningDetectors) { + this.runningDetectors = runningDetectors; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public static AuditActivity newActivity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) { + return new AuditActivity(totalJobs, totalDetectors, runningJobs, runningDetectors); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(TOTAL_JOBS.getPreferredName(), totalJobs); + builder.field(TOTAL_DETECTORS.getPreferredName(), totalDetectors); + builder.field(RUNNING_JOBS.getPreferredName(), runningJobs); + builder.field(RUNNING_DETECTORS.getPreferredName(), runningDetectors); + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(totalDetectors, totalJobs, runningDetectors, runningJobs, timestamp); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AuditActivity other = (AuditActivity) obj; + return Objects.equals(totalDetectors, other.totalDetectors) && + Objects.equals(totalJobs, other.totalJobs) && + Objects.equals(runningDetectors, other.runningDetectors) && + Objects.equals(runningJobs, other.runningJobs) && + Objects.equals(timestamp, other.timestamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/AuditMessage.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/AuditMessage.java new file mode 100644 index 00000000000..4e4af11af5d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/AuditMessage.java @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.notifications; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.utils.time.TimeUtils; + +import java.io.IOException; +import java.util.Date; +import java.util.Objects; + +public class AuditMessage extends ToXContentToBytes implements Writeable { + public static final ParseField TYPE = new ParseField("audit_message"); + + public static final ParseField MESSAGE = new ParseField("message"); + public static final ParseField LEVEL = new ParseField("level"); + public static final ParseField TIMESTAMP = new ParseField("timestamp"); + + public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), + AuditMessage::new); + + static { + PARSER.declareString(AuditMessage::setJobId, Job.ID); + PARSER.declareString(AuditMessage::setMessage, MESSAGE); + PARSER.declareField(AuditMessage::setLevel, p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Level.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, LEVEL, ValueType.STRING); + PARSER.declareField(AuditMessage::setTimestamp, p -> { + if (p.currentToken() == Token.VALUE_NUMBER) { + return new Date(p.longValue()); + } else if (p.currentToken() == Token.VALUE_STRING) { + return new Date(TimeUtils.dateStringToEpoch(p.text())); + } + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + }, TIMESTAMP, ValueType.VALUE); + } + + private String jobId; + private String message; + private Level level; + private Date timestamp; + + public AuditMessage() { + // Default constructor + } + + private AuditMessage(String jobId, String message, Level level) { + this.jobId = jobId; + this.message = message; + this.level = level; + timestamp = new Date(); + } + + public AuditMessage(StreamInput in) throws IOException { + jobId = in.readOptionalString(); + message = in.readOptionalString(); + if (in.readBoolean()) { + level = Level.readFromStream(in); + } + if (in.readBoolean()) { + timestamp = new Date(in.readLong()); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeOptionalString(jobId); + out.writeOptionalString(message); + boolean hasLevel = level != null; + out.writeBoolean(hasLevel); + if (hasLevel) { + level.writeTo(out); + } + boolean hasTimestamp = timestamp != null; + out.writeBoolean(hasTimestamp); + if (hasTimestamp) { + out.writeLong(timestamp.getTime()); + } + } + + public String getJobId() { + return jobId; + } + + public void setJobId(String jobId) { + this.jobId = jobId; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Level getLevel() { + return level; + } + + public void setLevel(Level level) { + this.level = level; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public static AuditMessage newInfo(String jobId, String message) { + return new AuditMessage(jobId, message, Level.INFO); + } + + public static AuditMessage newWarning(String jobId, String message) { + return new AuditMessage(jobId, message, Level.WARNING); + } + + public static AuditMessage newActivity(String jobId, String message) { + return new AuditMessage(jobId, message, Level.ACTIVITY); + } + + public static AuditMessage newError(String jobId, String message) { + return new AuditMessage(jobId, message, Level.ERROR); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (jobId != null) { + builder.field(Job.ID.getPreferredName(), jobId); + } + if (message != null) { + builder.field(MESSAGE.getPreferredName(), message); + } + if (level != null) { + builder.field(LEVEL.getPreferredName(), level); + } + if (timestamp != null) { + builder.field(TIMESTAMP.getPreferredName(), timestamp.getTime()); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, message, level, timestamp); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + AuditMessage other = (AuditMessage) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(message, other.message) && + Objects.equals(level, other.level) && + Objects.equals(timestamp, other.timestamp); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java new file mode 100644 index 00000000000..00ea92ffa80 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/Auditor.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.notifications; + +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class Auditor { + + public static final String NOTIFICATIONS_INDEX = ".ml-notifications"; + private static final Logger LOGGER = Loggers.getLogger(Auditor.class); + + private final Client client; + private final String jobId; + + public Auditor(Client client, String jobId) { + this.client = Objects.requireNonNull(client); + this.jobId = jobId; + } + + public void info(String message) { + indexDoc(AuditMessage.TYPE.getPreferredName(), AuditMessage.newInfo(jobId, message)); + } + + public void warning(String message) { + indexDoc(AuditMessage.TYPE.getPreferredName(), AuditMessage.newWarning(jobId, message)); + } + + public void error(String message) { + indexDoc(AuditMessage.TYPE.getPreferredName(), AuditMessage.newError(jobId, message)); + } + + public void activity(int totalJobs, int totalDetectors, int runningJobs, int runningDetectors) { + String type = AuditActivity.TYPE.getPreferredName(); + indexDoc(type, AuditActivity.newActivity(totalJobs, totalDetectors, runningJobs, runningDetectors)); + } + + private void indexDoc(String type, ToXContent toXContent) { + // TODO: (norelease): Fix the assertion tripping in internal engine for index requests without an id being retried: + client.prepareIndex(NOTIFICATIONS_INDEX, type, UUIDs.base64UUID()) + .setSource(toXContentBuilder(toXContent)) + .execute(new ActionListener() { + @Override + public void onResponse(IndexResponse indexResponse) { + LOGGER.trace("Successfully persisted {}", type); + } + + @Override + public void onFailure(Exception e) { + LOGGER.error(new ParameterizedMessage("Error writing {}", new Object[]{true}, e)); + } + }); + } + + private XContentBuilder toXContentBuilder(ToXContent toXContent) { + try { + return toXContent.toXContent(jsonBuilder(), ToXContent.EMPTY_PARAMS); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/Level.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/Level.java new file mode 100644 index 00000000000..a283a253040 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/notifications/Level.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.notifications; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Locale; + +public enum Level implements Writeable { + INFO, ACTIVITY, WARNING, ERROR; + + /** + * Case-insensitive from string method. + * + * @param value + * String representation + * @return The condition type + */ + public static Level fromString(String value) { + return Level.valueOf(value.toUpperCase(Locale.ROOT)); + } + + public static Level readFromStream(StreamInput in) throws IOException { + int ordinal = in.readVInt(); + if (ordinal < 0 || ordinal >= values().length) { + throw new IOException("Unknown Level ordinal [" + ordinal + "]"); + } + return values()[ordinal]; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(ordinal()); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java new file mode 100644 index 00000000000..f901fbd40fc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestDeleteDatafeedAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.datafeeds; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.io.IOException; + +public class RestDeleteDatafeedAction extends BaseRestHandler { + + public RestDeleteDatafeedAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, MlPlugin.BASE_PATH + "datafeeds/{" + + DatafeedConfig.ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + DeleteDatafeedAction.Request deleteDatafeedRequest = new DeleteDatafeedAction.Request(datafeedId); + return channel -> client.execute(DeleteDatafeedAction.INSTANCE, deleteDatafeedRequest, new AcknowledgedRestListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java new file mode 100644 index 00000000000..aca00585445 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedStatsAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.datafeeds; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetDatafeedsStatsAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.io.IOException; + +public class RestGetDatafeedStatsAction extends BaseRestHandler { + + public RestGetDatafeedStatsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "datafeeds/{" + DatafeedConfig.ID.getPreferredName() + "}/_stats", this); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "datafeeds/_stats", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + if (Strings.isNullOrEmpty(datafeedId)) { + datafeedId = GetDatafeedsStatsAction.ALL; + } + GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); + return channel -> client.execute(GetDatafeedsStatsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java new file mode 100644 index 00000000000..242ede7ae4a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestGetDatafeedsAction.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.datafeeds; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.io.IOException; + +public class RestGetDatafeedsAction extends BaseRestHandler { + + public RestGetDatafeedsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "datafeeds/{" + DatafeedConfig.ID.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "datafeeds", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + if (datafeedId == null) { + datafeedId = GetDatafeedsAction.ALL; + } + GetDatafeedsAction.Request request = new GetDatafeedsAction.Request(datafeedId); + return channel -> client.execute(GetDatafeedsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java new file mode 100644 index 00000000000..bc63f2694e3 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.datafeeds; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.PutDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.io.IOException; + +public class RestPutDatafeedAction extends BaseRestHandler { + + public RestPutDatafeedAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, MlPlugin.BASE_PATH + "datafeeds/{" + + DatafeedConfig.ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + XContentParser parser = restRequest.contentParser(); + PutDatafeedAction.Request putDatafeedRequest = PutDatafeedAction.Request.parseRequest(datafeedId, parser); + return channel -> client.execute(PutDatafeedAction.INSTANCE, putDatafeedRequest, new RestToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java new file mode 100644 index 00000000000..6db1572f955 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStartDatafeedAction.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.datafeeds; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.persistent.PersistentActionResponse; + +import java.io.IOException; + +public class RestStartDatafeedAction extends BaseRestHandler { + + private static final String DEFAULT_START = "0"; + + public RestStartDatafeedAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "datafeeds/{" + DatafeedConfig.ID.getPreferredName() + "}/_start", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + StartDatafeedAction.Request jobDatafeedRequest; + if (restRequest.hasContentOrSourceParam()) { + XContentParser parser = restRequest.contentOrSourceParamParser(); + jobDatafeedRequest = StartDatafeedAction.Request.parseRequest(datafeedId, parser); + } else { + long startTimeMillis = parseDateOrThrow(restRequest.param(StartDatafeedAction.START_TIME.getPreferredName(), + DEFAULT_START), StartDatafeedAction.START_TIME.getPreferredName()); + Long endTimeMillis = null; + if (restRequest.hasParam(StartDatafeedAction.END_TIME.getPreferredName())) { + endTimeMillis = parseDateOrThrow(restRequest.param(StartDatafeedAction.END_TIME.getPreferredName()), + StartDatafeedAction.END_TIME.getPreferredName()); + } + jobDatafeedRequest = new StartDatafeedAction.Request(datafeedId, startTimeMillis); + jobDatafeedRequest.setEndTime(endTimeMillis); + } + return channel -> { + client.execute(StartDatafeedAction.INSTANCE, jobDatafeedRequest, + new RestBuilderListener(channel) { + + @Override + public RestResponse buildResponse(PersistentActionResponse r, XContentBuilder builder) throws Exception { + builder.startObject(); + builder.field("started", true); + builder.endObject(); + return new BytesRestResponse(RestStatus.OK, builder); + } + }); + }; + } + + static long parseDateOrThrow(String date, String paramName) { + try { + return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseMillis(date); + } catch (IllegalArgumentException e) { + String msg = Messages.getMessage(Messages.REST_INVALID_DATETIME_PARAMS, paramName, date); + throw new ElasticsearchParseException(msg, e); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java new file mode 100644 index 00000000000..6e2fb892af6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestStopDatafeedAction.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.datafeeds; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.StopDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; + +import java.io.IOException; + +public class RestStopDatafeedAction extends BaseRestHandler { + + public RestStopDatafeedAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "datafeeds/{" + + DatafeedConfig.ID.getPreferredName() + "}/_stop", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + StopDatafeedAction.Request jobDatafeedRequest = new StopDatafeedAction.Request( + restRequest.param(DatafeedConfig.ID.getPreferredName())); + return channel -> client.execute(StopDatafeedAction.INSTANCE, jobDatafeedRequest, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java new file mode 100644 index 00000000000..7760016b1b9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestDeleteFilterAction.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.filter; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.DeleteFilterAction; +import org.elasticsearch.xpack.ml.action.DeleteFilterAction.Request; + +import java.io.IOException; + +public class RestDeleteFilterAction extends BaseRestHandler { + + public RestDeleteFilterAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, + MlPlugin.BASE_PATH + "filters/{" + Request.FILTER_ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + Request request = new Request(restRequest.param(Request.FILTER_ID.getPreferredName())); + return channel -> client.execute(DeleteFilterAction.INSTANCE, request, new AcknowledgedRestListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java new file mode 100644 index 00000000000..e6b2c5c33db --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestGetFiltersAction.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.filter; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetFiltersAction; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.job.config.MlFilter; + +import java.io.IOException; + +public class RestGetFiltersAction extends BaseRestHandler { + + public RestGetFiltersAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + "filters/{" + MlFilter.ID.getPreferredName() + "}", + this); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + "filters/", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + GetFiltersAction.Request getListRequest = new GetFiltersAction.Request(); + String filterId = restRequest.param(MlFilter.ID.getPreferredName()); + if (!Strings.isNullOrEmpty(filterId)) { + getListRequest.setFilterId(filterId); + } + if (restRequest.hasParam(PageParams.FROM.getPreferredName()) + || restRequest.hasParam(PageParams.SIZE.getPreferredName()) + || Strings.isNullOrEmpty(filterId)) { + getListRequest.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + } + return channel -> client.execute(GetFiltersAction.INSTANCE, getListRequest, new RestStatusToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java new file mode 100644 index 00000000000..08c24257f65 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/filter/RestPutFilterAction.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.filter; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.PutFilterAction; + +import java.io.IOException; + +public class RestPutFilterAction extends BaseRestHandler { + + public RestPutFilterAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, MlPlugin.BASE_PATH + "filters", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + XContentParser parser = restRequest.contentOrSourceParamParser(); + PutFilterAction.Request putListRequest = PutFilterAction.Request.parseRequest(parser); + return channel -> client.execute(PutFilterAction.INSTANCE, putListRequest, new AcknowledgedRestListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java new file mode 100644 index 00000000000..cb2bd72ec55 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestCloseJobAction.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.CloseJobAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestCloseJobAction extends BaseRestHandler { + + public RestCloseJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_close", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + CloseJobAction.Request request = new CloseJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + if (restRequest.hasParam("close_timeout")) { + request.setCloseTimeout(TimeValue.parseTimeValue(restRequest.param("close_timeout"), "close_timeout")); + } + return channel -> client.execute(CloseJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java new file mode 100644 index 00000000000..d988316ab37 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestDeleteJobAction.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.DeleteJobAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestDeleteJobAction extends BaseRestHandler { + + public RestDeleteJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + DeleteJobAction.Request deleteJobRequest = new DeleteJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + return channel -> client.execute(DeleteJobAction.INSTANCE, deleteJobRequest, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java new file mode 100644 index 00000000000..af5575ff05b --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestFlushJobAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.FlushJobAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestFlushJobAction extends BaseRestHandler { + + private final boolean DEFAULT_CALC_INTERIM = false; + private final String DEFAULT_START = ""; + private final String DEFAULT_END = ""; + private final String DEFAULT_ADVANCE_TIME = ""; + + public RestFlushJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_flush", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + final FlushJobAction.Request request; + if (restRequest.hasContentOrSourceParam()) { + XContentParser parser = restRequest.contentOrSourceParamParser(); + request = FlushJobAction.Request.parseRequest(jobId, parser); + } else { + request = new FlushJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + request.setCalcInterim(restRequest.paramAsBoolean(FlushJobAction.Request.CALC_INTERIM.getPreferredName(), + DEFAULT_CALC_INTERIM)); + request.setStart(restRequest.param(FlushJobAction.Request.START.getPreferredName(), DEFAULT_START)); + request.setEnd(restRequest.param(FlushJobAction.Request.END.getPreferredName(), DEFAULT_END)); + request.setAdvanceTime(restRequest.param(FlushJobAction.Request.ADVANCE_TIME.getPreferredName(), DEFAULT_ADVANCE_TIME)); + } + + return channel -> client.execute(FlushJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java new file mode 100644 index 00000000000..fcba341e3c4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobStatsAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetJobsStatsAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestGetJobStatsAction extends BaseRestHandler { + + public RestGetJobStatsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_stats", this); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "anomaly_detectors/_stats", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + if (Strings.isNullOrEmpty(jobId)) { + jobId = Job.ALL; + } + GetJobsStatsAction.Request request = new GetJobsStatsAction.Request(jobId); + return channel -> client.execute(GetJobsStatsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java new file mode 100644 index 00000000000..f98722ba855 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestGetJobsAction.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetJobsAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestGetJobsAction extends BaseRestHandler { + + public RestGetJobsAction(Settings settings, RestController controller) { + super(settings); + + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + + "anomaly_detectors", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + if (Strings.isNullOrEmpty(jobId)) { + jobId = Job.ALL; + } + GetJobsAction.Request request = new GetJobsAction.Request(jobId); + return channel -> client.execute(GetJobsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java new file mode 100644 index 00000000000..f75e1c9c082 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestOpenJobAction.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.OpenJobAction; +import org.elasticsearch.xpack.ml.action.PostDataAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestOpenJobAction extends BaseRestHandler { + + public RestOpenJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_open", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + OpenJobAction.Request request = new OpenJobAction.Request(restRequest.param(Job.ID.getPreferredName())); + request.setIgnoreDowntime(restRequest.paramAsBoolean(OpenJobAction.Request.IGNORE_DOWNTIME.getPreferredName(), false)); + if (restRequest.hasParam("open_timeout")) { + TimeValue openTimeout = restRequest.paramAsTime("open_timeout", TimeValue.timeValueSeconds(30)); + request.setOpenTimeout(openTimeout); + } + return channel -> { + client.execute(OpenJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); + }; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java new file mode 100644 index 00000000000..5d89dd1f56e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostDataAction.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.PostDataAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestPostDataAction extends BaseRestHandler { + + private static final String DEFAULT_RESET_START = ""; + private static final String DEFAULT_RESET_END = ""; + + public RestPostDataAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_data", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + PostDataAction.Request request = new PostDataAction.Request(restRequest.param(Job.ID.getPreferredName())); + request.setResetStart(restRequest.param(PostDataAction.Request.RESET_START.getPreferredName(), DEFAULT_RESET_START)); + request.setResetEnd(restRequest.param(PostDataAction.Request.RESET_END.getPreferredName(), DEFAULT_RESET_END)); + request.setContent(restRequest.content()); + + return channel -> client.execute(PostDataAction.INSTANCE, request, new RestStatusToXContentListener<>(channel)); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java new file mode 100644 index 00000000000..e2267003af1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPostJobUpdateAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestPostJobUpdateAction extends BaseRestHandler { + public RestPostJobUpdateAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/_update", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + XContentParser parser = restRequest.contentParser(); + UpdateJobAction.Request updateJobRequest = UpdateJobAction.Request.parseRequest(jobId, parser); + return channel -> client.execute(UpdateJobAction.INSTANCE, updateJobRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java new file mode 100644 index 00000000000..3f9a932df20 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/job/RestPutJobAction.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.job; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestPutJobAction extends BaseRestHandler { + + public RestPutJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + XContentParser parser = restRequest.contentParser(); + PutJobAction.Request putJobRequest = PutJobAction.Request.parseRequest(jobId, parser); + return channel -> client.execute(PutJobAction.INSTANCE, putJobRequest, new RestToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java new file mode 100644 index 00000000000..a1029b3222d --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestDeleteModelSnapshotAction.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.DeleteModelSnapshotAction; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; + +import java.io.IOException; + +public class RestDeleteModelSnapshotAction extends BaseRestHandler { + + public RestDeleteModelSnapshotAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/model_snapshots/{" + ModelSnapshot.SNAPSHOT_ID.getPreferredName() + "}", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + DeleteModelSnapshotAction.Request deleteModelSnapshot = new DeleteModelSnapshotAction.Request( + restRequest.param(Job.ID.getPreferredName()), + restRequest.param(ModelSnapshot.SNAPSHOT_ID.getPreferredName())); + + return channel -> client.execute(DeleteModelSnapshotAction.INSTANCE, deleteModelSnapshot, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java new file mode 100644 index 00000000000..9e106dd5425 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestGetModelSnapshotsAction.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetModelSnapshotsAction; +import org.elasticsearch.xpack.ml.action.GetModelSnapshotsAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.action.util.PageParams; + +import java.io.IOException; + +public class RestGetModelSnapshotsAction extends BaseRestHandler { + + private final String ALL = "_all"; + private final String ALL_SNAPSHOT_IDS = null; + + // Even though these are null, setting up the defaults in case + // we want to change them later + private final String DEFAULT_SORT = null; + private final String DEFAULT_START = null; + private final String DEFAULT_END = null; + private final String DEFAULT_DESCRIPTION = null; + private final boolean DEFAULT_DESC_ORDER = true; + + public RestGetModelSnapshotsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/model_snapshots/{" + Request.SNAPSHOT_ID.getPreferredName() + "}", this); + // endpoints that support body parameters must also accept POST + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/model_snapshots/{" + Request.SNAPSHOT_ID.getPreferredName() + "}", this); + + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/model_snapshots", this); + // endpoints that support body parameters must also accept POST + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/model_snapshots", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + String snapshotId = restRequest.param(Request.SNAPSHOT_ID.getPreferredName()); + if (ALL.equals(snapshotId)) { + snapshotId = ALL_SNAPSHOT_IDS; + } + Request getModelSnapshots; + if (restRequest.hasContentOrSourceParam()) { + XContentParser parser = restRequest.contentOrSourceParamParser(); + getModelSnapshots = Request.parseRequest(jobId, snapshotId, parser); + } else { + getModelSnapshots = new Request(jobId, snapshotId); + getModelSnapshots.setSort(restRequest.param(Request.SORT.getPreferredName(), DEFAULT_SORT)); + if (restRequest.hasParam(Request.START.getPreferredName())) { + getModelSnapshots.setStart(restRequest.param(Request.START.getPreferredName(), DEFAULT_START)); + } + if (restRequest.hasParam(Request.END.getPreferredName())) { + getModelSnapshots.setEnd(restRequest.param(Request.END.getPreferredName(), DEFAULT_END)); + } + if (restRequest.hasParam(Request.DESCRIPTION.getPreferredName())) { + getModelSnapshots.setDescriptionString(restRequest.param(Request.DESCRIPTION.getPreferredName(), DEFAULT_DESCRIPTION)); + } + getModelSnapshots.setDescOrder(restRequest.paramAsBoolean(Request.DESC.getPreferredName(), DEFAULT_DESC_ORDER)); + getModelSnapshots.setPageParams(new PageParams( + restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + } + + return channel -> client.execute(GetModelSnapshotsAction.INSTANCE, getModelSnapshots, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java new file mode 100644 index 00000000000..2febb5094de --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestRevertModelSnapshotAction.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; + +public class RestRevertModelSnapshotAction extends BaseRestHandler { + + private final boolean DELETE_INTERVENING_DEFAULT = false; + + public RestRevertModelSnapshotAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/model_snapshots/{" + + RevertModelSnapshotAction.Request.SNAPSHOT_ID.getPreferredName() + "}/_revert", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + String snapshotId = restRequest.param(RevertModelSnapshotAction.Request.SNAPSHOT_ID.getPreferredName()); + RevertModelSnapshotAction.Request request; + if (restRequest.hasContentOrSourceParam()) { + XContentParser parser = restRequest.contentOrSourceParamParser(); + request = RevertModelSnapshotAction.Request.parseRequest(jobId, snapshotId, parser); + } else { + request = new RevertModelSnapshotAction.Request(jobId, snapshotId); + request.setDeleteInterveningResults(restRequest + .paramAsBoolean(RevertModelSnapshotAction.Request.DELETE_INTERVENING.getPreferredName(), DELETE_INTERVENING_DEFAULT)); + } + return channel -> client.execute(RevertModelSnapshotAction.INSTANCE, request, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java new file mode 100644 index 00000000000..67e170bab72 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/modelsnapshots/RestUpdateModelSnapshotAction.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.modelsnapshots; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestStatusToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; + +import java.io.IOException; + +public class RestUpdateModelSnapshotAction extends BaseRestHandler { + + public RestUpdateModelSnapshotAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/model_snapshots/{" + ModelSnapshot.SNAPSHOT_ID +"}/_update", + this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + XContentParser parser = restRequest.contentParser(); + UpdateModelSnapshotAction.Request updateModelSnapshot = UpdateModelSnapshotAction.Request.parseRequest( + restRequest.param(Job.ID.getPreferredName()), + restRequest.param(ModelSnapshot.SNAPSHOT_ID.getPreferredName()), + parser); + + return channel -> + client.execute(UpdateModelSnapshotAction.INSTANCE, updateModelSnapshot, new RestStatusToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java new file mode 100644 index 00000000000..7a9957437e0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetBucketsAction; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.action.util.PageParams; + +import java.io.IOException; + +public class RestGetBucketsAction extends BaseRestHandler { + + public RestGetBucketsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + + "}/results/buckets/{" + Bucket.TIMESTAMP.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + + "}/results/buckets/{" + Bucket.TIMESTAMP.getPreferredName() + "}", this); + + controller.registerHandler(RestRequest.Method.GET, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/buckets", this); + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/buckets", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + final GetBucketsAction.Request request; + if (restRequest.hasContent()) { + XContentParser parser = restRequest.contentParser(); + request = GetBucketsAction.Request.parseRequest(jobId, parser); + } else { + request = new GetBucketsAction.Request(jobId); + + // Check if the REST param is set first so mutually exclusive + // options will only cause an error if set + if (restRequest.hasParam(GetBucketsAction.Request.TIMESTAMP.getPreferredName())) { + String timestamp = restRequest.param(GetBucketsAction.Request.TIMESTAMP.getPreferredName()); + if (timestamp != null && !timestamp.isEmpty()) { + request.setTimestamp(timestamp); + } + } + // multiple bucket options + if (restRequest.hasParam(PageParams.FROM.getPreferredName()) || restRequest.hasParam(PageParams.SIZE.getPreferredName())) { + request.setPageParams( + new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + } + if (restRequest.hasParam(GetBucketsAction.Request.START.getPreferredName())) { + request.setStart(restRequest.param(GetBucketsAction.Request.START.getPreferredName())); + } + if (restRequest.hasParam(GetBucketsAction.Request.END.getPreferredName())) { + request.setEnd(restRequest.param(GetBucketsAction.Request.END.getPreferredName())); + } + if (restRequest.hasParam(GetBucketsAction.Request.ANOMALY_SCORE.getPreferredName())) { + request.setAnomalyScore( + Double.parseDouble(restRequest.param(GetBucketsAction.Request.ANOMALY_SCORE.getPreferredName(), "0.0"))); + } + if (restRequest.hasParam(GetBucketsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName())) { + request.setMaxNormalizedProbability( + Double.parseDouble(restRequest.param( + GetBucketsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName(), "0.0"))); + } + if (restRequest.hasParam(GetBucketsAction.Request.PARTITION_VALUE.getPreferredName())) { + request.setPartitionValue(restRequest.param(GetBucketsAction.Request.PARTITION_VALUE.getPreferredName())); + } + + // single and multiple bucket options + request.setExpand(restRequest.paramAsBoolean(GetBucketsAction.Request.EXPAND.getPreferredName(), false)); + request.setIncludeInterim(restRequest.paramAsBoolean(GetBucketsAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + } + + return channel -> client.execute(GetBucketsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java new file mode 100644 index 00000000000..0c1a92ee38a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetCategoriesAction.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetCategoriesAction; +import org.elasticsearch.xpack.ml.action.GetCategoriesAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.action.util.PageParams; + +import java.io.IOException; + +public class RestGetCategoriesAction extends BaseRestHandler { + + public RestGetCategoriesAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/categories/{" + + Request.CATEGORY_ID.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.GET, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/categories", this); + + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/categories/{" + + Request.CATEGORY_ID.getPreferredName() + "}", this); + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/categories", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + Request request; + String jobId = restRequest.param(Job.ID.getPreferredName()); + String categoryId = restRequest.param(Request.CATEGORY_ID.getPreferredName()); + BytesReference bodyBytes = restRequest.content(); + + if (bodyBytes != null && bodyBytes.length() > 0) { + XContentParser parser = restRequest.contentParser(); + request = GetCategoriesAction.Request.parseRequest(jobId, parser); + request.setCategoryId(categoryId); + } else { + + request = new Request(jobId); + if (!Strings.isNullOrEmpty(categoryId)) { + request.setCategoryId(categoryId); + } + if (restRequest.hasParam(Request.FROM.getPreferredName()) + || restRequest.hasParam(Request.SIZE.getPreferredName()) + || Strings.isNullOrEmpty(categoryId)){ + + request.setPageParams(new PageParams( + restRequest.paramAsInt(Request.FROM.getPreferredName(), 0), + restRequest.paramAsInt(Request.SIZE.getPreferredName(), 100) + )); + } + } + + return channel -> client.execute(GetCategoriesAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java new file mode 100644 index 00000000000..de3607b05f4 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetInfluencersAction; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.action.util.PageParams; + +import java.io.IOException; + +public class RestGetInfluencersAction extends BaseRestHandler { + + public RestGetInfluencersAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/influencers", this); + // endpoints that support body parameters must also accept POST + controller.registerHandler(RestRequest.Method.POST, + MlPlugin.BASE_PATH + "anomaly_detectors/{" + Job.ID.getPreferredName() + "}/results/influencers", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + String start = restRequest.param(GetInfluencersAction.Request.START.getPreferredName()); + String end = restRequest.param(GetInfluencersAction.Request.END.getPreferredName()); + final GetInfluencersAction.Request request; + if (restRequest.hasContent()) { + XContentParser parser = restRequest.contentParser(); + request = GetInfluencersAction.Request.parseRequest(jobId, parser); + } else { + request = new GetInfluencersAction.Request(jobId); + request.setStart(start); + request.setEnd(end); + request.setIncludeInterim(restRequest.paramAsBoolean(GetInfluencersAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + request.setAnomalyScore( + Double.parseDouble(restRequest.param(GetInfluencersAction.Request.ANOMALY_SCORE.getPreferredName(), "0.0"))); + request.setSort(restRequest.param(GetInfluencersAction.Request.SORT_FIELD.getPreferredName(), + Influencer.ANOMALY_SCORE.getPreferredName())); + request.setDecending(restRequest.paramAsBoolean(GetInfluencersAction.Request.DESCENDING_SORT.getPreferredName(), true)); + } + + return channel -> client.execute(GetInfluencersAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java new file mode 100644 index 00000000000..baa5dfb1707 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.results; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.GetRecordsAction; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.action.util.PageParams; + +import java.io.IOException; + +public class RestGetRecordsAction extends BaseRestHandler { + + public RestGetRecordsAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.GET, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/results/records", this); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "anomaly_detectors/{" + + Job.ID.getPreferredName() + "}/results/records", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String jobId = restRequest.param(Job.ID.getPreferredName()); + final GetRecordsAction.Request request; + if (restRequest.hasContent()) { + XContentParser parser = restRequest.contentParser(); + request = GetRecordsAction.Request.parseRequest(jobId, parser); + } + else { + request = new GetRecordsAction.Request(jobId); + request.setStart(restRequest.param(GetRecordsAction.Request.START.getPreferredName())); + request.setEnd(restRequest.param(GetRecordsAction.Request.END.getPreferredName())); + request.setIncludeInterim(restRequest.paramAsBoolean(GetRecordsAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); + request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), + restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); + request.setAnomalyScore( + Double.parseDouble(restRequest.param(GetRecordsAction.Request.ANOMALY_SCORE_FILTER.getPreferredName(), "0.0"))); + request.setSort(restRequest.param(GetRecordsAction.Request.SORT.getPreferredName(), + AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName())); + request.setDecending(restRequest.paramAsBoolean(GetRecordsAction.Request.DESCENDING.getPreferredName(), true)); + request.setMaxNormalizedProbability( + Double.parseDouble(restRequest.param(GetRecordsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName(), "0.0"))); + String partitionValue = restRequest.param(GetRecordsAction.Request.PARTITION_VALUE.getPreferredName()); + if (partitionValue != null) { + request.setPartitionValue(partitionValue); + } + } + + return channel -> client.execute(GetRecordsAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java new file mode 100644 index 00000000000..77ab16f8be1 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateDetectorAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.validate; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.ValidateDetectorAction; + +import java.io.IOException; + +public class RestValidateDetectorAction extends BaseRestHandler { + + public RestValidateDetectorAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "anomaly_detectors/_validate/detector", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + XContentParser parser = restRequest.contentOrSourceParamParser(); + ValidateDetectorAction.Request validateDetectorRequest = ValidateDetectorAction.Request.parseRequest(parser); + return channel -> + client.execute(ValidateDetectorAction.INSTANCE, validateDetectorRequest, new AcknowledgedRestListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java new file mode 100644 index 00000000000..2adf957e484 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/rest/validate/RestValidateJobConfigAction.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.validate; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.ValidateJobConfigAction; + +import java.io.IOException; + +public class RestValidateJobConfigAction extends BaseRestHandler { + + public RestValidateJobConfigAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.POST, MlPlugin.BASE_PATH + "anomaly_detectors/_validate", this); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + XContentParser parser = restRequest.contentOrSourceParamParser(); + ValidateJobConfigAction.Request validateConfigRequest = ValidateJobConfigAction.Request.parseRequest(parser); + return channel -> + client.execute(ValidateJobConfigAction.INSTANCE, validateConfigRequest, new AcknowledgedRestListener<>(channel)); + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java new file mode 100644 index 00000000000..22539124dcf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/DomainSplitFunction.java @@ -0,0 +1,277 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils; + +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.ResourceBundle; + +public final class DomainSplitFunction { + + public static final String function; + public static final Map params; + + DomainSplitFunction() {} + + static { + Map paramsMap = new HashMap<>(); + + ResourceBundle resource = ResourceBundle.getBundle("org/elasticsearch/xpack/ml/transforms/exact", Locale.getDefault()); + Enumeration keys = resource.getKeys(); + Map exact = new HashMap<>(2048); + while (keys.hasMoreElements()) { + String key = keys.nextElement(); + String value = resource.getString(key); + exact.put(key, value); + } + exact = Collections.unmodifiableMap(exact); + + Map under = new HashMap<>(30); + under.put("bd", "i"); + under.put("np", "i"); + under.put("jm", "i"); + under.put("fj", "i"); + under.put("fk", "i"); + under.put("ye", "i"); + under.put("sch.uk", "i"); + under.put("bn", "i"); + under.put("kitakyushu.jp", "i"); + under.put("kobe.jp", "i"); + under.put("ke", "i"); + under.put("sapporo.jp", "i"); + under.put("kh", "i"); + under.put("mm", "i"); + under.put("il", "i"); + under.put("yokohama.jp", "i"); + under.put("ck", "i"); + under.put("nagoya.jp", "i"); + under.put("sendai.jp", "i"); + under.put("kw", "i"); + under.put("er", "i"); + under.put("mz", "i"); + under.put("platform.sh", "p"); + under.put("gu", "i"); + under.put("nom.br", "i"); + under.put("zm", "i"); + under.put("pg", "i"); + under.put("ni", "i"); + under.put("kawasaki.jp", "i"); + under.put("zw", "i"); + under = Collections.unmodifiableMap(under); + + Map excluded = new HashMap<>(9); + excluded.put("city.yokohama.jp", "i"); + excluded.put("teledata.mz", "i"); + excluded.put("city.kobe.jp", "i"); + excluded.put("city.sapporo.jp", "i"); + excluded.put("city.kawasaki.jp", "i"); + excluded.put("city.nagoya.jp", "i"); + excluded.put("www.ck", "i"); + excluded.put("city.sendai.jp", "i"); + excluded.put("city.kitakyushu.jp", "i"); + excluded = Collections.unmodifiableMap(excluded); + + + paramsMap.put("excluded", excluded); + paramsMap.put("under", under); + paramsMap.put("exact", exact); + params = Collections.unmodifiableMap(paramsMap); + } + + static { + String fn = "String replaceDots(String input) {\n" + + " String output = input;\n" + + " if (output.indexOf('。') >= 0) {\n" + + " output = output.replace('。', '.');\n" + + " }\n" + + " if (output.indexOf('．') >= 0) {\n" + + " output = output.replace('．', '.');\n" + + " }\n" + + " if (output.indexOf('｡') >= 0) {\n" + + " output = output.replace('｡', '.');\n" + + " }\n" + + " return output;\n" + + "}\n" + + "List split(String value) {\n" + + " int nextWord = 0;\n" + + " List splits = [];\n" + + " for(int i = 0; i < value.length(); i++) {\n" + + " if(value.charAt(i) == (char)'.') {\n" + + " splits.add(value.substring(nextWord, i));\n" + + " nextWord = i+1;\n" + + " }\n" + + " }\n" + + " if (nextWord != value.length()) {\n" + + " splits.add(value.substring(nextWord, value.length()));\n" + + " }\n" + + " return splits;\n" + + "}\n" + + "List splitDomain(String domain) {\n" + + " String dotDomain = replaceDots(domain);\n" + + " return split(dotDomain);\n" + + "}\n" + + "boolean validateSyntax(List parts) {\n" + + " int lastIndex = parts.length - 1;\n" + + " /* Validate the last part specially, as it has different syntax rules. */\n" + + " if (!validatePart(parts[lastIndex], true)) {\n" + + " return false;\n" + + " }\n" + + " for (int i = 0; i < lastIndex; i++) {\n" + + " String part = parts[i];\n" + + " if (!validatePart(part, false)) {\n" + + " return false;\n" + + " }\n" + + " }\n" + + " return true;\n" + + "}\n" + + "boolean validatePart(String part, boolean isFinalPart) {\n" + + " int MAX_DOMAIN_PART_LENGTH = 63;\n" + + " if (part.length() < 1 || part.length() > MAX_DOMAIN_PART_LENGTH) {\n" + + " return false;\n" + + " }\n" + + " int offset = 0;\n" + + " int strLen = part.length();\n" + + " while (offset < strLen) {\n" + + " int curChar = part.charAt(offset);\n" + + " offset += 1;\n" + + " if (!(Character.isLetterOrDigit(curChar) || curChar == (char)'-' || curChar == (char)'_')) {\n" + + " return false;\n" + + " }\n" + + " }\n" + + " if (part.charAt(0) == (char)'-' || part.charAt(0) == (char)'_' ||\n" + + " part.charAt(part.length() - 1) == (char)'-' || part.charAt(part.length() - 1) == (char)'_') {\n" + + " return false;\n" + + " }\n" + + " if (isFinalPart && Character.isDigit(part.charAt(0))) {\n" + + " return false;\n" + + " }\n" + + " return true;\n" + + "}\n" + + "int findPublicSuffix(Map params, List parts) {\n" + + " int partsSize = parts.size();\n" + + "\n" + + " for (int i = 0; i < partsSize; i++) {\n" + + " StringJoiner joiner = new StringJoiner('.');\n" + + " for (String s : parts.subList(i, partsSize)) {\n" + + " joiner.add(s);\n" + + " }\n" + + " /* parts.subList(i, partsSize).each(joiner::add); */\n" + + " String ancestorName = joiner.toString();\n" + + "\n" + + " if (params['exact'].containsKey(ancestorName)) {\n" + + " return i;\n" + + " }\n" + + "\n" + + " /* Excluded domains (e.g. !nhs.uk) use the next highest\n" + + " domain as the effective public suffix (e.g. uk). */\n" + + "\n" + + " if (params['excluded'].containsKey(ancestorName)) {\n" + + " return i + 1;\n" + + " }\n" + + "\n" + + " List pieces = split(ancestorName);\n" + + " if (pieces.length >= 2 && params['under'].containsKey(pieces[1])) {\n" + + " return i;\n" + + " }\n" + + " }\n" + + "\n" + + " return -1;\n" + + "}\n" + + "String ancestor(List parts, int levels) {\n" + + " StringJoiner joiner = new StringJoiner('.');\n" + + " for (String s : parts.subList(levels, parts.size())) {\n" + + " joiner.add(s);\n" + + " }\n" + + " String name = joiner.toString();\n" + + " if (name.endsWith('.')) {\n" + + " name = name.substring(0, name.length() - 1);\n" + + " }\n" + + " return name;\n" + + "}\n" + + "String topPrivateDomain(String name, List parts, int publicSuffixIndex) {\n" + + " if (publicSuffixIndex == 1) {\n" + + " return name;\n" + + " }\n" + + " if (!(publicSuffixIndex > 0)) {\n" + + " throw new IllegalArgumentException('Not under a public suffix: ' + name);\n" + + " }\n" + + " return ancestor(parts, publicSuffixIndex - 1);\n" + + "}\n" + + "List domainSplit(String host, Map params) {\n" + + " int MAX_DNS_NAME_LENGTH = 253;\n" + + " int MAX_LENGTH = 253;\n" + + " int MAX_PARTS = 127;\n" + + " if ('host'.isEmpty()) {\n" + + " return ['',''];\n" + + " }\n" + + " host = host.trim();\n" + + " if (host.contains(':')) {\n" + + " return ['', host];\n" + + " }\n" + + " boolean tentativeIP = true;\n" + + " for(int i = 0; i < host.length(); i++) {\n" + + " if (!(Character.isDigit(host.charAt(i)) || host.charAt(i) == (char)'.')) {\n" + + " tentativeIP = false;\n" + + " break;\n" + + " }\n" + + " }\n" + + " if (tentativeIP) {\n" + + " /* special-snowflake rules now... */\n" + + " if (host == '.') {\n" + + " return ['',''];\n" + + " }\n" + + " return ['', host];\n" + + " }\n" + + " def normalizedHost = host;\n" + + " normalizedHost = normalizedHost.toLowerCase();\n" + + " List parts = splitDomain(normalizedHost);\n" + + " int publicSuffixIndex = findPublicSuffix(params, parts);\n" + + " if (publicSuffixIndex == 0) {\n" + + " return ['', host];\n" + + " }\n" + + " String highestRegistered = '';\n" + + " /* for the case where the host is internal like .local so is not a recognised public suffix */\n" + + " if (publicSuffixIndex == -1) {\n" + + " if (!parts.isEmpty()) {\n" + + " if (parts.size() == 1) {\n" + + " return ['', host];\n" + + " }\n" + + " if (parts.size() > 2) {\n" + + " boolean allNumeric = true;\n" + + " String value = parts.get(parts.size() - 1);\n" + + " for (int i = 0; i < value.length(); i++) {\n" + + " if (!Character.isDigit(value.charAt(i))) {\n" + + " allNumeric = false;\n" + + " break;\n" + + " }\n" + + " }\n" + + " if (allNumeric) {\n" + + " highestRegistered = parts.get(parts.size() - 2) + '.' + parts.get(parts.size() - 1);\n" + + " } else {\n" + + " highestRegistered = parts.get(parts.size() - 1);\n" + + " }\n" + + "\n" + + " } else {\n" + + " highestRegistered = parts.get(parts.size() - 1);\n" + + " }\n" + + " }\n" + + " } else {\n" + + " /* HRD is the top private domain */\n" + + " highestRegistered = topPrivateDomain(normalizedHost, parts, publicSuffixIndex);\n" + + " }\n" + + " String subDomain = host.substring(0, host.length() - highestRegistered.length());\n" + + " if (subDomain.endsWith('.')) {\n" + + " subDomain = subDomain.substring(0, subDomain.length() - 1);\n" + + " }\n" + + " return [subDomain, highestRegistered];\n" + + "}\n"; + fn = fn.replace("\n",""); + function = fn; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/ExceptionsHelper.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/ExceptionsHelper.java new file mode 100644 index 00000000000..efec9439a59 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/ExceptionsHelper.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +// NORELEASE: add cause exceptions! +public class ExceptionsHelper { + + public static ResourceNotFoundException missingJobException(String jobId) { + return new ResourceNotFoundException(Messages.getMessage(Messages.JOB_UNKNOWN_ID, jobId)); + } + + public static ResourceAlreadyExistsException jobAlreadyExists(String jobId) { + throw new ResourceAlreadyExistsException(Messages.getMessage(Messages.JOB_CONFIG_ID_ALREADY_TAKEN, jobId)); + } + + public static ResourceNotFoundException missingDatafeedException(String datafeedId) { + throw new ResourceNotFoundException(Messages.getMessage(Messages.DATAFEED_NOT_FOUND, datafeedId)); + } + + public static ElasticsearchException serverError(String msg) { + return new ElasticsearchException(msg); + } + + public static ElasticsearchException serverError(String msg, Throwable cause) { + return new ElasticsearchException(msg, cause); + } + + public static ElasticsearchStatusException conflictStatusException(String msg) { + return new ElasticsearchStatusException(msg, RestStatus.CONFLICT); + } + + public static ElasticsearchParseException parseException(ParseField parseField, Throwable cause) { + throw new ElasticsearchParseException("Failed to parse [" + parseField.getPreferredName() + "]", cause); + } + + /** + * A more REST-friendly Object.requireNonNull() + */ + public static T requireNonNull(T obj, String paramName) { + if (obj == null) { + throw new IllegalArgumentException("[" + paramName + "] must not be null."); + } + return obj; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java new file mode 100644 index 00000000000..94295b944d8 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/JobStateObserver.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateObserver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.Allocation; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; + +import java.util.function.Consumer; +import java.util.function.Predicate; + +public class JobStateObserver { + + private static final Logger LOGGER = Loggers.getLogger(JobStateObserver.class); + + private final ThreadPool threadPool; + private final ClusterService clusterService; + + public JobStateObserver(ThreadPool threadPool, ClusterService clusterService) { + this.threadPool = threadPool; + this.clusterService = clusterService; + } + + public void waitForState(String jobId, TimeValue waitTimeout, JobState expectedState, Consumer handler) { + ClusterStateObserver observer = + new ClusterStateObserver(clusterService, LOGGER, threadPool.getThreadContext()); + JobStatePredicate jobStatePredicate = new JobStatePredicate(jobId, expectedState); + observer.waitForNextChange(new ClusterStateObserver.Listener() { + @Override + public void onNewClusterState(ClusterState state) { + handler.accept(null); + } + + @Override + public void onClusterServiceClose() { + Exception e = new IllegalArgumentException("Cluster service closed while waiting for job state to change to [" + + expectedState + "]"); + handler.accept(new IllegalStateException(e)); + } + + @Override + public void onTimeout(TimeValue timeout) { + if (jobStatePredicate.test(clusterService.state())) { + handler.accept(null); + } else { + Exception e = new IllegalArgumentException("Timeout expired while waiting for job state to change to [" + + expectedState + "]"); + handler.accept(e); + } + } + }, jobStatePredicate, waitTimeout); + } + + private static class JobStatePredicate implements Predicate { + + private final String jobId; + private final JobState expectedState; + + JobStatePredicate(String jobId, JobState expectedState) { + this.jobId = jobId; + this.expectedState = expectedState; + } + + @Override + public boolean test(ClusterState newState) { + MlMetadata metadata = newState.getMetaData().custom(MlMetadata.TYPE); + if (metadata != null) { + Allocation allocation = metadata.getAllocations().get(jobId); + if (allocation != null) { + return allocation.getState() == expectedState; + } + } + return false; + } + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/MlStrings.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/MlStrings.java new file mode 100644 index 00000000000..503d81aedc6 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/MlStrings.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils; + +import java.util.regex.Pattern; + +/** + * Another String utilities class. Class name is prefixed with Ml to avoid confusion + * with one of the myriad String utility classes out there. + */ +public final class MlStrings { + + private static final Pattern NEEDS_QUOTING = Pattern.compile("\\W"); + + /** + * Valid user id pattern. + * Matches a string that contains lower case characters, digits, hyphens, underscores or dots. + * The string may start and end only in lower case characters or digits. + * Note that '.' is allowed but not documented. + */ + private static final Pattern VALID_ID_CHAR_PATTERN = Pattern.compile("[a-z0-9](?:[a-z0-9_\\-\\.]*[a-z0-9])?"); + + private MlStrings() { + } + + /** + * Surrounds with double quotes the given {@code input} if it contains + * any non-word characters. Any double quotes contained in {@code input} + * will be escaped. + * + * @param input any non null string + * @return {@code input} when it does not contain non-word characters, or a new string + * that contains {@code input} surrounded by double quotes otherwise + */ + public static String doubleQuoteIfNotAlphaNumeric(String input) { + if (!NEEDS_QUOTING.matcher(input).find()) { + return input; + } + + StringBuilder quoted = new StringBuilder(); + quoted.append('\"'); + + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + if (c == '\"' || c == '\\') { + quoted.append('\\'); + } + quoted.append(c); + } + + quoted.append('\"'); + return quoted.toString(); + } + + public static boolean isValidId(String id) { + return id != null && VALID_ID_CHAR_PATTERN.matcher(id).matches(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java new file mode 100644 index 00000000000..d41a6e1c917 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/NamedPipeHelper.java @@ -0,0 +1,317 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils; + +import org.apache.lucene.util.Constants; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.SpecialPermission; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.env.Environment; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.time.Duration; + + +/** + * Opens named pipes that are created elsewhere. + * + * In production, these will have been created in C++ code, as the procedure for creating them is + * platform dependent and uses native OS calls that are not easily available in Java. + * + * Once the named pipes have been created elsewhere Java can open them like normal files, however, + * there are complications: + * - On *nix, when opening a pipe for output Java will create a normal file of the requested name + * if the named pipe doesn't already exist. To avoid this, named pipes are only opened for + * for output once the expected file name exists in the file system. + * - On Windows, the server end of a pipe needs to reset it between client connects. Methods like + * File.isFile() and File.exists() on Windows internally call the Win32 API function CreateFile() + * followed by GetFileInformationByHandle(), and if the CreateFile() succeeds it counts as opening + * the named pipe, requiring it to be reset on the server side before subsequent access. To avoid + * this, the check for whether a given path represents a named pipe is done using simple string + * comparison on Windows. + */ +public class NamedPipeHelper { + + /** + * Try this often to open named pipes that we're waiting on another process to create. + */ + private static final long PAUSE_TIME_MS = 20; + + /** + * On Windows named pipes are ALWAYS accessed via this path; it is impossible to put them + * anywhere else. + */ + private static final String WIN_PIPE_PREFIX = "\\\\.\\pipe\\"; + + public NamedPipeHelper() { + // Do nothing - the only reason there's a constructor is to allow mocking + } + + /** + * The default path where named pipes will be created. On *nix they can be created elsewhere + * (subject to security manager constraints), but on Windows this is the ONLY place they can + * be created. + * @return The directory prefix as a string. + */ + public String getDefaultPipeDirectoryPrefix(Environment env) { + // The return type is String because we don't want any (too) clever path processing removing + // the seemingly pointless . in the path used on Windows. + if (Constants.WINDOWS) { + return WIN_PIPE_PREFIX; + } + // Use the Java temporary directory. The Elasticsearch bootstrap sets up the security + // manager to allow this to be read from and written to. Also, the code that spawns our + // daemon passes on this location to the C++ code using the $TMPDIR environment variable. + // All these factors need to align for everything to work in production. If any changes + // are made here then CNamedPipeFactory::defaultPath() in the C++ code will probably + // also need to be changed. + return env.tmpFile().toString() + PathUtils.getDefaultFileSystem().getSeparator(); + } + + /** + * Open a named pipe created elsewhere for input. + * + * @param path + * Path of named pipe to open. + * @param timeout + * How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException + * if the named pipe cannot be opened. + */ + @SuppressForbidden(reason = "Environment doesn't have path for Windows named pipes") + public InputStream openNamedPipeInputStream(String path, Duration timeout) throws IOException { + return openNamedPipeInputStream(PathUtils.get(path), timeout); + } + + /** + * Open a named pipe created elsewhere for input. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + public InputStream openNamedPipeInputStream(Path file, Duration timeout) throws IOException { + long timeoutMillisRemaining = timeout.toMillis(); + + // Can't use Files.isRegularFile() on on named pipes on Windows, as it renders them unusable, + // but luckily there's an even simpler check (that's not possible on *nix) + if (Constants.WINDOWS && !file.toString().startsWith(WIN_PIPE_PREFIX)) { + throw new IOException(file + " is not a named pipe"); + } + + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + // Try to open the file periodically until the timeout expires, then, if + // it's still not available throw the exception from FileInputStream + while (true) { + // On Windows Files.isRegularFile() will render a genuine named pipe unusable + if (!Constants.WINDOWS && Files.isRegularFile(file)) { + throw new IOException(file + " is not a named pipe"); + } + try { + PrivilegedInputPipeOpener privilegedInputPipeOpener = new PrivilegedInputPipeOpener(file); + return AccessController.doPrivileged(privilegedInputPipeOpener); + } catch (RuntimeException e) { + if (timeoutMillisRemaining <= 0) { + propagatePrivilegedException(e); + } + long thisSleep = Math.min(timeoutMillisRemaining, PAUSE_TIME_MS); + timeoutMillisRemaining -= thisSleep; + try { + Thread.sleep(thisSleep); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + propagatePrivilegedException(e); + } + } + } + } + + /** + * Open a named pipe created elsewhere for output. + * + * @param path + * Path of named pipe to open. + * @param timeout + * How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException + * if the named pipe cannot be opened. + */ + @SuppressForbidden(reason = "Environment doesn't have path for Windows named pipes") + public OutputStream openNamedPipeOutputStream(String path, Duration timeout) throws IOException { + return openNamedPipeOutputStream(PathUtils.get(path), timeout); + } + + /** + * Open a named pipe created elsewhere for output. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + public OutputStream openNamedPipeOutputStream(Path file, Duration timeout) throws IOException { + if (Constants.WINDOWS) { + return openNamedPipeOutputStreamWindows(file, timeout); + } + return openNamedPipeOutputStreamUnix(file, timeout); + } + + /** + * The logic here is very similar to that of opening an input stream, because on Windows + * Java cannot create a regular file when asked to open a named pipe that doesn't exist. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + private OutputStream openNamedPipeOutputStreamWindows(Path file, Duration timeout) throws IOException { + long timeoutMillisRemaining = timeout.toMillis(); + + // Can't use File.isFile() on Windows, but luckily there's an even simpler check (that's not possible on *nix) + if (!file.toString().startsWith(WIN_PIPE_PREFIX)) { + throw new IOException(file + " is not a named pipe"); + } + + SecurityManager sm = System.getSecurityManager(); + if (sm != null) { + sm.checkPermission(new SpecialPermission()); + } + + // Try to open the file periodically until the timeout expires, then, if + // it's still not available throw the exception from FileOutputStream + while (true) { + try { + PrivilegedOutputPipeOpener privilegedOutputPipeOpener = new PrivilegedOutputPipeOpener(file); + return AccessController.doPrivileged(privilegedOutputPipeOpener); + } catch (RuntimeException e) { + if (timeoutMillisRemaining <= 0) { + propagatePrivilegedException(e); + } + long thisSleep = Math.min(timeoutMillisRemaining, PAUSE_TIME_MS); + timeoutMillisRemaining -= thisSleep; + try { + Thread.sleep(thisSleep); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + propagatePrivilegedException(e); + } + } + } + } + + /** + * This has to use different logic to the input pipe case to avoid the danger of creating + * a regular file when the named pipe does not exist when the method is first called. + * @param file The named pipe to open. + * @param timeout How long to wait for the named pipe to exist. + * @return A stream opened to read from the named pipe. + * @throws IOException if the named pipe cannot be opened. + */ + private OutputStream openNamedPipeOutputStreamUnix(Path file, Duration timeout) throws IOException { + long timeoutMillisRemaining = timeout.toMillis(); + + // Periodically check whether the file exists until the timeout expires, then, if + // it's still not available throw a FileNotFoundException + while (timeoutMillisRemaining > 0 && !Files.exists(file)) { + long thisSleep = Math.min(timeoutMillisRemaining, PAUSE_TIME_MS); + timeoutMillisRemaining -= thisSleep; + try { + Thread.sleep(thisSleep); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + break; + } + } + + if (Files.isRegularFile(file)) { + throw new IOException(file + " is not a named pipe"); + } + + if (!Files.exists(file)) { + throw new FileNotFoundException("Cannot open " + file + " (No such file or directory)"); + } + + // There's a race condition here in that somebody could delete the named pipe at this point + // causing the line below to create a regular file. Not sure what can be done about this + // without using low level OS calls... + + return Files.newOutputStream(file); + } + + /** + * To work around the limitation that privileged actions cannot throw checked exceptions the classes + * below wrap IOExceptions in RuntimeExceptions. If such an exception needs to be propagated back + * to a user of this class then it's nice if they get the original IOException rather than having + * it wrapped in a RuntimeException. However, the privileged calls could also possibly throw other + * RuntimeExceptions, so this method accounts for this case too. + */ + private void propagatePrivilegedException(RuntimeException e) throws IOException { + Throwable ioe = ExceptionsHelper.unwrap(e, IOException.class); + if (ioe != null) { + throw (IOException)ioe; + } + throw e; + } + + /** + * Used to work around the limitation that privileged actions cannot throw checked exceptions. + */ + private static class PrivilegedInputPipeOpener implements PrivilegedAction { + + private final Path file; + + PrivilegedInputPipeOpener(Path file) { + this.file = file; + } + + @SuppressForbidden(reason = "Files.newInputStream doesn't work with Windows named pipes") + public InputStream run() { + try { + return new FileInputStream(file.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } + + /** + * Used to work around the limitation that privileged actions cannot throw checked exceptions. + */ + private static class PrivilegedOutputPipeOpener implements PrivilegedAction { + + private final Path file; + + PrivilegedOutputPipeOpener(Path file) { + this.file = file; + } + + @SuppressForbidden(reason = "Files.newOutputStream doesn't work with Windows named pipes") + public OutputStream run() { + try { + return new FileOutputStream(file.toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/DateTimeFormatterTimestampConverter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/DateTimeFormatterTimestampConverter.java new file mode 100644 index 00000000000..af128b4d7e2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/DateTimeFormatterTimestampConverter.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils.time; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; + +/** + *

This class implements {@link TimestampConverter} using the {@link DateTimeFormatter} + * of the Java 8 time API for parsing timestamps and other classes of that API for converting + * timestamps to epoch times. + * + *

Objects of this class are immutable and thread-safe + * + */ +public class DateTimeFormatterTimestampConverter implements TimestampConverter { + private final DateTimeFormatter formatter; + private final boolean hasTimeZone; + private final ZoneId defaultZoneId; + + private DateTimeFormatterTimestampConverter(DateTimeFormatter dateTimeFormatter, boolean hasTimeZone, ZoneId defaultTimezone) { + formatter = dateTimeFormatter; + this.hasTimeZone = hasTimeZone; + defaultZoneId = defaultTimezone; + } + + /** + * Creates a formatter according to the given pattern + * @param pattern the pattern to be used by the formatter, not null. + * See {@link DateTimeFormatter} for the syntax of the accepted patterns + * @param defaultTimezone the timezone to be used for dates without timezone information. + * @return a {@code TimestampConverter} + * @throws IllegalArgumentException if the pattern is invalid or cannot produce a full timestamp + * (e.g. contains a date but not a time) + */ + public static TimestampConverter ofPattern(String pattern, ZoneId defaultTimezone) { + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .parseLenient() + .appendPattern(pattern) + .parseDefaulting(ChronoField.YEAR_OF_ERA, LocalDate.now(defaultTimezone).getYear()) + .toFormatter(); + + String now = formatter.format(ZonedDateTime.ofInstant(Instant.ofEpochSecond(0), ZoneOffset.UTC)); + try { + TemporalAccessor parsed = formatter.parse(now); + boolean hasTimeZone = parsed.isSupported(ChronoField.INSTANT_SECONDS); + if (hasTimeZone) { + Instant.from(parsed); + } + else { + LocalDateTime.from(parsed); + } + return new DateTimeFormatterTimestampConverter(formatter, hasTimeZone, defaultTimezone); + } + catch (DateTimeException e) { + throw new IllegalArgumentException("Timestamp cannot be derived from pattern: " + pattern); + } + } + + @Override + public long toEpochSeconds(String timestamp) { + return toInstant(timestamp).getEpochSecond(); + } + + @Override + public long toEpochMillis(String timestamp) { + return toInstant(timestamp).toEpochMilli(); + } + + private Instant toInstant(String timestamp) { + TemporalAccessor parsed = formatter.parse(timestamp); + if (hasTimeZone) { + return Instant.from(parsed); + } + return LocalDateTime.from(parsed).atZone(defaultZoneId).toInstant(); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/TimeUtils.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/TimeUtils.java new file mode 100644 index 00000000000..cce0016f5b5 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/TimeUtils.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils.time; + +import org.elasticsearch.index.mapper.DateFieldMapper; + +public final class TimeUtils { + private TimeUtils() { + // Do nothing + } + + /** + * First tries to parse the date first as a Long and convert that to an + * epoch time. If the long number has more than 10 digits it is considered a + * time in milliseconds else if 10 or less digits it is in seconds. If that + * fails it tries to parse the string using + * {@link DateFieldMapper#DEFAULT_DATE_TIME_FORMATTER} + * + * If the date string cannot be parsed -1 is returned. + * + * @return The epoch time in milliseconds or -1 if the date cannot be + * parsed. + */ + public static long dateStringToEpoch(String date) { + try { + long epoch = Long.parseLong(date); + if (date.trim().length() <= 10) { // seconds + return epoch * 1000; + } else { + return epoch; + } + } catch (NumberFormatException nfe) { + // not a number + } + + try { + return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseMillis(date); + } catch (IllegalArgumentException e) { + } + // Could not do the conversion + return -1; + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/TimestampConverter.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/TimestampConverter.java new file mode 100644 index 00000000000..885c2871dbf --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/utils/time/TimestampConverter.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.utils.time; + +import java.time.format.DateTimeParseException; + +/** + * A converter that enables conversions of textual timestamps to epoch seconds + * or milliseconds according to a given pattern. + */ +public interface TimestampConverter { + /** + * Converts the a textual timestamp into an epoch in seconds + * + * @param timestamp the timestamp to convert, not null. The timestamp is expected to + * be formatted according to the pattern of the formatter. In addition, the pattern is + * assumed to contain both date and time information. + * @return the epoch in seconds for the given timestamp + * @throws DateTimeParseException if unable to parse the given timestamp + */ + long toEpochSeconds(String timestamp); + + /** + * Converts the a textual timestamp into an epoch in milliseconds + * + * @param timestamp the timestamp to convert, not null. The timestamp is expected to + * be formatted according to the pattern of the formatter. In addition, the pattern is + * assumed to contain both date and time information. + * @return the epoch in milliseconds for the given timestamp + * @throws DateTimeParseException if unable to parse the given timestamp + */ + long toEpochMillis(String timestamp); +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java new file mode 100644 index 00000000000..db20020951e --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/CompletionPersistentTaskAction.java @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +/** + * Action that is used by executor node to indicate that the persistent action finished or failed on the node and needs to be + * removed from the cluster state in case of successful completion or restarted on some other node in case of failure. + */ +public class CompletionPersistentTaskAction extends Action { + + public static final CompletionPersistentTaskAction INSTANCE = new CompletionPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/completion"; + + private CompletionPersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest { + + private long taskId; + + private Exception exception; + + public Request() { + + } + + public Request(long taskId, Exception exception) { + this.taskId = taskId; + this.exception = exception; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + exception = in.readException(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + out.writeException(exception); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskId == request.taskId && + Objects.equals(exception, request.exception); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, exception); + } + } + + public static class Response extends AcknowledgedResponse { + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, CompletionPersistentTaskAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, CompletionPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, request.exception, new ActionListener() { + @Override + public void onResponse(Empty empty) { + listener.onResponse(newResponse()); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java new file mode 100644 index 00000000000..9c639d4b1a2 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionCoordinator.java @@ -0,0 +1,389 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.logging.log4j.util.Supplier; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Provider; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress.PersistentTaskInProgress; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.transport.TransportResponse.Empty; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import static java.util.Objects.requireNonNull; + +/** + * This component is responsible for coordination of execution of persistent actions on individual nodes. It runs on all + * non-transport client nodes in the cluster and monitors cluster state changes to detect started commands. + */ +public class PersistentActionCoordinator extends AbstractComponent implements ClusterStateListener { + private final Map runningTasks = new HashMap<>(); + private final PersistentActionService persistentActionService; + private final PersistentActionRegistry persistentActionRegistry; + private final TaskManager taskManager; + private final PersistentActionExecutor persistentActionExecutor; + + + public PersistentActionCoordinator(Settings settings, + PersistentActionService persistentActionService, + PersistentActionRegistry persistentActionRegistry, + TaskManager taskManager, + PersistentActionExecutor persistentActionExecutor) { + super(settings); + this.persistentActionService = persistentActionService; + this.persistentActionRegistry = persistentActionRegistry; + this.taskManager = taskManager; + this.persistentActionExecutor = persistentActionExecutor; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + PersistentTasksInProgress tasks = event.state().custom(PersistentTasksInProgress.TYPE); + PersistentTasksInProgress previousTasks = event.previousState().custom(PersistentTasksInProgress.TYPE); + + if (Objects.equals(tasks, previousTasks) == false || event.nodesChanged()) { + // We have some changes let's check if they are related to our node + String localNodeId = event.state().getNodes().getLocalNodeId(); + Set notVisitedTasks = new HashSet<>(runningTasks.keySet()); + if (tasks != null) { + for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { + if (localNodeId.equals(taskInProgress.getExecutorNode())) { + PersistentTaskId persistentTaskId = new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()); + RunningPersistentTask persistentTask = runningTasks.get(persistentTaskId); + if (persistentTask == null) { + // New task - let's start it + startTask(taskInProgress); + } else { + // The task is still running + notVisitedTasks.remove(persistentTaskId); + if (persistentTask.getState() == State.FAILED_NOTIFICATION) { + // We tried to notify the master about this task before but the notification failed and + // the master doesn't seem to know about it - retry notification + restartCompletionNotification(persistentTask); + } + } + } + } + } + + for (PersistentTaskId id : notVisitedTasks) { + RunningPersistentTask task = runningTasks.get(id); + if (task.getState() == State.NOTIFIED || task.getState() == State.FAILED) { + // Result was sent to the caller and the caller acknowledged acceptance of the result + finishTask(id); + } else if (task.getState() == State.FAILED_NOTIFICATION) { + // We tried to send result to master, but it failed and master doesn't know about this task + // this shouldn't really happen, unless this node is severally out of sync with the master + logger.warn("failed to notify master about task {}", task.getId()); + finishTask(id); + } else { + // task is running locally, but master doesn't know about it - that means that the persistent task was removed + // cancel the task without notifying master + cancelTask(id); + } + } + + } + + } + + private void startTask(PersistentTaskInProgress taskInProgress) { + PersistentActionRegistry.PersistentActionHolder holder = + persistentActionRegistry.getPersistentActionHolderSafe(taskInProgress.getAction()); + PersistentTask task = (PersistentTask) taskManager.register("persistent", taskInProgress.getAction() + "[c]", + taskInProgress.getRequest()); + boolean processed = false; + try { + RunningPersistentTask runningPersistentTask = new RunningPersistentTask(task, taskInProgress.getId()); + task.setStatusProvider(runningPersistentTask); + task.setPersistentTaskId(taskInProgress.getId()); + PersistentTaskListener listener = new PersistentTaskListener(runningPersistentTask); + try { + runningTasks.put(new PersistentTaskId(taskInProgress.getId(), taskInProgress.getAllocationId()), runningPersistentTask); + persistentActionExecutor.executeAction(taskInProgress.getRequest(), task, holder, listener); + } catch (Exception e) { + // Submit task failure + listener.onFailure(e); + } + processed = true; + } finally { + if (processed == false) { + // something went wrong - unregistering task + taskManager.unregister(task); + } + } + } + + private void finishTask(PersistentTaskId persistentTaskId) { + RunningPersistentTask task = runningTasks.remove(persistentTaskId); + if (task != null && task.getTask() != null) { + taskManager.unregister(task.getTask()); + } + } + + private void cancelTask(PersistentTaskId persistentTaskId) { + RunningPersistentTask task = runningTasks.remove(persistentTaskId); + if (task != null && task.getTask() != null) { + if (task.markAsCancelled()) { + persistentActionService.sendCancellation(task.getTask().getId(), new ActionListener() { + @Override + public void onResponse(CancelTasksResponse cancelTasksResponse) { + } + + @Override + public void onFailure(Exception e) { + // There is really nothing we can do in case of failure here + logger.warn((Supplier) () -> new ParameterizedMessage("failed to cancel task {}", task.getId()), e); + } + }); + } + } + } + + + private void restartCompletionNotification(RunningPersistentTask task) { + logger.trace("resending notification for task {}", task.getId()); + if (task.getState() == State.CANCELLED) { + taskManager.unregister(task.getTask()); + } else { + if (task.restartCompletionNotification()) { + persistentActionService.sendCompletionNotification(task.getId(), task.getFailure(), new PublishedResponseListener(task)); + } else { + logger.warn("attempt to resend notification for task {} in the {} state", task.getId(), task.getState()); + } + } + } + + private void startCompletionNotification(RunningPersistentTask task, Exception e) { + if (task.getState() == State.CANCELLED) { + taskManager.unregister(task.getTask()); + } else { + logger.trace("sending notification for failed task {}", task.getId()); + if (task.startNotification(e)) { + persistentActionService.sendCompletionNotification(task.getId(), e, new PublishedResponseListener(task)); + } else { + logger.warn("attempt to send notification for task {} in the {} state", task.getId(), task.getState()); + } + } + } + + private class PersistentTaskListener implements ActionListener { + private final RunningPersistentTask task; + + PersistentTaskListener(final RunningPersistentTask task) { + this.task = task; + } + + @Override + public void onResponse(Empty response) { + startCompletionNotification(task, null); + } + + @Override + public void onFailure(Exception e) { + if (task.getTask().isCancelled()) { + // The task was explicitly cancelled - no need to restart it, just log the exception if it's not TaskCancelledException + if (e instanceof TaskCancelledException == false) { + logger.warn((Supplier) () -> new ParameterizedMessage( + "cancelled task {} failed with an exception, cancellation reason [{}]", + task.getId(), task.getTask().getReasonCancelled()), e); + } + startCompletionNotification(task, null); + } else { + startCompletionNotification(task, e); + } + } + } + + private class PublishedResponseListener implements ActionListener { + private final RunningPersistentTask task; + + PublishedResponseListener(final RunningPersistentTask task) { + this.task = task; + } + + + @Override + public void onResponse(CompletionPersistentTaskAction.Response response) { + logger.trace("notification for task {} was successful", task.getId()); + if (task.markAsNotified() == false) { + logger.warn("attempt to mark task {} in the {} state as NOTIFIED", task.getId(), task.getState()); + } + taskManager.unregister(task.getTask()); + } + + @Override + public void onFailure(Exception e) { + logger.warn((Supplier) () -> new ParameterizedMessage("notification for task {} failed - retrying", task.getId()), e); + if (task.notificationFailed() == false) { + logger.warn("attempt to mark restart notification for task {} in the {} state failed", task.getId(), task.getState()); + } + } + } + + public enum State { + STARTED, // the task is currently running + CANCELLED, // the task is cancelled + FAILED, // the task is done running and trying to notify caller + FAILED_NOTIFICATION, // the caller notification failed + NOTIFIED // the caller was notified, the task can be removed + } + + private static class PersistentTaskId { + private final long id; + private final long allocationId; + + PersistentTaskId(long id, long allocationId) { + this.id = id; + this.allocationId = allocationId; + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTaskId that = (PersistentTaskId) o; + return id == that.id && + allocationId == that.allocationId; + } + + @Override + public int hashCode() { + return Objects.hash(id, allocationId); + } + } + + private static class RunningPersistentTask implements Provider { + private final PersistentTask task; + private final long id; + private final AtomicReference state; + @Nullable + private Exception failure; + + RunningPersistentTask(PersistentTask task, long id) { + this(task, id, State.STARTED); + } + + RunningPersistentTask(PersistentTask task, long id, State state) { + this.task = task; + this.id = id; + this.state = new AtomicReference<>(state); + } + + public PersistentTask getTask() { + return task; + } + + public long getId() { + return id; + } + + public State getState() { + return state.get(); + } + + public Exception getFailure() { + return failure; + } + + public boolean startNotification(Exception failure) { + boolean result = state.compareAndSet(State.STARTED, State.FAILED); + if (result) { + this.failure = failure; + } + return result; + } + + public boolean notificationFailed() { + return state.compareAndSet(State.FAILED, State.FAILED_NOTIFICATION); + } + + public boolean restartCompletionNotification() { + return state.compareAndSet(State.FAILED_NOTIFICATION, State.FAILED); + } + + public boolean markAsNotified() { + return state.compareAndSet(State.FAILED, State.NOTIFIED); + } + + public boolean markAsCancelled() { + return state.compareAndSet(State.STARTED, State.CANCELLED); + } + + @Override + public Task.Status get() { + return new Status(state.get()); + } + } + + public static class Status implements Task.Status { + public static final String NAME = "persistent_executor"; + + private final State state; + + public Status(State state) { + this.state = requireNonNull(state, "State cannot be null"); + } + + public Status(StreamInput in) throws IOException { + state = State.valueOf(in.readString()); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("state", state.toString()); + builder.endObject(); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(state.toString()); + } + + @Override + public String toString() { + return Strings.toString(this); + } + + public State getState() { + return state; + } + + @Override + public boolean isFragment() { + return false; + } + } + +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java new file mode 100644 index 00000000000..64700448441 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionExecutor.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; + +/** + * This component is responsible for execution of persistent actions. + */ +public class PersistentActionExecutor { + private final ThreadPool threadPool; + + public PersistentActionExecutor(ThreadPool threadPool) { + this.threadPool = threadPool; + } + + public void executeAction(Request request, + PersistentTask task, + PersistentActionRegistry.PersistentActionHolder holder, + ActionListener listener) { + threadPool.executor(holder.getExecutor()).execute(new AbstractRunnable() { + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + + @SuppressWarnings("unchecked") + @Override + protected void doRun() throws Exception { + try { + holder.getPersistentAction().nodeOperation(task, request, listener); + } catch (Exception ex) { + listener.onFailure(ex); + } + + } + }); + + } + +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java new file mode 100644 index 00000000000..155f87345cc --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRegistry.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.common.collect.MapBuilder; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; + +import java.util.Collections; +import java.util.Map; + +/** + * Components that registers all persistent actions + */ +public class PersistentActionRegistry extends AbstractComponent { + + private volatile Map> actions = Collections.emptyMap(); + + private final Object actionHandlerMutex = new Object(); + + public PersistentActionRegistry(Settings settings) { + super(settings); + } + + public void registerPersistentAction(String action, + TransportPersistentAction persistentAction) { + registerPersistentAction(new PersistentActionHolder<>(action, persistentAction, persistentAction.getExecutor())); + } + + private void registerPersistentAction( + PersistentActionHolder reg) { + + synchronized (actionHandlerMutex) { + PersistentActionHolder replaced = actions.get(reg.getAction()); + actions = MapBuilder.newMapBuilder(actions).put(reg.getAction(), reg).immutableMap(); + if (replaced != null) { + logger.warn("registered two handlers for persistent action {}, handlers: {}, {}", reg.getAction(), reg, replaced); + } + } + } + + public void removeHandler(String action) { + synchronized (actionHandlerMutex) { + actions = MapBuilder.newMapBuilder(actions).remove(action).immutableMap(); + } + } + + @SuppressWarnings("unchecked") + public PersistentActionHolder getPersistentActionHolderSafe(String action) { + PersistentActionHolder holder = (PersistentActionHolder) actions.get(action); + if (holder == null) { + throw new IllegalStateException("Unknown persistent action [" + action + "]"); + } + return holder; + } + + public + TransportPersistentAction getPersistentActionSafe(String action) { + PersistentActionHolder holder = getPersistentActionHolderSafe(action); + return holder.getPersistentAction(); + } + + public static final class PersistentActionHolder { + + private final String action; + private final TransportPersistentAction persistentAction; + private final String executor; + + + public PersistentActionHolder(String action, TransportPersistentAction persistentAction, String executor) { + this.action = action; + this.persistentAction = persistentAction; + this.executor = executor; + } + + public String getAction() { + return action; + } + + public TransportPersistentAction getPersistentAction() { + return persistentAction; + } + + public String getExecutor() { + return executor; + } + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java new file mode 100644 index 00000000000..0f0609bc2ba --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionRequest.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +/** + * Base class for a request for a persistent action + */ +public abstract class PersistentActionRequest extends ActionRequest implements NamedWriteable, ToXContent { + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId) { + return new PersistentTask(id, type, action, getDescription(), parentTaskId); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java new file mode 100644 index 00000000000..1c99ae085d0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionResponse.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Objects; + +/** + * Response upon a successful start or an persistent action + */ +public class PersistentActionResponse extends ActionResponse { + private long taskId; + + public PersistentActionResponse() { + super(); + } + + public PersistentActionResponse(long taskId) { + this.taskId = taskId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + } + + public long getTaskId() { + return taskId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentActionResponse that = (PersistentActionResponse) o; + return taskId == that.taskId; + } + + @Override + public int hashCode() { + return Objects.hash(taskId); + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java new file mode 100644 index 00000000000..4cffc9a5874 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentActionService.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; +import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +/** + * Service responsible for executing restartable actions that can survive disappearance of a coordinating and executor nodes. + */ +public class PersistentActionService extends AbstractComponent { + + private final Client client; + private final ClusterService clusterService; + + public PersistentActionService(Settings settings, ClusterService clusterService, Client client) { + super(settings); + this.client = client; + this.clusterService = clusterService; + } + + public void sendRequest(String action, Request request, + ActionListener listener) { + StartPersistentTaskAction.Request startRequest = new StartPersistentTaskAction.Request(action, request); + try { + client.execute(StartPersistentTaskAction.INSTANCE, startRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void sendCompletionNotification(long taskId, Exception failure, + ActionListener listener) { + CompletionPersistentTaskAction.Request restartRequest = new CompletionPersistentTaskAction.Request(taskId, failure); + try { + client.execute(CompletionPersistentTaskAction.INSTANCE, restartRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void sendCancellation(long taskId, ActionListener listener) { + DiscoveryNode localNode = clusterService.localNode(); + CancelTasksRequest cancelTasksRequest = new CancelTasksRequest(); + cancelTasksRequest.setTaskId(new TaskId(localNode.getId(), taskId)); + cancelTasksRequest.setReason("persistent action was removed"); + try { + client.admin().cluster().cancelTasks(cancelTasksRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void updateStatus(long taskId, Task.Status status, ActionListener listener) { + UpdatePersistentTaskStatusAction.Request updateStatusRequest = new UpdatePersistentTaskStatusAction.Request(taskId, status); + try { + client.execute(UpdatePersistentTaskStatusAction.INSTANCE, updateStatusRequest, listener); + } catch (Exception e) { + listener.onFailure(e); + } + } + +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTask.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTask.java new file mode 100644 index 00000000000..405c1beecfb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTask.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.common.inject.Provider; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; + +/** + * Task that returns additional state information + */ +public class PersistentTask extends CancellableTask { + private Provider statusProvider; + + private long persistentTaskId; + + public PersistentTask(long id, String type, String action, String description, TaskId parentTask) { + super(id, type, action, description, parentTask); + } + + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + + @Override + public Status getStatus() { + Provider statusProvider = this.statusProvider; + if (statusProvider != null) { + return statusProvider.get(); + } else { + return null; + } + } + + public void setStatusProvider(Provider statusProvider) { + assert this.statusProvider == null; + this.statusProvider = statusProvider; + } + + public long getPersistentTaskId() { + return persistentTaskId; + } + + public void setPersistentTaskId(long persistentTaskId) { + this.persistentTaskId = persistentTaskId; + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java new file mode 100644 index 00000000000..762aa8e03bb --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTaskClusterService.java @@ -0,0 +1,281 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress.PersistentTaskInProgress; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Component that runs only on the master node and is responsible for assigning running tasks to nodes + */ +public class PersistentTaskClusterService extends AbstractComponent implements ClusterStateListener { + + private final ClusterService clusterService; + private final PersistentActionRegistry registry; + + public PersistentTaskClusterService(Settings settings, PersistentActionRegistry registry, ClusterService clusterService) { + super(settings); + this.clusterService = clusterService; + clusterService.addListener(this); + this.registry = registry; + + } + + /** + * Creates a new persistent task on master node + * + * @param action the action name + * @param request request + * @param listener the listener that will be called when task is started + */ + public void createPersistentTask(String action, Request request, + ActionListener listener) { + clusterService.submitStateUpdateTask("create persistent task", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + final String executorNodeId = executorNode(action, currentState, request); + PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); + long nextId; + if (tasksInProgress != null) { + nextId = tasksInProgress.getCurrentId() + 1; + } else { + nextId = 1; + } + return createPersistentTask(currentState, new PersistentTaskInProgress<>(nextId, action, request, executorNodeId)); + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(((PersistentTasksInProgress) newState.custom(PersistentTasksInProgress.TYPE)).getCurrentId()); + } + }); + } + + + /** + * Restarts a record about a running persistent task from cluster state + * + * @param id the id of a persistent task + * @param failure the reason for restarting the task or null if the task completed successfully + * @param listener the listener that will be called when task is removed + */ + public void completeOrRestartPersistentTask(long id, Exception failure, ActionListener listener) { + final String source; + if (failure != null) { + logger.warn("persistent task " + id + " failed, restarting", failure); + source = "restart persistent task"; + } else { + source = "finish persistent task"; + } + clusterService.submitStateUpdateTask(source, new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress == null) { + // Nothing to do, the task was already deleted + return currentState; + } + if (failure != null) { + // If the task failed - we need to restart it on another node, otherwise we just remove it + PersistentTaskInProgress taskInProgress = tasksInProgress.getTask(id); + if (taskInProgress != null) { + String executorNode = executorNode(taskInProgress.getAction(), currentState, taskInProgress.getRequest()); + return updatePersistentTask(currentState, new PersistentTaskInProgress<>(taskInProgress, executorNode)); + } + return currentState; + } else { + return removePersistentTask(currentState, id); + } + + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(Empty.INSTANCE); + } + }); + } + + /** + * Update task status + * + * @param id the id of a persistent task + * @param status new status + * @param listener the listener that will be called when task is removed + */ + public void updatePersistentTaskStatus(long id, Task.Status status, ActionListener listener) { + clusterService.submitStateUpdateTask("update task status", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress tasksInProgress = currentState.custom(PersistentTasksInProgress.TYPE); + if (tasksInProgress == null) { + // Nothing to do, the task no longer exists + return currentState; + } + PersistentTaskInProgress task = tasksInProgress.getTask(id); + if (task != null) { + return updatePersistentTask(currentState, new PersistentTaskInProgress<>(task, status)); + } + return currentState; + } + + @Override + public void onFailure(String source, Exception e) { + listener.onFailure(e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + listener.onResponse(Empty.INSTANCE); + } + }); + } + + private ClusterState updatePersistentTask(ClusterState oldState, PersistentTaskInProgress newTask) { + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + Map> taskMap = new HashMap<>(); + taskMap.putAll(oldTasks.taskMap()); + taskMap.put(newTask.getId(), newTask); + ClusterState.Builder builder = ClusterState.builder(oldState); + PersistentTasksInProgress newTasks = new PersistentTasksInProgress(oldTasks.getCurrentId(), Collections.unmodifiableMap(taskMap)); + return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); + } + + private ClusterState createPersistentTask(ClusterState oldState, PersistentTaskInProgress newTask) { + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + Map> taskMap = new HashMap<>(); + if (oldTasks != null) { + taskMap.putAll(oldTasks.taskMap()); + } + taskMap.put(newTask.getId(), newTask); + ClusterState.Builder builder = ClusterState.builder(oldState); + PersistentTasksInProgress newTasks = new PersistentTasksInProgress(newTask.getId(), Collections.unmodifiableMap(taskMap)); + return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); + } + + private ClusterState removePersistentTask(ClusterState oldState, long taskId) { + PersistentTasksInProgress oldTasks = oldState.custom(PersistentTasksInProgress.TYPE); + if (oldTasks != null) { + Map> taskMap = new HashMap<>(); + ClusterState.Builder builder = ClusterState.builder(oldState); + taskMap.putAll(oldTasks.taskMap()); + taskMap.remove(taskId); + PersistentTasksInProgress newTasks = + new PersistentTasksInProgress(oldTasks.getCurrentId(), Collections.unmodifiableMap(taskMap)); + return builder.putCustom(PersistentTasksInProgress.TYPE, newTasks).build(); + } else { + // no tasks - nothing to do + return oldState; + } + } + + private String executorNode(String action, ClusterState currentState, Request request) { + TransportPersistentAction persistentAction = registry.getPersistentActionSafe(action); + persistentAction.validate(request, currentState); + DiscoveryNode executorNode = persistentAction.executorNode(request, currentState); + final String executorNodeId; + if (executorNode == null) { + // The executor node not available yet, we will create task with empty executor node and try + // again later + executorNodeId = null; + } else { + executorNodeId = executorNode.getId(); + } + return executorNodeId; + } + + @Override + public void clusterChanged(ClusterChangedEvent event) { + if (event.localNodeMaster()) { + PersistentTasksInProgress tasks = event.state().custom(PersistentTasksInProgress.TYPE); + if (tasks != null && (event.nodesChanged() || event.previousState().nodes().isLocalNodeElectedMaster() == false)) { + // We need to check if removed nodes were running any of the tasks and reassign them + boolean reassignmentRequired = false; + Set removedNodes = event.nodesDelta().removedNodes().stream().map(DiscoveryNode::getId).collect(Collectors.toSet()); + for (PersistentTaskInProgress taskInProgress : tasks.tasks()) { + if (taskInProgress.getExecutorNode() == null) { + // there is an unassigned task - we need to try assigning it + reassignmentRequired = true; + break; + } + if (removedNodes.contains(taskInProgress.getExecutorNode())) { + // The caller node disappeared, we need to assign a new caller node + reassignmentRequired = true; + break; + } + } + if (reassignmentRequired) { + reassignTasks(); + } + } + } + } + + /** + * Evaluates the cluster state and tries to assign tasks to nodes + */ + public void reassignTasks() { + clusterService.submitStateUpdateTask("reassign persistent tasks", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) throws Exception { + PersistentTasksInProgress tasks = currentState.custom(PersistentTasksInProgress.TYPE); + ClusterState newClusterState = currentState; + DiscoveryNodes nodes = currentState.nodes(); + if (tasks != null) { + // We need to check if removed nodes were running any of the tasks and reassign them + for (PersistentTaskInProgress task : tasks.tasks()) { + if (task.getExecutorNode() == null || nodes.nodeExists(task.getExecutorNode()) == false) { + // there is an unassigned task - we need to try assigning it + String executorNode = executorNode(task.getAction(), currentState, task.getRequest()); + if (Objects.equals(executorNode, task.getExecutorNode()) == false) { + newClusterState = updatePersistentTask(newClusterState, new PersistentTaskInProgress<>(task, executorNode)); + } + } + } + } + return newClusterState; + } + + @Override + public void onFailure(String source, Exception e) { + logger.warn("Unsuccessful persistent task reassignment", e); + } + + @Override + public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) { + + } + }); + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksInProgress.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksInProgress.java new file mode 100644 index 00000000000..d5b11d529e7 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/PersistentTasksInProgress.java @@ -0,0 +1,256 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.AbstractNamedDiffable; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.NamedDiff; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.Task.Status; + +import java.io.IOException; +import java.util.Collection; +import java.util.Map; +import java.util.Objects; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * A cluster state record that contains a list of all running persistent tasks + */ +public final class PersistentTasksInProgress extends AbstractNamedDiffable implements ClusterState.Custom { + public static final String TYPE = "persistent_tasks"; + + // TODO: Implement custom Diff for tasks + private final Map> tasks; + + private final long currentId; + + public PersistentTasksInProgress(long currentId, Map> tasks) { + this.currentId = currentId; + this.tasks = tasks; + } + + public Collection> tasks() { + return this.tasks.values(); + } + + public Map> taskMap() { + return this.tasks; + } + + public PersistentTaskInProgress getTask(long id) { + return this.tasks.get(id); + } + + public Collection> findTasks(String actionName, Predicate> predicate) { + return this.tasks().stream() + .filter(p -> actionName.equals(p.getAction())) + .filter(predicate) + .collect(Collectors.toList()); + } + + public boolean tasksExist(String actionName, Predicate> predicate) { + return this.tasks().stream() + .filter(p -> actionName.equals(p.getAction())) + .anyMatch(predicate); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTasksInProgress that = (PersistentTasksInProgress) o; + return currentId == that.currentId && + Objects.equals(tasks, that.tasks); + } + + @Override + public int hashCode() { + return Objects.hash(tasks, currentId); + } + + public long getNumberOfTasksOnNode(String nodeId, String action) { + return tasks.values().stream().filter(task -> action.equals(task.action) && nodeId.equals(task.executorNode)).count(); + } + + @Override + public Version getMinimalSupportedVersion() { + return Version.V_5_3_0_UNRELEASED; + } + + /** + * A record that represents a single running persistent task + */ + public static class PersistentTaskInProgress implements Writeable, ToXContent { + private final long id; + private final long allocationId; + private final String action; + private final Request request; + @Nullable + private final Status status; + @Nullable + private final String executorNode; + + + public PersistentTaskInProgress(long id, String action, Request request, String executorNode) { + this(id, 0L, action, request, null, executorNode); + } + + public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, String newExecutorNode) { + this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId + 1L, + persistentTaskInProgress.action, persistentTaskInProgress.request, null, newExecutorNode); + } + + public PersistentTaskInProgress(PersistentTaskInProgress persistentTaskInProgress, Status status) { + this(persistentTaskInProgress.id, persistentTaskInProgress.allocationId, + persistentTaskInProgress.action, persistentTaskInProgress.request, status, persistentTaskInProgress.executorNode); + } + + private PersistentTaskInProgress(long id, long allocationId, String action, Request request, Status status, String executorNode) { + this.id = id; + this.allocationId = allocationId; + this.action = action; + this.request = request; + this.status = status; + this.executorNode = executorNode; + // Update parent request for starting tasks with correct parent task ID + request.setParentTask("cluster", id); + } + + @SuppressWarnings("unchecked") + private PersistentTaskInProgress(StreamInput in) throws IOException { + id = in.readLong(); + allocationId = in.readLong(); + action = in.readString(); + request = (Request) in.readNamedWriteable(PersistentActionRequest.class); + status = in.readOptionalNamedWriteable(Task.Status.class); + executorNode = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(id); + out.writeLong(allocationId); + out.writeString(action); + out.writeNamedWriteable(request); + out.writeOptionalNamedWriteable(status); + out.writeOptionalString(executorNode); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PersistentTaskInProgress that = (PersistentTaskInProgress) o; + return id == that.id && + allocationId == that.allocationId && + Objects.equals(action, that.action) && + Objects.equals(request, that.request) && + Objects.equals(status, that.status) && + Objects.equals(executorNode, that.executorNode); + } + + @Override + public int hashCode() { + return Objects.hash(id, allocationId, action, request, status, executorNode); + } + + public long getId() { + return id; + } + + public long getAllocationId() { + return allocationId; + } + + public String getAction() { + return action; + } + + public Request getRequest() { + return request; + } + + @Nullable + public String getExecutorNode() { + return executorNode; + } + + @Nullable + public Status getStatus() { + return status; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("uuid", id); + builder.field("action", action); + builder.field("request"); + request.toXContent(builder, params); + if (status != null) { + builder.field("status", status, params); + } + builder.field("executor_node", executorNode); + } + builder.endObject(); + return builder; + } + + @Override + public boolean isFragment() { + return false; + } + } + + @Override + public String getWriteableName() { + return TYPE; + } + + public PersistentTasksInProgress(StreamInput in) throws IOException { + currentId = in.readLong(); + tasks = in.readMap(StreamInput::readLong, PersistentTaskInProgress::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeLong(currentId); + out.writeMap(tasks, StreamOutput::writeLong, (stream, value) -> { + value.writeTo(stream); + }); + } + + public static NamedDiff readDiffFrom(StreamInput in) throws IOException { + return readDiffFrom(ClusterState.Custom.class, TYPE, in); + } + + public long getCurrentId() { + return currentId; + } + + + @Override + public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.field("current_id", currentId); + builder.startArray("running_tasks"); + for (PersistentTaskInProgress entry : tasks.values()) { + entry.toXContent(builder, params); + } + builder.endArray(); + return builder; + } + +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java new file mode 100644 index 00000000000..77af43c6c99 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/RemovePersistentTaskAction.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +public class RemovePersistentTaskAction extends Action { + + public static final RemovePersistentTaskAction INSTANCE = new RemovePersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/remove"; + + private RemovePersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest { + + private long taskId; + + public Request() { + + } + + public Request(long taskId) { + this.taskId = taskId; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskId == request.taskId; + } + + @Override + public int hashCode() { + return Objects.hash(taskId); + } + } + + public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, RemovePersistentTaskAction action) { + super(client, action, new Request()); + } + + public final RequestBuilder setTaskId(long taskId) { + request.setTaskId(taskId); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, RemovePersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.completeOrRestartPersistentTask(request.taskId, null, new ActionListener() { + @Override + public void onResponse(Empty empty) { + listener.onResponse(new Response(true)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java new file mode 100644 index 00000000000..e0f137cc811 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/StartPersistentTaskAction.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +/** + * Internal action used by TransportPersistentAction to add the record for the persistent action to the cluster state. + */ +public class StartPersistentTaskAction extends Action { + + public static final StartPersistentTaskAction INSTANCE = new StartPersistentTaskAction(); + public static final String NAME = "cluster:admin/persistent/start"; + + private StartPersistentTaskAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + public static class Request extends MasterNodeRequest { + + private String action; + + private PersistentActionRequest request; + + public Request() { + + } + + public Request(String action, PersistentActionRequest request) { + this.action = action; + this.request = request; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + action = in.readString(); + request = in.readOptionalNamedWriteable(PersistentActionRequest.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(action); + out.writeOptionalNamedWriteable(request); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request1 = (Request) o; + return Objects.equals(action, request1.action) && + Objects.equals(request, request1.request); + } + + @Override + public int hashCode() { + return Objects.hash(action, request); + } + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, StartPersistentTaskAction action) { + super(client, action, new Request()); + } + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + PersistentActionRegistry persistentActionRegistry, + PersistentActionService persistentActionService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, StartPersistentTaskAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + PersistentActionExecutor executor = new PersistentActionExecutor(threadPool); + clusterService.addListener(new PersistentActionCoordinator(settings, persistentActionService, persistentActionRegistry, + transportService.getTaskManager(), executor)); + } + + @Override + protected String executor() { + return ThreadPool.Names.GENERIC; + } + + @Override + protected PersistentActionResponse newResponse() { + return new PersistentActionResponse(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, + final ActionListener listener) { + persistentTaskClusterService.createPersistentTask(request.action, request.request, new ActionListener() { + @Override + public void onResponse(Long newTaskId) { + listener.onResponse(new PersistentActionResponse(newTaskId)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} + + diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/TransportPersistentAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/TransportPersistentAction.java new file mode 100644 index 00000000000..b065aa54cc9 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/TransportPersistentAction.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.util.function.Predicate; +import java.util.function.Supplier; + +/** + * An action that can survive restart of requesting or executing node. + * These actions are using cluster state rather than only transport service to send requests and responses. + */ +public abstract class TransportPersistentAction + extends HandledTransportAction { + + private final String executor; + private final PersistentActionService persistentActionService; + + protected TransportPersistentAction(Settings settings, String actionName, boolean canTripCircuitBreaker, ThreadPool threadPool, + TransportService transportService, PersistentActionService persistentActionService, + PersistentActionRegistry persistentActionRegistry, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Supplier requestSupplier, String executor) { + super(settings, actionName, canTripCircuitBreaker, threadPool, transportService, actionFilters, indexNameExpressionResolver, + requestSupplier); + this.executor = executor; + this.persistentActionService = persistentActionService; + persistentActionRegistry.registerPersistentAction(actionName, this); + } + + /** + * Returns the node id where the request has to be executed, + *

+ * The default implementation returns the least loaded data node + */ + public DiscoveryNode executorNode(Request request, ClusterState clusterState) { + return selectLeastLoadedNode(clusterState, DiscoveryNode::isDataNode); + } + + /** + * Finds the least loaded node that satisfies the selector criteria + */ + protected DiscoveryNode selectLeastLoadedNode(ClusterState clusterState, Predicate selector) { + long minLoad = Long.MAX_VALUE; + DiscoveryNode minLoadedNode = null; + PersistentTasksInProgress persistentTasksInProgress = clusterState.custom(PersistentTasksInProgress.TYPE); + for (DiscoveryNode node : clusterState.getNodes()) { + if (selector.test(node)) { + if (persistentTasksInProgress == null) { + // We don't have any task running yet, pick the first available node + return node; + } + long numberOfTasks = persistentTasksInProgress.getNumberOfTasksOnNode(node.getId(), actionName); + if (minLoad > numberOfTasks) { + minLoad = numberOfTasks; + minLoadedNode = node; + } + } + } + return minLoadedNode; + } + + /** + * Checks the current cluster state for compatibility with the request + *

+ * Throws an exception if the supplied request cannot be executed on the cluster in the current state. + */ + public void validate(Request request, ClusterState clusterState) { + + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + persistentActionService.sendRequest(actionName, request, listener); + } + + /** + * Updates the persistent task status in the cluster state. + *

+ * The status can be used to store the current progress of the task or provide an insight for the + * task allocator about the state of the currently running tasks. + */ + protected void updatePersistentTaskStatus(PersistentTask task, Task.Status status, ActionListener listener) { + persistentActionService.updateStatus(task.getPersistentTaskId(), status, + new ActionListener() { + @Override + public void onResponse(UpdatePersistentTaskStatusAction.Response response) { + listener.onResponse(Empty.INSTANCE); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + + /** + * This operation will be executed on the executor node. + *

+ * If nodeOperation throws an exception or triggers listener.onFailure() method, the task will be restarted, + * possibly on a different node. If listener.onResponse() is called, the task is considered to be successfully + * completed and will be removed from the cluster state and not restarted. + */ + protected abstract void nodeOperation(PersistentTask task, Request request, ActionListener listener); + + public String getExecutor() { + return executor; + } +} \ No newline at end of file diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java new file mode 100644 index 00000000000..d068a647ab0 --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/UpdatePersistentTaskStatusAction.java @@ -0,0 +1,210 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.persistent; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportResponse.Empty; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.Objects; + +public class UpdatePersistentTaskStatusAction extends Action { + + public static final UpdatePersistentTaskStatusAction INSTANCE = new UpdatePersistentTaskStatusAction(); + public static final String NAME = "cluster:admin/persistent/update_status"; + + private UpdatePersistentTaskStatusAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client, this); + } + + @Override + public Response newResponse() { + return new Response(); + } + + public static class Request extends MasterNodeRequest { + + private long taskId; + + private Task.Status status; + + public Request() { + + } + + public Request(long taskId, Task.Status status) { + this.taskId = taskId; + this.status = status; + } + + public void setTaskId(long taskId) { + this.taskId = taskId; + } + + public void setStatus(Task.Status status) { + this.status = status; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + taskId = in.readLong(); + status = in.readOptionalNamedWriteable(Task.Status.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeLong(taskId); + out.writeOptionalNamedWriteable(status); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return taskId == request.taskId && + Objects.equals(status, request.status); + } + + @Override + public int hashCode() { + return Objects.hash(taskId, status); + } + } + + public static class Response extends AcknowledgedResponse { + public Response() { + super(); + } + + public Response(boolean acknowledged) { + super(acknowledged); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + readAcknowledged(in); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + writeAcknowledged(out); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AcknowledgedResponse that = (AcknowledgedResponse) o; + return isAcknowledged() == that.isAcknowledged(); + } + + @Override + public int hashCode() { + return Objects.hash(isAcknowledged()); + } + + } + + public static class RequestBuilder extends MasterNodeOperationRequestBuilder { + + protected RequestBuilder(ElasticsearchClient client, UpdatePersistentTaskStatusAction action) { + super(client, action, new Request()); + } + + public final RequestBuilder setTaskId(long taskId) { + request.setTaskId(taskId); + return this; + } + + public final RequestBuilder setStatus(Task.Status status) { + request.setStatus(status); + return this; + } + + } + + public static class TransportAction extends TransportMasterNodeAction { + + private final PersistentTaskClusterService persistentTaskClusterService; + + @Inject + public TransportAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters, + PersistentTaskClusterService persistentTaskClusterService, + IndexNameExpressionResolver indexNameExpressionResolver) { + super(settings, UpdatePersistentTaskStatusAction.NAME, transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver, Request::new); + this.persistentTaskClusterService = persistentTaskClusterService; + } + + @Override + protected String executor() { + return ThreadPool.Names.MANAGEMENT; + } + + @Override + protected Response newResponse() { + return new Response(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + // Cluster is not affected but we look up repositories in metadata + return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); + } + + @Override + protected final void masterOperation(final Request request, ClusterState state, final ActionListener listener) { + persistentTaskClusterService.updatePersistentTaskStatus(request.taskId, request.status, new ActionListener() { + @Override + public void onResponse(Empty empty) { + listener.onResponse(new Response(true)); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } + } +} diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/package-info.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/package-info.java new file mode 100644 index 00000000000..86fab277a5a --- /dev/null +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/persistent/package-info.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +/** + * The Persistent Actions are actions responsible for executing restartable actions that can survive disappearance of a + * coordinating and executor nodes. + *

+ * In order to be resilient to node restarts, the persistent actions are using the cluster state instead of a transport service to send + * requests and responses. The execution is done in six phases: + *

+ * 1. The coordinating node sends an ordinary transport request to the master node to start a new persistent action. This action is handled + * by the {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which is using + * {@link org.elasticsearch.xpack.persistent.PersistentTaskClusterService} to update cluster state with the record about running persistent + * task. + *

+ * 2. The master node updates the {@link org.elasticsearch.xpack.persistent.PersistentTasksInProgress} in the cluster state to indicate that + * there is a new persistent action + * running in the system. + *

+ * 3. The {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} running on every node in the cluster monitors changes in + * the cluster state and starts execution of all new actions assigned to the node it is running on. + *

+ * 4. If the action fails to start on the node, the {@link org.elasticsearch.xpack.persistent.PersistentActionCoordinator} uses the + * {@link org.elasticsearch.xpack.persistent.PersistentTasksInProgress} to notify the + * {@link org.elasticsearch.xpack.persistent.PersistentActionService}, which reassigns the action to another node in the cluster. + *

+ * 5. If action finishes successfully on the node and calls listener.onResponse(), the corresponding persistent action is removed from the + * cluster state. + *

+ * 6. The {@link org.elasticsearch.xpack.persistent.RemovePersistentTaskAction} action can be also used to remove the persistent action. + */ +package org.elasticsearch.xpack.persistent; \ No newline at end of file diff --git a/elasticsearch/src/main/resources/org/elasticsearch/xpack/ml/job/messages/ml_messages.properties b/elasticsearch/src/main/resources/org/elasticsearch/xpack/ml/job/messages/ml_messages.properties new file mode 100644 index 00000000000..3bd46d8e8ba --- /dev/null +++ b/elasticsearch/src/main/resources/org/elasticsearch/xpack/ml/job/messages/ml_messages.properties @@ -0,0 +1,175 @@ + +# Machine Learning API messages + +autodetect.flush.failed.unexpected.death =[{0}] Flush failed: Unexpected death of the Autodetect process flushing job. + +cpu.limit.jobs = Cannot start job with id ''{0}''. The maximum number of concurrently running jobs is limited as a function of the number of CPU cores see this error code''s help documentation for details of how to elevate the setting + +datastore.error.deleting = Error deleting index ''{0}'' +datastore.error.deleting.missing.index = Cannot delete job - no index with id ''{0}'' in the database +datastore.error.executing.script = Error executing script ''{0}'' + +invalid.id = Invalid {0}; ''{1}'' must be lowercase alphanumeric, may contain hyphens or underscores, may not start with underscore +inconsistent.id = Inconsistent {0}; ''{1}'' specified in the body differs from ''{2}'' specified as a URL argument + +license.limit.detectors = Cannot create new job - your license limits you to {0,number,integer} detector(s), but you have configured {1,number,integer}. +license.limit.detectors.reactivate = Cannot reactivate job with id ''{0}'' - your license limits you to {1,number,integer} concurrently running detectors. You must close a job before you can reactivate another. +license.limit.jobs = Cannot create new job - your license limits you to {0,number,integer} concurrently running job(s). You must close a job before you can create a new one. +license.limit.jobs.reactivate = Cannot reactivate job with id ''{0}'' - your license limits you to {1,number,integer} concurrently running jobs. You must close a job before you can reactivate another. +license.limit.partitions = Cannot create new job - your license disallows partition fields, but you have configured one. + +job.audit.created = Job created +job.audit.deleted = Job deleted +job.audit.paused = Job paused +job.audit.resumed = Job resumed +job.audit.updated = Job updated: {0} +job.audit.reverted = Job model snapshot reverted to ''{0}'' +job.audit.old.results.deleted = Deleted results prior to {0} +job.audit.snapshot.deleted = Job model snapshot ''{0}'' deleted +job.audit.datafeed.started.from.to = Datafeed started (from: {0} to: {1}) +job.audit.datafeed.started.realtime = Datafeed started in real-time +job.audit.datafeed.continued.realtime = Datafeed continued in real-time +job.audit.datafeed.lookback.completed = Datafeed lookback completed +job.audit.datafeed.stopped = Datafeed stopped +job.audit.datafeed.no.data = Datafeed has been retrieving no data for a while +job.audit.datafeed.data.seen.again = Datafeed has started retrieving data again +job.audit.datafeed.data.analysis.error = Datafeed is encountering errors submitting data for analysis: {0} +job.audit.datafeed.data.extraction.error = Datafeed is encountering errors extracting data: {0} +job.audit.datafeed.recovered = Datafeed has recovered data extraction and analysis + +system.audit.started = System started +system.audit.shutdown = System shut down + +job.cannot.delete.while.running = Cannot delete job ''{0}'' while it is {1} +job.cannot.pause = Cannot pause job ''{0}'' while its status is {1} +job.cannot.resume = Cannot resume job ''{0}'' while its status is {1} + +job.config.byField.incompatible.function = by_field_name cannot be used with function ''{0}'' +job.config.byField.needs.another = by_field_name must be used in conjunction with field_name or function +job.config.categorization.filters.require.categorization.field.name = categorization_filters require setting categorization_field_name +job.config.categorization.filters.contains.duplicates = categorization_filters contain duplicates +job.config.categorization.filter.contains.empty = categorization_filters are not allowed to contain empty strings +job.config.categorization.filter.contains.invalid.regex = categorization_filters contains invalid regular expression ''{0}'' +job.config.condition.invalid.operator = Invalid operator for condition +job.config.condition.invalid.value.null = Invalid condition: the value field cannot be null +job.config.condition.invalid.value.numeric = Invalid condition value: cannot parse a double from string ''{0}'' +job.config.condition.invalid.value.regex = Invalid condition value: ''{0}'' is not a valid regular expression +job.config.detectionrule.condition.categorical.invalid.option = Invalid detector rule: a categorical rule_condition does not support {0} +job.config.detectionrule.condition.categorical.missing.option = Invalid detector rule: a categorical rule_condition requires {0} to be set +job.config.detectionrule.condition.invalid.fieldname = Invalid detector rule: field_name has to be one of {0}; actual was ''{1}'' +job.config.detectionrule.condition.missing.fieldname = Invalid detector rule: missing field_name in rule_condition where field_value ''{0}'' is set +job.config.detectionrule.condition.numerical.invalid.operator = Invalid detector rule: operator ''{0}'' is not allowed +job.config.detectionrule.condition.numerical.invalid.option = Invalid detector rule: a numerical rule_condition does not support {0} +job.config.detectionrule.condition.numerical.missing.option = Invalid detector rule: a numerical rule_condition requires {0} to be set +job.config.detectionrule.condition.numerical.with.fieldname.requires.fieldvalue = Invalid detector rule: a numerical rule_condition with field_name requires that field_value is set +job.config.detectionrule.invalid.targetfieldname = Invalid detector rule: target_field_name has to be one of {0}; actual was ''{1}'' +job.config.detectionrule.missing.targetfieldname = Invalid detector rule: missing target_field_name where target_field_value ''{0}'' is set +job.config.detectionrule.not.supported.by.function = Invalid detector rule: function {0} does not support rules +job.config.detectionrule.requires.at.least.one.condition = Invalid detector rule: at least one rule_condition is required +job.config.fieldname.incompatible.function = field_name cannot be used with function ''{0}'' +job.config.function.requires.byfield = by_field_name must be set when the ''{0}'' function is used +job.config.function.requires.fieldname = field_name must be set when the ''{0}'' function is used +job.config.function.requires.overfield = over_field_name must be set when the ''{0}'' function is used +job.config.function.incompatible.presummarized = The ''{0}'' function cannot be used in jobs that will take pre-summarized input +job.config.id.already.taken = The job cannot be created with the Id ''{0}''. The Id is already used. +job.config.id.too.long = The job id cannot contain more than {0,number,integer} characters. +job.config.invalid.fieldname.chars = Invalid field name ''{0}''. Field names including over, by and partition fields cannot contain any of these characters: {1} +job.config.invalid.timeformat = Invalid Time format string ''{0}'' +job.config.missing.analysisconfig = An analysis_config must be set +job.config.model.debug.config.invalid.bounds.percentile = Invalid model_debug_config: bounds_percentile must be in the range [0, 100] +job.config.field.value.too.low = {0} cannot be less than {1,number}. Value = {2,number} +job.config.no.analysis.field = One of function, field_name, by_field_name or over_field_name must be set +job.config.no.analysis.field.not.count = Unless the function is 'count' one of field_name, by_field_name or over_field_name must be set +job.config.no.detectors = No detectors configured +job.config.overField.incompatible.function = over_field_name cannot be used with function ''{0}'' +job.config.overField.needs.another = over_field_name must be used in conjunction with field_name or function +job.config.overlapping.buckets.incompatible.function = Overlapping buckets cannot be used with function ''{0}'' +job.config.multiple.bucketspans.require.bucket_span = Multiple bucket_spans require a bucket_span to be specified +job.config.multiple.bucketspans.must.be.multiple = Multiple bucket_span ''{0}'' must be a multiple of the main bucket_span ''{1}'' +job.config.per.partition.normalization.requires.partition.field = If the job is configured with Per-Partition Normalization enabled a detector must have a partition field +job.config.per.partition.normalization.cannot.use.influencers = A job configured with Per-Partition Normalization cannot use influencers + +job.config.update.analysis.limits.parse.error = JSON parse error reading the update value for analysis_limits +job.config.update.analysis.limits.cannot.be.null = Invalid update value for analysis_limits: null +job.config.update.analysis.limits.model.memory.limit.cannot.be.decreased = Invalid update value for analysis_limits: model_memory_limit cannot be decreased; existing is {0}, update had {1} +job.config.update.categorization.filters.invalid = Invalid update value for categorization_filters: value must be an array of strings; actual was: {0} +job.config.update.custom.settings.invalid = Invalid update value for custom_settings: value must be an object +job.config.update.description.invalid = Invalid update value for job description: value must be a string +job.config.update.detectors.invalid = Invalid update value for detectors: value must be an array +job.config.update.detectors.invalid.detector.index = Invalid index: valid range is [{0}, {1}]; actual was: {2} +job.config.update.detectors.detector.index.should.be.integer = Invalid index: integer expected; actual was: {0} +job.config.update.detectors.missing.params = Invalid update value for detectors: requires {0} and at least one of {1} +job.config.update.detectors.description.should.be.string = Invalid description: string expected; actual was: {0} +job.config.update.detectors.rules.parse.error = JSON parse error reading the update value for detectorRules +job.config.update.failed = Update failed. Please see the logs to trace the cause of the failure. +job.config.update.ignore.downtime.parse.error = Invalid update value for ignore_downtime: expected one of {0}; actual was: {1} +job.config.update.invalid.key = Invalid key ''{0}'' +job.config.update.job.is.not.closed = Cannot update key ''{0}'' while job is not closed; current status is {1} +job.config.update.model.debug.config.parse.error = JSON parse error reading the update value for ModelDebugConfig +job.config.update.requires.non.empty.object = Update requires JSON that contains a non-empty object +job.config.update.parse.error = JSON parse error reading the job update +job.config.update.background.persist.interval.invalid = Invalid update value for background_persist_interval: value must be an exact number of seconds no less than 3600 +job.config.update.renormalization.window.days.invalid = Invalid update value for renormalization_window_days: value must be an exact number of days +job.config.update.model.snapshot.retention.days.invalid = Invalid update value for model_snapshot_retention_days: value must be an exact number of days +job.config.update.results.retention.days.invalid = Invalid update value for results_retention_days: value must be an exact number of days +job.config.update.datafeed.config.parse.error = JSON parse error reading the update value for datafeed_config +job.config.update.datafeed.config.cannot.be.null = Invalid update value for datafeed_config: null +datafeed.config.cannot.use.script.fields.with.aggs = script_fields cannot be used in combination with aggregations + +job.config.unknown.function = Unknown function ''{0}'' + +job.index.already.exists = Cannot create index ''{0}'' as it already exists + +datafeed.config.invalid.option.value = Invalid {0} value ''{1}'' in datafeed configuration + +datafeed.does.not.support.job.with.latency = A job configured with datafeed cannot support latency +datafeed.aggregations.requires.job.with.summary.count.field = A job configured with a datafeed with aggregations must have summary_count_field_name ''{0}'' + +job.data.concurrent.use.close = Cannot close job {0} while another connection {2}is {1} the job +job.data.concurrent.use.flush = Cannot flush job {0} while another connection {2}is {1} the job +job.data.concurrent.use.update = Cannot update job {0} while it is in use +job.data.concurrent.use.upload = Cannot write to job {0} while another connection {2}is {1} the job + +job.missing.quantiles = Cannot read persisted quantiles for job ''{0}'' +job.unknown.id = No known job with id ''{0}'' + +datafeed.cannot.start = Cannot start datafeed [{0}] while its status is {1} +datafeed.cannot.stop.in.current.state = Cannot stop datafeed [{0}] while its status is {1} +datafeed.cannot.update.in.current.state = Cannot update datafeed [{0}] while its status is {1} +datafeed.cannot.delete.in.current.state = Cannot delete datafeed [{0}] while its status is {1} +datafeed.failed.to.stop = Failed to stop datafeed +datafeed.not.found = No datafeed with id [{0}] exists + +json.job.config.mapping.error = JSON mapping error reading the job configuration +json.job.config.parse.error = JSON parse error reading the job configuration + +json.detector.config.mapping.error = JSON mapping error reading the detector configuration +json.detector.config.parse.error = JSON parse error reading the detector configuration + +rest.action.not.allowed.for.datafeed.job = This action is not allowed for a datafeed job + +rest.invalid.datetime.params = Query param ''{0}'' with value ''{1}'' cannot be parsed as a date or converted to a number (epoch). +rest.invalid.flush.params.missing.argument = Invalid flush parameters: ''{0}'' has not been specified. +rest.invalid.flush.params.unexpected = Invalid flush parameters: unexpected ''{0}''. +rest.invalid.reset.params = Invalid reset range parameters: ''{0}'' has not been specified. +rest.invalid.from = Parameter 'from' cannot be < 0 +rest.invalid.size = Parameter 'size' cannot be < 0 +rest.invalid.from.size.sum = The sum of parameters ''from'' and ''size'' cannot be higher than {0}. Please use filters to reduce the number of results. +rest.start.after.end = Invalid time range: end time ''{0}'' is earlier than start time ''{1}''. +rest.reset.bucket.no.latency = Bucket resetting is not supported when no latency is configured. +rest.job.not.closed.revert = Can only revert to a model snapshot when the job is closed. +rest.no.such.model.snapshot = No matching model snapshot exists for job ''{0}'' +rest.description.already.used = Model snapshot description ''{0}'' has already been used for job ''{1}'' +rest.cannot.delete.highest.priority = Model snapshot ''{0}'' is the active snapshot for job ''{1}'', so cannot be deleted + +process.action.closed.job = closed +process.action.closing.job = closing +process.action.deleting.job = deleting +process.action.flushing.job = flushing +process.action.pausing.job = pausing +process.action.resuming.job = resuming +process.action.reverting.job = reverting the model snapshot for +process.action.sleeping.job = holding +process.action.updating.job = updating +process.action.writing.job = writing to + diff --git a/elasticsearch/src/main/resources/org/elasticsearch/xpack/ml/transforms/exact.properties b/elasticsearch/src/main/resources/org/elasticsearch/xpack/ml/transforms/exact.properties new file mode 100644 index 00000000000..df9a6058611 --- /dev/null +++ b/elasticsearch/src/main/resources/org/elasticsearch/xpack/ml/transforms/exact.properties @@ -0,0 +1,8105 @@ +co.gl=i +co.gg=i +av.it=i +watari.miyagi.jp=i +xn--srreisa-q1a.no=i +fot.br=i +co.gy=i +gov.ee=i +gov.eg=i +konyvelo.hu=i +tanohata.iwate.jp=i +tel=i +gov.et=i +fj.cn=i +london.museum=i +pics=i +xn--55qx5d=i +energy=i +indianapolis.museum=i +xn--rlingen-mxa.no=i +gov.ge=i +click=i +airport.aero=i +is-a-patsfan.org=p +cim.br=i +co.im=i +is-a-rockstar.com=p +ceb=i +co.in=i +glade=i +nishihara.okinawa.jp=i +bihoro.hokkaido.jp=i +date.hokkaido.jp=i +vald-aosta.it=i +co.it=i +nt.edu.au=i +ceo=i +co.ir=i +gold=i +skype=i +yamazoe.nara.jp=i +golf=i +gov.gn=i +gov.gh=i +gov.gi=i +tondabayashi.osaka.jp=i +co.je=i +hammarfeasta.no=i +gov.gr=i +cfa=i +flir=i +trento.it=i +cfd=i +bomlo.no=i +shimotsuma.ibaraki.jp=i +xn--ngbc5azd=i +xn--80ao21a=i +gov.ie=i +co.hu=i +ome.tokyo.jp=i +thd=i +vgs.no=i +sale=i +gov.hk=i +historichouses.museum=i +淡马锡=i +agents.aero=i +co.id=i +co.ca=p +konskowola.pl=i +co.cm=i +co.cl=i +co.ci=i +gov.in=i +analytics=i +xn--smna-gra.no=i +xn--1qqw23a=i +co.cr=i +fitjar.no=i +gov.ir=i +uto.kumamoto.jp=i +gov.iq=i +gov.it=i +gov.is=i +kaminokawa.tochigi.jp=i +tmp.br=i +co.bb=i +中文网=i +eurovision=i +kasamatsu.gifu.jp=i +co.ba=i +gov.kg=i +modalen.no=i +eidsvoll.no=i +gov.ki=i +miyada.nagano.jp=i +co.bi=i +shikaoi.hokkaido.jp=i +gov.jo=i +co.bw=i +ringerike.no=i +shingu.wakayama.jp=i +兵庫.jp=i +przeworsk.pl=i +stpetersburg.museum=i +environmentalconservation.museum=i +alfaromeo=i +gov.la=i +gov.lc=i +gov.lb=i +tjx=i +trani-andria-barletta.it=i +from-az.net=p +gov.lk=i +gov.kp=i +shichinohe.aomori.jp=i +nagatoro.saitama.jp=i +sørfold.no=i +futtsu.chiba.jp=i +gov.kn=i +gov.km=i +ostrowiec.pl=i +gdynia.pl=p +cc.nj.us=i +trust.museum=i +gov.kz=i +gov.ky=i +gov.ma=i +lefrak=i +السعودیۃ=i +forde.no=i +katsuragi.wakayama.jp=i +gov.ml=i +gov.mk=i +maibara.shiga.jp=i +gov.me=i +room=i +gov.mg=i +architecture.museum=i +goog=i +gov.lr=i +missoula.museum=i +muncie.museum=i +isa-hockeynut.com=p +karpacz.pl=i +sapo=i +shari.hokkaido.jp=i +enna.it=i +gov.ly=i +toyoake.aichi.jp=i +कॉम=i +kokubunji.tokyo.jp=i +gov.lt=i +個人.hk=i +gov.lv=i +genting=i +shonai.yamagata.jp=i +السعودیة=i +yatsushiro.kumamoto.jp=i +better-than.tv=p +cc.ms.us=i +sola.no=i +gov.ng=i +iron.museum=i +gov.mr=i +gov.ms=i +gov.mn=i +journalist.aero=i +gov.mo=i +gov.my=i +gov.mv=i +gov.mu=i +gov.mw=i +佐賀.jp=i +meløy.no=i +xn--kput3i=i +gov.om=i +gov.nr=i +sydney=i +kárášjohka.no=i +hikari.yamaguchi.jp=i +røros.no=i +prof.pr=i +coach=i +sarl=i +lillehammer.no=i +hita.oita.jp=i +co.ae=i +pink=i +air-surveillance.aero=i +山形.jp=i +ping=i +save=i +nishiazai.shiga.jp=i +gov.pl=i +protection=i +gov.pn=i +dabur=i +gov.ph=i +culture.museum=i +co.ag=i +gov.pk=i +nyuzen.toyama.jp=i +ando.nara.jp=i +hirakata.osaka.jp=i +xn--80aswg=i +co.at=i +higashinaruse.akita.jp=i +top=i +nærøy.no=i +federation.aero=i +gok.pk=i +co.ao=i +jp.eu.org=p +mikawa.yamagata.jp=i +myphotos.cc=p +artcenter.museum=i +gotv=i +association.museum=i +இலங்கை=i +kamoenai.hokkaido.jp=i +watarai.mie.jp=i +gov.qa=i +haus=i +barclaycard=i +xn--80au.xn--90a3ac=i +obu.aichi.jp=i +newyork.museum=i +nakano.tokyo.jp=i +k12.co.us=i +gov.pt=i +konan.shiga.jp=i +gov.ps=i +gorizia.it=i +tsuiki.fukuoka.jp=i +gov.pr=i +mb.ca=i +gov.py=i +com=i +alaska.museum=i +chikugo.fukuoka.jp=i +aeroport.fr=i +of.by=i +xn--fiq228c5hs=i +trader.aero=i +discovery.museum=i +gifu.gifu.jp=i +gs.rl.no=i +health.museum=i +s.se=i +muenchen.museum=i +xn--mk1bu44c=i +mihama.mie.jp=i +ørland.no=i +fhsk.se=i +nativeamerican.museum=i +gov.sh=i +gov.sg=i +gov.sd=i +gov.sc=i +h.se=i +exhibition.museum=i +kamo.kyoto.jp=i +gov.sl=i +vipsinaapp.com=p +christmas=i +kaneyama.fukushima.jp=i +gov.rw=i +markets=i +gov.rs=i +gov.ru=i +trv=i +gov.sb=i +takanabe.miyazaki.jp=i +gov.sa=i +xn--tor131o.jp=i +gov.tj=i +tsubata.ishikawa.jp=i +kawagoe.saitama.jp=i +saitama.saitama.jp=i +gov.tr=i +crimea.ua=i +gov.tm=i +gov.tl=i +gov.to=i +gov.tn=i +gov.sx=i +fukuoka.jp=i +gov.sy=i +gov.st=i +omiya.saitama.jp=i +sekigahara.gifu.jp=i +yamanobe.yamagata.jp=i +xn--wgbl6a=i +inderoy.no=i +tamaki.mie.jp=i +matsukawa.nagano.jp=i +hyuga.miyazaki.jp=i +parachuting.aero=i +gov.uk=i +alipay=i +crs=i +whoswho=i +izumo.shimane.jp=i +tysvær.no=i +warman=i +saxo=i +grimstad.no=i +csc=i +k12.ak.us=i +gov.tt=i +flickr=i +gov.tw=i +newspaper.museum=i +gov.ua=i +takayama.nagano.jp=i +productions=i +brindisi.it=i +iizuna.nagano.jp=i +trd.br=i +网址=i +hotel.tz=i +hyatt=i +casino.hu=i +tv.na=i +tui=i +zachpomor.pl=i +åkrehamn.no=i +miyako.fukuoka.jp=i +takehara.hiroshima.jp=i +from-dc.com=p +yoshinogari.saga.jp=i +wolterskluwer=i +neat-url.com=p +tvedestrand.no=i +miyagi.jp=i +baltimore.museum=i +honeywell=i +poltava.ua=i +po.gov.pl=i +anan.nagano.jp=i +from-wi.com=p +kosai.shizuoka.jp=i +serveftp.net=p +nj.us=i +suwa.nagano.jp=i +tvs=i +saratov.ru=i +eisenbahn.museum=i +fuossko.no=i +chicago.museum=i +olayan=i +onyourside=i +大分.jp=i +co.za=i +shikokuchuo.ehime.jp=i +kumamoto.jp=i +kitchen=i +saskatchewan.museum=i +omotego.fukushima.jp=i +sango.nara.jp=i +foodnetwork=i +team=i +services=i +kwpsp.gov.pl=i +فلسطين=i +marketing=i +xn--vard-jra.no=i +watch=i +iwi.nz=i +ekloges.cy=i +gs.of.no=i +consultant.aero=i +vardo.no=i +kafjord.no=i +taxi.br=i +co.st=i +бел=i +tv.im=i +ieee=i +tv.it=i +co.sz=i +co.th=i +us-east-1.amazonaws.com=p +bearalváhki.no=i +tver.ru=i +makurazaki.kagoshima.jp=i +presidio.museum=i +miyashiro.saitama.jp=i +fukui.fukui.jp=i +xn--mgbaam7a8h=i +za.bz=p +xn--h1aegh.museum=i +kobayashi.miyazaki.jp=i +co.tm=i +co.tj=i +tech=i +dlugoleka.pl=i +co.rw=i +shinjuku.tokyo.jp=i +glass=i +realestate.pl=i +dudinka.ru=i +xn--55qw42g=i +co.rs=i +snasa.no=i +fukagawa.hokkaido.jp=i +tsuwano.shimane.jp=i +bieszczady.pl=i +vlaanderen.museum=i +posts-and-telecommunications.museum=i +khakassia.ru=i +pol.dz=i +hsbc=i +co.uz=i +co.us=i +lebtimnetz.de=p +kawai.nara.jp=i +aquarium.museum=i +farmers=i +house=i +bbs.tr=i +råholt.no=i +co.vi=i +co.ve=i +khakassia.su=i +isa.us=i +naturhistorisches.museum=i +warabi.saitama.jp=i +klæbu.no=i +urn.arpa=i +st.no=i +co.tt=i +manno.kagawa.jp=i +co.ua=p +xn--gmq050i.hk=i +from-nd.com=p +co.tz=i +lifestyle=i +cc.ak.us=i +plants.museum=i +co.ug=i +frontdoor=i +k12.sc.us=i +osteroy.no=i +chitose.hokkaido.jp=i +pramerica=i +finance=i +co.uk=i +xn--seral-lra.no=i +krødsherad.no=i +lib.me.us=i +co.om=i +abbvie=i +polkowice.pl=i +hiraizumi.iwate.jp=i +xn--andy-ira.no=i +xn--efvy88h=i +co.pl=p +terni.it=i +prato.it=i +gallery=i +agrica.za=i +flowers=i +co.no=p +co.nl=p +co.nz=i +omega=i +dyndns-remote.com=p +from-fl.com=p +مليسيا=i +shiogama.miyagi.jp=i +botanical.museum=i +malvik.no=i +railway.museum=i +nordre-land.no=i +nanto.toyama.jp=i +broker.aero=i +gs.oslo.no=i +fundacio.museum=i +شبكة=i +warszawa.pl=i +turek.pl=i +av.tr=i +holdings=i +s3-us-west-2.amazonaws.com=p +iheya.okinawa.jp=i +es.kr=i +co.pn=i +ak.us=i +kuzumaki.iwate.jp=i +uvic.museum=i +co.pw=i +saijo.ehime.jp=i +mito.ibaraki.jp=i +xn--hcesuolo-7ya35b.no=i +mitoyo.kagawa.jp=i +e12.ve=i +obama.nagasaki.jp=i +sweden.museum=i +minamitane.kagoshima.jp=i +bari.it=i +skanland.no=i +gov.au=i +gov.ar=i +dad=i +gov.as=i +ubs=i +zoology.museum=i +gov.ba=i +한국=i +gangaviika.no=i +gov.az=i +co.kr=i +gov.af=i +utsira.no=i +gov.ae=i +kitagawa.miyazaki.jp=i +is-a-techie.com=p +gov.ac=i +audible=i +dnsdojo.net=p +xn--zf0ao64a.tw=i +boleslawiec.pl=i +shimofusa.chiba.jp=i +zentsuji.kagawa.jp=i +day=i +gov.al=i +agro.pl=i +yekaterinburg.ru=i +co.lc=i +gov.by=i +tv.sd=i +gov.bs=i +gov.br=i +gov.bt=i +varese.it=i +cz.it=i +fuoisku.no=i +vrn.ru=i +oyama.tochigi.jp=i +wakayama.jp=i +gov.bz=i +co.jp=i +gov.bf=i +gov.bh=i +gov.bb=i +us-gov-west-1.compute.amazonaws.com=p +gov.bo=i +yonezawa.yamagata.jp=i +gov.bm=i +hn.cn=i +gov.cx=i +gildeskål.no=i +gov.cy=i +wskr.gov.pl=i +gov.cu=i +co.mw=i +selfip.org=p +co.mu=i +k12.vi.us=i +nsk.ru=i +bodø.no=i +co.na=i +gov.cd=i +govt.nz=i +drammen.no=i +gov.co=i +agrigento.it=i +florø.no=i +sor-odal.no=i +gov.cl=i +gov.cn=i +gov.cm=i +gov.dz=i +firestone=i +est-mon-blogueur.com=p +loyalist.museum=i +yokaichiba.chiba.jp=i +is-into-games.com=p +taiji.wakayama.jp=i +gov.ec=i +tv.tz=i +takahama.fukui.jp=i +takagi.nagano.jp=i +annaka.gunma.jp=i +co.ls=i +co.ma=i +dds=i +k12.vt.us=i +tv.tr=i +build=i +xn--ygbi2ammx=i +co.mg=i +redumbrella=i +co.me=i +gov.dm=i +gov.do=i +tuva.su=i +tas.au=i +lel.br=i +hakodate.hokkaido.jp=i +yonaguni.okinawa.jp=i +lazio.it=i +harima.hyogo.jp=i +dev=i +stada=i +sakado.saitama.jp=i +nuremberg.museum=i +ivgu.no=i +tuva.ru=i +xn--4pvxs.jp=i +oregontrail.museum=i +artanddesign.museum=i +barum.no=i +xn--btsfjord-9za.no=i +chichibu.saitama.jp=i +ms.it=i +investments=i +fussa.tokyo.jp=i +meeres.museum=i +aomori.aomori.jp=i +xn--holtlen-hxa.no=i +ntr.br=i +fvg.it=i +我爱你=i +xn--1ctwo.jp=i +torino.it=i +latrobe=i +unnan.shimane.jp=i +tsuyama.okayama.jp=i +business=i +dhl=i +yoita.niigata.jp=i +mutsu.aomori.jp=i +shoes=i +xn--k7yn95e.jp=i +trentino-sudtirol.it=i +naturalsciences.museum=i +kanegasaki.iwate.jp=i +merseine.nu=p +amli.no=i +ikano=i +hadano.kanagawa.jp=i +kasaoka.okayama.jp=i +frog.museum=i +ascolipiceno.it=i +قطر=i +hotel.hu=i +zamami.okinawa.jp=i +uji.kyoto.jp=i +gs.mr.no=i +okinawa.jp=i +yoro.gifu.jp=i +homelinux.com=p +lib.tn.us=i +magadan.ru=i +ms.kr=i +larsson.museum=i +fukumitsu.toyama.jp=i +fujikawa.shizuoka.jp=i +essex.museum=i +gs.nt.no=i +lerdal.no=i +takahagi.ibaraki.jp=i +kaisei.kanagawa.jp=i +final=i +toyota=i +xn--nmesjevuemie-tcba.no=i +hamura.tokyo.jp=i +loan=i +usarts.museum=i +treviso.it=i +gr.jp=i +training=i +schaeffler=i +codespot.com=p +yaese.okinawa.jp=i +takahashi.okayama.jp=i +uscountryestate.museum=i +xn--mgba3a4f16a.ir=i +company=i +bifuka.hokkaido.jp=i +dyndns-free.com=p +kyoto=i +sakyo.kyoto.jp=i +6.bg=i +xn--ystre-slidre-ujb.no=i +teva=i +kamisu.ibaraki.jp=i +time.museum=i +cinema.museum=i +gr.it=i +dvrdns.org=p +kamogawa.chiba.jp=i +tomiya.miyagi.jp=i +arts.co=i +nu.it=i +uno=i +hotel.lk=i +baby=i +is-a-photographer.com=p +ato.br=i +valdaosta.it=i +imizu.toyama.jp=i +press.aero=i +stathelle.no=i +langevåg.no=i +urasoe.okinawa.jp=i +tateyama.toyama.jp=i +village.museum=i +tourism.tn=i +nakadomari.aomori.jp=i +nishiokoppe.hokkaido.jp=i +cng.br=i +aland.fi=i +zarow.pl=i +uol=i +yuza.yamagata.jp=i +chloe=i +media.hu=i +xn--pssu33l.jp=i +umb.it=i +bálát.no=i +milan.it=i +usdecorativearts.museum=i +kawai.iwate.jp=i +wakasa.tottori.jp=i +barcelona.museum=i +aca.pro=i +dnp=i +oyamazaki.kyoto.jp=i +friulive-giulia.it=i +taranto.it=i +univ.sn=i +yawara.ibaraki.jp=i +nozawaonsen.nagano.jp=i +ups=i +capitalone=i +ol.no=i +dog=i +trading=i +carraramassa.it=i +si.it=i +pn.it=i +dot=i +bø.nordland.no=i +jerusalem.museum=i +x.bg=i +davvenjarga.no=i +dnsdojo.org=p +googleapis.com=p +rzgw.gov.pl=i +en.it=i +m.bg=i +pc.it=i +dupont=i +evenášši.no=i +rygge.no=i +quebec=i +b.br=i +s3-external-2.amazonaws.com=p +fe.it=i +tranoy.no=i +trentinostirol.it=i +b.bg=i +bradesco=i +kannami.shizuoka.jp=i +alta.no=i +maif=i +morioka.iwate.jp=i +amsterdam.museum=i +pc.pl=i +hi.cn=i +lib.nm.us=i +kanzaki.saga.jp=i +credit=i +tv.bo=i +andria-trani-barletta.it=i +artgallery.museum=i +xn--sr-varanger-ggb.no=i +koya.wakayama.jp=i +loft=i +大阪.jp=i +tv.br=i +arts.nf=i +bridgestone=i +nebraska.museum=i +homeunix.com=p +arna.no=i +tv.bb=i +lillesand.no=i +málatvuopmi.no=i +fidelity=i +maintenance.aero=i +godaddy=i +moåreke.no=i +ms.us=i +seranishi.hiroshima.jp=i +isen.kagoshima.jp=i +higashisumiyoshi.osaka.jp=i +bungotakada.oita.jp=i +dtv=i +genova.it=i +mango=i +minobu.yamanashi.jp=i +rg.it=i +dk.eu.org=p +xn--linds-pra.no=i +my.eu.org=p +yachiyo.ibaraki.jp=i +properties=i +ako.hyogo.jp=i +ltd.lk=i +observer=i +bank=i +shiki.saitama.jp=i +ing.pa=i +சிங்கப்பூர்=i +band=i +cc.hi.us=i +volkswagen=i +enebakk.no=i +uenohara.yamanashi.jp=i +ggf.br=i +arts.ro=i +dnsalias.com=p +zushi.kanagawa.jp=i +campobasso.it=i +czest.pl=i +rost.no=i +dwg=i +nesoddtangen.no=i +verbania.it=i +traniandriabarletta.it=i +dynathome.net=p +compute-1.amazonaws.com=p +xn--oppegrd-ixa.no=i +cc.sd.us=i +ide.kyoto.jp=i +xn--lrdal-sra.no=i +bykle.no=i +os.hedmark.no=i +gemological.museum=i +ભારત=i +yachiyo.chiba.jp=i +star=i +ulan-ude.ru=i +xn--3bst00m=i +wildlife.museum=i +chernigov.ua=i +nasu.tochigi.jp=i +sekikawa.niigata.jp=i +hakui.ishikawa.jp=i +takarazuka.hyogo.jp=i +honjo.akita.jp=i +xn--hpmir-xqa.no=i +journal.aero=i +nago.okinawa.jp=i +k12.tr=i +fareast.ru=i +monzabrianza.it=i +blogdns.com=p +selfip.net=p +сайт=i +ltd.hk=p +seven=i +xn--9dbq2a=i +lahppi.no=i +touch.museum=i +daisen.akita.jp=i +xn--mgbpl2fh=i +tottori.tottori.jp=i +arts.ve=i +kawanabe.kagoshima.jp=i +rissa.no=i +xn--rros-gra.no=i +gmina.pl=i +birthplace.museum=i +kitakami.iwate.jp=i +edeka=i +esurance=i +hemne.no=i +equipment.aero=i +rel.ht=i +spydeberg.no=i +assisi.museum=i +auction=i +apple=i +matsumoto.kagoshima.jp=i +sakegawa.yamagata.jp=i +укр=i +saito.miyazaki.jp=i +gateway.museum=i +mihama.fukui.jp=i +passagens=i +saigawa.fukuoka.jp=i +kyowa.hokkaido.jp=i +xn--cck2b3b=i +rhcloud.com=p +ltd.gi=i +um.gov.pl=i +matsuzaki.shizuoka.jp=i +láhppi.no=i +ulvik.no=i +gov.nc.tr=i +xn--bhccavuotna-k7a.no=i +loten.no=i +flynnhub.com=p +is-an-engineer.com=p +fukuchiyama.kyoto.jp=i +pyatigorsk.ru=i +kashiba.nara.jp=i +wa.us=i +navigation.aero=i +ouda.nara.jp=i +奈良.jp=i +aizubange.fukushima.jp=i +ravenna.it=i +jx.cn=i +meguro.tokyo.jp=i +lib.wi.us=i +in-addr.arpa=i +haugesund.no=i +otofuke.hokkaido.jp=i +scientist.aero=i +kokonoe.oita.jp=i +tokigawa.saitama.jp=i +miyako.iwate.jp=i +setagaya.tokyo.jp=i +work=i +is-a-musician.com=p +radoy.no=i +kouzushima.tokyo.jp=i +udmurtia.ru=i +tattoo=i +香川.jp=i +ltd.uk=i +kiyosato.hokkaido.jp=i +tatsuno.hyogo.jp=i +rel.pl=i +slattum.no=i +ng.eu.org=p +jaworzno.pl=i +ca.eu.org=p +love=i +m.se=i +hi.us=i +kiyosu.aichi.jp=i +lucania.it=i +foundation=i +philadelphia.museum=i +eu.int=i +mortgage=i +is-a-doctor.com=p +cultural.museum=i +eat=i +中信=i +k12.ec=i +cc.wa.us=i +hakuba.nagano.jp=i +klodzko.pl=i +kyonan.chiba.jp=i +asakawa.fukushima.jp=i +tendo.yamagata.jp=i +b.se=i +rochester.museum=i +sannohe.aomori.jp=i +pmn.it=i +is-a-bulls-fan.com=p +iwakura.aichi.jp=i +russia.museum=i +rubtsovsk.ru=i +writesthisblog.com=p +k12.ga.us=i +grane.no=i +nu.ca=i +vet=i +香港=i +vang.no=i +nagi.okayama.jp=i +surgut.ru=i +legnica.pl=i +kitakata.miyazaki.jp=i +kuokgroup=i +tuscany.it=i +edu=i +xn--brum-voa.no=i +x.se=i +otaru.hokkaido.jp=i +tickets=i +नेट=i +malatvuopmi.no=i +mugi.tokushima.jp=i +insure=i +furukawa.miyagi.jp=i +urayasu.chiba.jp=i +laz.it=i +yoshikawa.saitama.jp=i +shirosato.ibaraki.jp=i +chuo.tokyo.jp=i +fukusaki.hyogo.jp=i +trentino-sud-tirol.it=i +monmouth.museum=i +s3-us-gov-west-1.amazonaws.com=p +bådåddjå.no=i +field.museum=i +steigen.no=i +ভারত=i +farmstead.museum=i +xn--czrw28b.tw=i +farmequipment.museum=i +myoko.niigata.jp=i +xn--vuq861b=i +chosei.chiba.jp=i +scot=i +fribourg.museum=i +edu.eu.org=p +рф=i +landrover=i +scor=i +sanok.pl=i +is-a-therapist.com=p +randaberg.no=i +電訊盈科=i +våler.østfold.no=i +vig=i +itakura.gunma.jp=i +is-a-liberal.com=p +vin=i +hatogaya.saitama.jp=i +traeumtgerade.de=p +name.cy=i +vip=i +sakaki.nagano.jp=i +sh.cn=i +priv.no=i +xn--pgbs0dh=i +koto.tokyo.jp=i +shinagawa.tokyo.jp=i +friulivenezia-giulia.it=i +grajewo.pl=i +community=i +katashina.gunma.jp=i +shop.ht=i +shop.hu=i +trentinosuedtirol.it=i +e164.arpa=i +vista=i +gsm.pl=i +name.eg=i +uconnect=i +kahoku.ishikawa.jp=i +oi.kanagawa.jp=i +name.et=i +広島.jp=i +cisco=i +asti.it=i +yahiko.niigata.jp=i +hitra.no=i +trogstad.no=i +tj.cn=i +xn--gecrj9c=i +ericsson=i +priv.me=i +izumizaki.fukushima.jp=i +lease=i +yamato.kanagawa.jp=i +ferrara.it=i +vinnytsia.ua=i +nom.ad=i +nordreisa.no=i +monza.it=i +nom.ag=i +rnd.ru=i +nesseby.no=i +ct.us=i +torahime.shiga.jp=i +tomioka.gunma.jp=i +florist=i +kindle=i +bostik=i +mo-i-rana.no=i +mr.no=i +monza-brianza.it=i +xn--porsgu-sta26f.no=i +nt.no=i +media.museum=i +stockholm.museum=i +chiropractic.museum=i +gs.hm.no=i +lucerne.museum=i +kikugawa.shizuoka.jp=i +shop.pl=i +xn--avery-yua.no=i +hs.kr=i +nayoro.hokkaido.jp=i +kamisunagawa.hokkaido.jp=i +hiratsuka.kanagawa.jp=i +kagami.kochi.jp=i +nom.co=i +musashimurayama.tokyo.jp=i +xn--skjervy-v1a.no=i +workshop.museum=i +mragowo.pl=i +alessandria.it=i +nanao.ishikawa.jp=i +priv.pl=i +xn--gjvik-wua.no=i +name.az=i +is-a-linux-user.org=p +ovre-eiker.no=i +seoul.kr=i +lt.eu.org=p +xn--lrenskog-54a.no=i +rybnik.pl=i +nsw.edu.au=i +forum=i +linde=i +nakagusuku.okinawa.jp=i +balsfjord.no=i +ogano.saitama.jp=i +andoy.no=i +spiegel=i +progressive=i +is-a-anarchist.com=p +education=i +obuse.nagano.jp=i +ky.us=i +swiebodzin.pl=i +gáivuotna.no=i +gr.eu.org=p +sicily.it=i +ichikawa.chiba.jp=i +dinosaur.museum=i +africamagic=i +friuli-venezia-giulia.it=i +sa-east-1.compute.amazonaws.com=p +misato.shimane.jp=i +nom.es=i +ltd.cy=i +valleeaoste.it=i +se.eu.org=p +tempioolbia.it=i +from-ak.com=p +nonoichi.ishikawa.jp=i +khv.ru=i +nt.ro=i +nagahama.shiga.jp=i +shimonoseki.yamaguchi.jp=i +موبايلي=i +scjohnson=i +name.jo=i +balsan.it=i +shizuoka.shizuoka.jp=i +xn--j6w193g=i +design.museum=i +okutama.tokyo.jp=i +nom.fr=i +kudamatsu.yamaguchi.jp=i +marumori.miyagi.jp=i +shimosuwa.nagano.jp=i +kasama.ibaraki.jp=i +hitachiota.ibaraki.jp=i +name.mk=i +name.my=i +name.mv=i +kep.tr=i +from-wy.com=p +bergamo.it=i +uy.com=p +name.na=i +name.ng=i +takayama.gunma.jp=i +hk.com=p +sarpsborg.no=i +eniwa.hokkaido.jp=i +sor-aurdal.no=i +网店=i +lplfinancial=i +sharp=i +higashikagawa.kagawa.jp=i +fhv.se=i +americanart.museum=i +embaixada.st=i +loabat.no=i +kitaaiki.nagano.jp=i +esq=i +est.pr=i +family.museum=i +hermes=i +hikone.shiga.jp=i +岩手.jp=i +est-a-la-masion.com=p +nishi.fukuoka.jp=i +estate=i +axis.museum=i +lindesnes.no=i +careers=i +union.aero=i +ok.us=i +kimobetsu.hokkaido.jp=i +eus=i +deloitte=i +xn--nit225k.jp=i +askoy.no=i +financial=i +nom.km=i +qld.gov.au=i +prudential=i +media.aero=i +tonosho.kagawa.jp=i +scrapper-site.net=p +vodka=i +nes.buskerud.no=i +troitsk.su=i +namdalseid.no=i +house.museum=i +badaddja.no=i +ozu.ehime.jp=i +xn--jvr189m=i +uchinada.ishikawa.jp=i +priv.hu=i +name.hr=i +ss.it=i +schokoladen.museum=i +xn--vermgensberatung-pwb=i +aurskog-holand.no=i +takatsuki.osaka.jp=i +tsurugi.ishikawa.jp=i +nom.mg=i +stavern.no=i +kartuzy.pl=i +xn--klt787d.jp=i +chambagri.fr=i +nord-odal.no=i +taxi.aero=i +xn--stre-toten-zcb.no=i +misconfused.org=p +airline.aero=i +saikai.nagasaki.jp=i +rockart.museum=i +kanan.osaka.jp=i +omigawa.chiba.jp=i +laakesvuemie.no=i +pug.it=i +watch-and-clock.museum=i +gs.svalbard.no=i +dielddanuorri.no=i +cern=i +eco.br=i +planetarium.museum=i +health.vn=i +leg.br=i +okinawa.okinawa.jp=i +tysfjord.no=i +from-ct.com=p +itoman.okinawa.jp=i +lib.gu.us=i +kasuga.fukuoka.jp=i +krokstadelva.no=i +xn--efvn9s.jp=i +pvt.ge=i +ambulance.museum=i +fosnes.no=i +taobao=i +xn--nqv7f=i +kawaue.gifu.jp=i +kvæfjord.no=i +western.museum=i +tenri.nara.jp=i +black=i +xbox=i +xn--krdsherad-m8a.no=i +zaporizhzhia.ua=i +anthropology.museum=i +xn--djty4k.jp=i +takahama.aichi.jp=i +griw.gov.pl=i +ikeda.nagano.jp=i +nakatombetsu.hokkaido.jp=i +itayanagi.aomori.jp=i +tel.tr=i +oamishirasato.chiba.jp=i +hdfc=i +tourism.pl=i +cambridge.museum=i +sula.no=i +microsoft=i +jl.cn=i +cc.vt.us=i +furniture=i +naklo.pl=i +tokai.ibaraki.jp=i +كوم=i +odo.br=i +taka.hyogo.jp=i +cc.wv.us=i +city.hu=i +volkenkunde.museum=i +modern.museum=i +miyoshi.hiroshima.jp=i +lombardia.it=i +beskidy.pl=i +valer.ostfold.no=i +skjåk.no=i +physio=i +hokuto.yamanashi.jp=i +ایران=i +navuotna.no=i +egyptian.museum=i +wa.gov.au=i +limanowa.pl=i +ibestad.no=i +museum=i +szczytno.pl=i +clothing=i +doomdns.com=p +brussel.museum=i +kushiro.hokkaido.jp=i +xn--uc0atv.tw=i +sar.it=i +historisch.museum=i +next=i +vestvagoy.no=i +bbva=i +xn--eveni-0qa01ga.no=i +hamatonbetsu.hokkaido.jp=i +cpa.pro=i +yamanashi.yamanashi.jp=i +shinjo.nara.jp=i +购物=i +wv.us=i +keisen.fukuoka.jp=i +wiw.gov.pl=i +0.bg=i +education.museum=i +sibenik.museum=i +evenassi.no=i +memorial=i +morotsuka.miyazaki.jp=i +minamiawaji.hyogo.jp=i +hjelmeland.no=i +cc.vi.us=i +news=i +green=i +yaita.tochigi.jp=i +پاکستان=i +folldal.no=i +swiftcover=i +xn--rht61e.jp=i +abeno.osaka.jp=i +xn--kfjord-iua.no=i +cn.eu.org=p +dynalias.net=p +xn--j1aef=i +play=i +g.bg=i +munakata.fukuoka.jp=i +dominic.ua=i +fan=i +s3.amazonaws.com=p +giehtavuoatna.no=i +aknoluokta.no=i +nt.au=i +بيتك=i +health.nz=i +xn--bod-2na.no=i +firmdale=i +sandnessjoen.no=i +narviika.no=i +brunel.museum=i +gop.pk=i +tranibarlettaandria.it=i +toyama.toyama.jp=i +sos.pl=i +xn--mk0axi.hk=i +study=i +smolensk.ru=i +newjersey.museum=i +skaun.no=i +irish=i +mosjoen.no=i +k12.ct.us=i +buryatia.ru=i +hanamaki.iwate.jp=i +geek.nz=i +showa.gunma.jp=i +ås.no=i +hellas.museum=i +wed=i +lenvik.no=i +radøy.no=i +extraspace=i +berlin=i +flatanger.no=i +wada.nagano.jp=i +intuit=i +vegas=i +toyokawa.aichi.jp=i +naturalhistory.museum=i +gamo.shiga.jp=i +cc.ct.us=i +convent.museum=i +otama.fukushima.jp=i +ojiya.niigata.jp=i +r.bg=i +okinoshima.shimane.jp=i +kazo.saitama.jp=i +kragerø.no=i +nogata.fukuoka.jp=i +booking=i +sejny.pl=i +togakushi.nagano.jp=i +livinghistory.museum=i +lib.in.us=i +rogers=i +hornindal.no=i +k12.id.us=i +xn--rhqv96g=i +vardø.no=i +donetsk.ua=i +vladimir.su=i +psc.br=i +belgorod.ru=i +hitachiomiya.ibaraki.jp=i +marker.no=i +nagai.yamagata.jp=i +vermögensberater=i +voyage=i +minamimaki.nagano.jp=i +ichinoseki.iwate.jp=i +from-il.com=p +xn--snsa-roa.no=i +shiroi.chiba.jp=i +asaka.saitama.jp=i +toda.saitama.jp=i +raid=i +bjerkreim.no=i +vladimir.ru=i +video.hu=i +game=i +gotdns.org=p +mizumaki.fukuoka.jp=i +salerno.it=i +nt.ca=i +minami-alps.yamanashi.jp=i +faith=i +ikawa.akita.jp=i +mifune.kumamoto.jp=i +kaita.hiroshima.jp=i +volyn.ua=i +allstate=i +kimino.wakayama.jp=i +macerata.it=i +xn--mgbqly7c0a67fbc=i +xn--8pvr4u.jp=i +diet=i +kiyama.saga.jp=i +win=i +lawyer=i +le.it=i +soccer=i +dontexist.com=p +bibai.hokkaido.jp=i +palermo.it=i +is-an-actress.com=p +service.gov.uk=p +verdal.no=i +xn--j1amh=i +kristiansand.no=i +iwaki.fukushima.jp=i +pb.ao=i +oguchi.aichi.jp=i +miki.hyogo.jp=i +ascoli-piceno.it=i +vistaprint=i +aridagawa.wakayama.jp=i +fit=i +xn--lesund-hua.no=i +otaki.chiba.jp=i +jølster.no=i +blogspot.re=p +tobe.ehime.jp=i +semine.miyagi.jp=i +surf=i +ac=i +ad=i +ae=i +blogspot.pt=p +af=i +ag=i +delmenhorst.museum=i +shika.ishikawa.jp=i +ai=i +tako.chiba.jp=i +al=i +am=i +an=i +ao=i +blogspot.qa=p +aq=i +ar=i +as=i +at=i +au=i +ogose.saitama.jp=i +aw=i +ax=i +az=i +xn--uc0atv.hk=i +ba=i +xn--mgbca7dzdo=i +bb=i +from-ca.com=p +nishikatsura.yamanashi.jp=i +be=i +bf=i +bg=i +emiliaromagna.it=i +bh=i +bi=i +bj=i +blogspot.sk=p +volda.no=i +blogspot.si=p +bm=i +tjome.no=i +bo=i +wme=i +blogspot.sn=p +wakayama.wakayama.jp=i +blogspot.td=p +br=i +cc.ga.us=i +bs=i +bt=i +kadogawa.miyazaki.jp=i +iwate.iwate.jp=i +bv=i +systems=i +bw=i +by=i +bz=i +achi.nagano.jp=i +hápmir.no=i +tamayu.shimane.jp=i +uk.eu.org=p +valer.hedmark.no=i +ca=i +cc=i +trentino-stirol.it=i +cd=i +blogspot.rs=p +cf=i +cg=i +ch=i +chernivtsi.ua=i +ci=i +sc.tz=i +blogspot.ru=p +cl=i +cm=i +cn=i +co=i +blogspot.ro=p +americanexpress=i +cr=i +sc.ug=i +cu=i +fly=i +med.pro=i +沖縄.jp=i +cv=i +blogspot.sg=p +kongsvinger.no=i +tatamotors=i +trentino-suedtirol.it=i +cw=i +cx=i +blogspot.se=p +selje.no=i +lødingen.no=i +cz=i +الجزائر=i +freiburg.museum=i +mil.za=i +googlecode.com=p +sc.us=i +isa-geek.net=p +shikatsu.aichi.jp=i +de=i +dyndns.info=p +blogspot.mk=p +blogspot.mr=p +dj=i +dk=i +gniezno.pl=i +dm=i +varggat.no=i +blogspot.md=p +ford=i +do=i +aid.pl=i +entertainment.aero=i +showtime=i +dz=i +sørreisa.no=i +blogspot.my=p +在线=i +ec=i +blogspot.mx=p +ee=i +alaheadju.no=i +eg=i +kunigami.okinawa.jp=i +信息=i +hiji.oita.jp=i +akagi.shimane.jp=i +xn--qqqt11m.jp=i +kakamigahara.gifu.jp=i +blogspot.li=p +temasek=i +horokanai.hokkaido.jp=i +koganei.tokyo.jp=i +es=i +et=i +eu=i +thruhere.net=p +groks-the.info=p +minamiyamashiro.kyoto.jp=i +blogspot.lt=p +blogspot.lu=p +inami.toyama.jp=i +michigan.museum=i +hole.no=i +salon=i +utah.museum=i +hobol.no=i +fi=i +gyeongbuk.kr=i +mazowsze.pl=i +fm=i +foo=i +fo=i +cesenaforli.it=i +fr=i +bedzin.pl=i +timekeeping.museum=i +witd.gov.pl=i +kurashiki.okayama.jp=i +pol.tr=i +toyama.jp=i +blogspot.pe=p +iamallama.com=p +ga=i +gb=i +gd=i +ge=i +gf=i +gg=i +digital=i +gh=i +blogspot.no=p +gi=i +blogspot.nl=p +gl=i +gm=i +miners.museum=i +gn=i +asakuchi.okayama.jp=i +amakusa.kumamoto.jp=i +gp=i +vi.it=i +gq=i +kiryu.gunma.jp=i +gr=i +gs=i +shirataka.yamagata.jp=i +iijima.nagano.jp=i +gt=i +vt.it=i +gw=i +mil.vc=i +gy=i +mil.ve=i +ushuaia.museum=i +floro.no=i +moma.museum=i +shimabara.nagasaki.jp=i +mil.uy=i +shingo.aomori.jp=i +sør-odal.no=i +piw.gov.pl=i +hábmer.no=i +choyo.kumamoto.jp=i +hk=i +இந்தியா=i +stavropol.ru=i +hm=i +sukumo.kochi.jp=i +hn=i +juif.museum=i +hr=i +ht=i +hu=i +bergbau.museum=i +cc.co.us=i +g.se=i +vi.us=i +mil.tw=i +id=i +ie=i +mil.tz=i +abarth=i +kahoku.yamagata.jp=i +xn--djrs72d6uy.jp=i +mil.to=i +frl=i +pol.ht=i +minamiizu.shizuoka.jp=i +im=i +nemuro.hokkaido.jp=i +in=i +anamizu.ishikawa.jp=i +mil.tr=i +io=i +wtc=i +graz.museum=i +wtf=i +flsmidth=i +iq=i +ir=i +ic.gov.pl=i +modelling.aero=i +is=i +matsuda.kanagawa.jp=i +it=i +mil.tj=i +mil.tm=i +hitachi.ibaraki.jp=i +xn--b-5ga.telemark.no=i +je=i +s3.eu-central-1.amazonaws.com=p +clinton.museum=i +klepp.no=i +mil.sy=i +kamijima.ehime.jp=i +shiso.hyogo.jp=i +lupin=i +alstahaug.no=i +jo=i +mill.museum=i +askøy.no=i +jp=i +mil.st=i +xn--givuotna-8ya.no=i +soni.nara.jp=i +mil.sh=i +k12.de.us=i +mil.ru=i +mil.rw=i +krasnodar.su=i +kg=i +namsos.no=i +tenei.fukushima.jp=i +ki=i +tsu.mie.jp=i +km=i +kn=i +hapmir.no=i +kanna.gunma.jp=i +kp=i +ftr=i +kr=i +midtre-gauldal.no=i +ky=i +kz=i +kitanakagusuku.okinawa.jp=i +taki.mie.jp=i +la=i +lb=i +lc=i +travelersinsurance=i +r.se=i +yanagawa.fukuoka.jp=i +vt.us=i +li=i +lk=i +xn--vrggt-xqad.no=i +iinet=i +lr=i +ls=i +xn--32vp30h.jp=i +lt=i +lu=i +lv=i +otari.nagano.jp=i +enterprises=i +photos=i +ly=i +locker=i +teramo.it=i +tokai.aichi.jp=i +ma=i +mc=i +mil.qa=i +md=i +me=i +mg=i +mh=i +mk=i +mil.py=i +ml=i +kai.yamanashi.jp=i +mn=i +mo=i +jeonbuk.kr=i +mil.pl=i +mp=i +mq=i +mr=i +xn--dnna-gra.no=i +karasjok.no=i +author=i +ms=i +2000.hu=i +sumy.ua=i +mt=i +blogspot.tw=p +mu=i +mv=i +mil.pe=i +xn--krjohka-hwab49j.no=i +rikuzentakata.iwate.jp=i +mw=i +mx=i +inagi.tokyo.jp=i +my=i +plus=i +mil.ph=i +funabashi.chiba.jp=i +na=i +nc=i +dish=i +blogspot.ug=p +ne=i +te.ua=i +nf=i +ng=i +prochowice.pl=i +nl=i +abu.yamaguchi.jp=i +meraker.no=i +fam.pk=i +no=i +tome.miyagi.jp=i +kawagoe.mie.jp=i +nr=i +nu=i +kazimierz-dolny.pl=i +storage=i +hiroshima.jp=i +marketplace.aero=i +nz=i +meland.no=i +mil.nz=i +セール=i +kaszuby.pl=i +fedje.no=i +uwajima.ehime.jp=i +psi.br=i +suzuki=i +namie.fukushima.jp=i +om=i +hjartdal.no=i +mil.no=i +yaroslavl.ru=i +blogspot.vn=p +mtpc=i +rendalen.no=i +ga.us=i +vaksdal.no=i +mil.ng=i +piemonte.it=i +cc.sc.us=i +pa=i +lindås.no=i +mil.my=i +pe=i +pf=i +fyi=i +cz.eu.org=p +ph=i +hamatama.saga.jp=i +kaneyama.yamagata.jp=i +luxury=i +elk.pl=i +noboribetsu.hokkaido.jp=i +pk=i +lancashire.museum=i +pl=i +pm=i +pn=i +mil.mv=i +yamagata.gifu.jp=i +mil.mg=i +pr=i +net.lv=i +ps=i +pt=i +pw=i +net.ly=i +net.lk=i +baths.museum=i +mb.it=i +py=i +med.sa=i +gov.vn=i +med.sd=i +net.lr=i +qa=i +ci.it=i +br.it=i +net.lc=i +mil.lv=i +minamata.kumamoto.jp=i +rovno.ua=i +s3.cn-north-1.amazonaws.com.cn=p +kure.hiroshima.jp=i +gov.vc=i +net.kz=i +toshiba=i +gov.ve=i +habmer.no=i +shunan.yamaguchi.jp=i +net.lb=i +net.la=i +lib.pa.us=i +net.ky=i +ibigawa.gifu.jp=i +seto.aichi.jp=i +gov.ws=i +net.kn=i +xn--mosjen-eya.no=i +kommune.no=i +lakas.hu=i +is-a-caterer.com=p +dyndns-mail.com=p +seat=i +shirakawa.gifu.jp=i +re=i +net.kg=i +xn--ygarden-p1a.no=i +mil.kz=i +net.ki=i +attorney=i +tsukui.kanagawa.jp=i +mil.km=i +no.it=i +nationalheritage.museum=i +k12.ky.us=i +mil.kr=i +ro=i +gratis=i +rs=i +mari.ru=i +mil.kg=i +ru=i +手机=i +folkebibl.no=i +bjarkoy.no=i +rw=i +chuo.fukuoka.jp=i +wodzislaw.pl=i +sc.cn=i +seek=i +sa=i +notteroy.no=i +hirosaki.aomori.jp=i +sb=i +museum.om=i +net.jo=i +nom.pa=i +sc=i +sd=i +se=i +nom.pe=i +sg=i +asahi.mie.jp=i +net.je=i +sh=i +pharmacy.museum=i +si=i +sj=i +sk=i +sl=i +sn.cn=i +honai.ehime.jp=i +sm=i +mil.jo=i +sn=i +from-ia.com=p +so=i +nom.pl=i +gotemba.shizuoka.jp=i +email=i +mihama.chiba.jp=i +sr=i +tahara.aichi.jp=i +net.iq=i +ct.it=i +askvoll.no=i +st=i +net.is=i +su=i +net.ir=i +sv=i +ørskog.no=i +sx=i +sy=i +sz=i +museum.no=i +xn--bck1b9a5dre4c=i +net.im=i +weatherchannel=i +tc=i +net.in=i +ip6.arpa=i +td=i +museum.mv=i +fræna.no=i +tf=i +tg=i +museum.mw=i +th=i +dyndns-wiki.com=p +net.id=i +tj=i +aosta-valley.it=i +tk=i +中國=i +tl=i +tm=i +中国=i +mil.in=i +tn=i +to=i +tp=i +slask.pl=i +tr=i +mil.iq=i +lilly=i +mil.id=i +coastaldefence.museum=i +mamurogawa.yamagata.jp=i +tt=i +tv=i +tkmaxx=i +tw=i +net.py=i +toyono.osaka.jp=i +ookuwa.nagano.jp=i +tz=i +mc.eu.org=p +beppu.oita.jp=i +fuefuki.yamanashi.jp=i +net.pr=i +ua=i +net.pt=i +net.ps=i +chikujo.fukuoka.jp=i +net.ph=i +gov.za=i +ug=i +priv.at=p +yusuhara.kochi.jp=i +政府.hk=i +net.pl=i +net.pk=i +children.museum=i +nom.re=i +uk=i +net.pn=i +yahoo=i +net.pa=i +nom.ro=i +mil.hn=i +arq.br=i +us=i +zhytomyr.ua=i +net.pe=i +salzburg.museum=i +ube.yamaguchi.jp=i +uy=i +clubmed=i +uz=i +botanicalgarden.museum=i +iwate.jp=i +museum.tt=i +fukuroi.shizuoka.jp=i +va=i +vc=i +ve=i +vg=i +asmatart.museum=i +tochigi.jp=i +kyknet=i +mil.gt=i +vi=i +publ.pt=i +net.om=i +schoenbrunn.museum=i +vn=i +naval.museum=i +orskog.no=i +åmli.no=i +vu=i +oarai.ibaraki.jp=i +ap.it=i +hirono.iwate.jp=i +net.nz=i +mil.ge=i +mil.gh=i +net.nr=i +rokunohe.aomori.jp=i +wf=i +xn--jrpeland-54a.no=i +net.nf=i +tsunan.niigata.jp=i +net.ng=i +gal=i +wales.museum=i +gap=i +nom.tm=i +university.museum=i +ws=i +kanonji.kagawa.jp=i +shiwa.iwate.jp=i +bg.it=i +net.mu=i +mx.na=i +net.mt=i +bauern.museum=i +penza.su=i +net.mw=i +med.pl=p +net.mv=i +honjo.saitama.jp=i +net.my=i +net.mx=i +net.ml=i +net.mo=i +can.museum=i +forex=i +med.pa=i +net.ms=i +xn--karmy-yua.no=i +net.me=i +net.mk=i +mil.eg=i +bushey.museum=i +net.ma=i +med.om=i +medio-campidano.it=i +public.museum=i +shell.museum=i +net.dm=i +penza.ru=i +mil.ec=i +net.do=i +rebun.hokkaido.jp=i +from-wv.com=p +chuvashia.ru=i +mil.do=i +cc.nd.us=i +windmill.museum=i +asaminami.hiroshima.jp=i +cloudfront.net=p +humanities.museum=i +net.cu=i +yt=i +hamaroy.no=i +lombardy.it=i +net.cw=i +net.cy=i +net.cm=i +net.co=i +net.cn=i +akishima.tokyo.jp=i +halsa.no=i +net.ci=i +mosvik.no=i +mil.cn=i +gdn=i +net.bz=i +mil.co=i +gb.net=p +fedex=i +k12.ok.us=i +net.br=i +ra.it=i +textile.museum=i +nakatsugawa.gifu.jp=i +net.bt=i +net.bs=i +mil.cl=i +ogawa.saitama.jp=i +sumita.iwate.jp=i +gea=i +net.bm=i +bygland.no=i +tingvoll.no=i +net.bo=i +sc.kr=i +net.bb=i +net.ba=i +uchiko.ehime.jp=i +sakai.osaka.jp=i +cosenza.it=i +tp.it=i +mil.by=i +tools=i +net.bh=i +slg.br=i +promo=i +amica=i +hirata.fukushima.jp=i +mil.bo=i +med.ly=i +madrid.museum=i +net.az=i +yakumo.shimane.jp=i +mil.br=i +lv.ua=i +te.it=i +tolga.no=i +xn--hbmer-xqa.no=i +net.ar=i +net.au=i +xn--skierv-uta.no=i +net.ai=i +pub.sa=i +mil.ba=i +datsun=i +toyotsu.fukuoka.jp=i +net.al=i +net.an=i +campidanomedio.it=i +net.ac=i +net.ae=i +net.ag=i +mil.az=i +net.af=i +mil.al=i +help=i +of.no=i +mil.ar=i +xn--uisz3g.jp=i +mil.ac=i +mil.ae=i +busan.kr=i +gbiz=i +xn--rde-ula.no=i +net.ht=i +war.museum=i +naumburg.museum=i +øyer.no=i +nøtterøy.no=i +vlog.br=i +vibovalentia.it=i +lamborghini=i +nom.za=i +xn--bhcavuotna-s4a.no=i +net.hk=i +rl.no=i +net.hn=i +namsskogan.no=i +mimata.miyazaki.jp=i +otago.museum=i +is.eu.org=p +shimokawa.hokkaido.jp=i +net.gy=i +horology.museum=i +xin=i +minamiminowa.nagano.jp=i +joburg=i +net.gn=i +net.gp=i +net.gr=i +kuki.saitama.jp=i +hofu.yamaguchi.jp=i +net.gt=i +net.gg=i +targi.pl=i +brussels=i +olayangroup=i +net.gl=i +lib.nh.us=i +hekinan.aichi.jp=i +dynalias.org=p +suedtirol.it=i +net.ge=i +sandvikcoromant=i +مصر=i +med.ec=i +askim.no=i +med.ee=i +千葉.jp=i +here=i +elblag.pl=i +philadelphiaarea.museum=i +decorativearts.museum=i +pruszkow.pl=i +oizumi.gunma.jp=i +宮城.jp=i +bruxelles.museum=i +kinder=i +marylhurst.museum=i +ichikai.tochigi.jp=i +security=i +med.ht=i +vagan.no=i +rømskog.no=i +ketrzyn.pl=i +yakage.okayama.jp=i +ایران.ir=i +net.et=i +a.prod.fastly.net=p +co.com=p +brønnøy.no=i +ass.km=i +romskog.no=i +net.eg=i +nd.us=i +mulhouse.museum=i +山口.jp=i +қаз=i +net.ec=i +tynset.no=i +podzone.org=p +net.dz=i +olawa.pl=i +xn--finny-yua.no=i +sb.ua=i +rv.ua=i +qpon=i +comsec=i +is-a-lawyer.com=p +med.br=i +is-very-nice.org=p +ivano-frankivsk.ua=i +higashikurume.tokyo.jp=i +uozu.toyama.jp=i +virgin=i +ไทย=i +tohnosho.chiba.jp=i +kofu.yamanashi.jp=i +gle=i +author.aero=i +sm.ua=i +arita.saga.jp=i +ferrari=i +ro.eu.org=p +mar.it=i +evje-og-hornnes.no=i +xn--kranghke-b0a.no=i +ringebu.no=i +akune.kagoshima.jp=i +melhus.no=i +网络.cn=i +gmo=i +forum.hu=i +moriyama.shiga.jp=i +piedmont.it=i +xn--vestvgy-ixa6o.no=i +gmx=i +pr.us=i +sálát.no=i +lib.pr.us=i +montreal.museum=i +net.za=i +brand.se=i +xn--c2br7g=i +chernihiv.ua=i +int.vn=i +k12.az.us=i +home.dyndns.org=p +galsa.no=i +ftpaccess.cc=p +ofunato.iwate.jp=i +福島.jp=i +spreadbetting=i +misugi.mie.jp=i +williamsburg.museum=i +barrell-of-knowledge.info=p +miyazaki.miyazaki.jp=i +int.ve=i +kwp.gov.pl=i +tas.edu.au=i +5.bg=i +goo=i +kr.com=p +tamakawa.fukushima.jp=i +gop=i +yamada.fukuoka.jp=i +got=i +rsvp=i +yorkshire.museum=i +gov=i +crew.aero=i +dgca.aero=i +omaha.museum=i +个人.hk=i +السعوديه=i +nishiawakura.okayama.jp=i +barrel-of-knowledge.info=p +eidskog.no=i +shiraoka.saitama.jp=i +xn--snase-nra.no=i +组织机构=i +taira.toyama.jp=i +ebetsu.hokkaido.jp=i +web.za=i +chernovtsy.ua=i +money.museum=i +lib.ks.us=i +ancona.it=i +samsung=i +sukagawa.fukushima.jp=i +a.bg=i +saroma.hokkaido.jp=i +from-va.com=p +návuotna.no=i +sande.more-og-romsdal.no=i +shingu.hyogo.jp=i +sexy=i +rzeszow.pl=i +is-a-student.com=p +mobily=i +contemporary.museum=i +com.ve=i +net.ua=i +com.vc=i +channel=i +net.tt=i +openair.museum=i +misato.miyagi.jp=i +com.uz=i +net.tw=i +com.uy=i +andriabarlettatrani.it=i +net.tn=i +net.tm=i +int.pt=i +net.to=i +moseushi.hokkaido.jp=i +kisofukushima.nagano.jp=i +net.tr=i +kumenan.okayama.jp=i +toya.hokkaido.jp=i +trentino-a-adige.it=i +net.th=i +meldal.no=i +urawa.saitama.jp=i +net.tj=i +com.ug=i +tempio-olbia.it=i +xn--ryken-vua.no=i +com.ua=i +batsfjord.no=i +isahaya.nagasaki.jp=i +w.bg=i +xn--mlatvuopmi-s4a.no=i +net.st=i +com.tw=i +net.sy=i +for-better.biz=p +net.sl=i +cal.it=i +com.tt=i +net.so=i +com.tr=i +l.bg=i +com.to=i +saga.saga.jp=i +net.sc=i +com.tm=i +net.sb=i +com.tn=i +yaotsu.gifu.jp=i +net.sd=i +net.sg=i +com.tj=i +net.sh=i +living.museum=i +guernsey.museum=i +dyndns-at-work.com=p +net.sa=i +from-sd.com=p +net.ru=i +السعودية=i +net.rw=i +deatnu.no=i +com.ws=i +endofinternet.net=p +salangen.no=i +kikonai.hokkaido.jp=i +tiaa=i +tonami.toyama.jp=i +tula.ru=i +lans.museum=i +osterøy.no=i +soo.kagoshima.jp=i +com.vu=i +agematsu.nagano.jp=i +kaufen=i +net.qa=i +int.mv=i +int.mw=i +xxx=i +malbork.pl=i +com.vn=i +tokushima.jp=i +com.vi=i +tula.su=i +lib.ma.us=i +kira.aichi.jp=i +tadaoka.osaka.jp=i +sex.hu=i +xyz=i +nasushiobara.tochigi.jp=i +nååmesjevuemie.no=i +marugame.kagawa.jp=i +徳島.jp=i +id.us=i +dell-ogliastra.it=i +int.tj=i +eastafrica.museum=i +matsubushi.saitama.jp=i +ooshika.nagano.jp=i +q-a.eu.org=p +taketomi.okinawa.jp=i +int.tt=i +army=i +xn--wgbh1c=i +eiheiji.fukui.jp=i +xn--y9a3aq=i +net.ws=i +aizumi.tokushima.jp=i +takaishi.osaka.jp=i +pinb.gov.pl=i +sandnes.no=i +asso.eu.org=p +ohkura.yamagata.jp=i +umaji.kochi.jp=i +shikama.miyagi.jp=i +net.vu=i +arpa=i +adult.ht=i +net.vn=i +saku.nagano.jp=i +int.ru=i +sondre-land.no=i +int.rw=i +iglesias-carbonia.it=i +goto.nagasaki.jp=i +land=i +net.ve=i +azumino.nagano.jp=i +lib.nc.us=i +lib.ny.us=i +naturalhistorymuseum.museum=i +net.vi=i +net.vc=i +loabát.no=i +daigo.ibaraki.jp=i +bielawa.pl=i +frosinone.it=i +net.uy=i +dyndns.biz=p +divtasvuodna.no=i +haibara.shizuoka.jp=i +net.uz=i +abruzzo.it=i +gobo.wakayama.jp=i +culturalcenter.museum=i +frei.no=i +øksnes.no=i +net.uk=i +vågan.no=i +fujikawaguchiko.yamanashi.jp=i +com.mx=i +com.my=i +walmart=i +com.mv=i +京都.jp=i +com.mw=i +com.mt=i +com.mu=i +biz.cy=i +com.ms=i +com.mo=i +com.ml=i +com.mk=i +com.mg=i +sciencecenters.museum=i +xn--ryrvik-bya.no=i +kesennuma.miyagi.jp=i +events=i +sx.cn=i +com.ly=i +rennesøy.no=i +farsund.no=i +matsue.shimane.jp=i +com.lv=i +xn--fzys8d69uvgm=i +it.ao=i +higashikagura.hokkaido.jp=i +com.lr=i +com.lk=i +wang=i +báhccavuotna.no=i +fc.it=i +com.lc=i +kujukuri.chiba.jp=i +com.la=i +circle=i +com.lb=i +bialystok.pl=i +com.pa=i +is-a-cpa.com=p +xn--hery-ira.nordland.no=i +czeladz.pl=i +chase=i +arte=i +nishimera.miyazaki.jp=i +juegos=i +biz.et=i +xn--rovu88b=i +coop.br=i +kuwana.mie.jp=i +com.om=i +kashihara.nara.jp=i +wroclaw.pl=i +xn--hmmrfeasta-s4ac.no=i +minowa.nagano.jp=i +gouv.ci=i +greta.fr=i +is-a-geek.com=p +yawata.kyoto.jp=i +kitashiobara.fukushima.jp=i +is-an-actor.com=p +com.nr=i +landes.museum=i +sakae.chiba.jp=i +aarp=i +napoli.it=i +com.ng=i +community.museum=i +arai.shizuoka.jp=i +com.nf=i +sandiego.museum=i +com.na=i +gouv.bj=i +otake.hiroshima.jp=i +pescara.it=i +kagamino.okayama.jp=i +mobi.gp=i +xn--5rtp49c.jp=i +voting=i +misato.saitama.jp=i +bahccavuotna.no=i +asahi.yamagata.jp=i +nishiwaki.hyogo.jp=i +tromso.no=i +school=i +kasai.hyogo.jp=i +xn--mgberp4a5d4ar=i +fukudomi.saga.jp=i +com.qa=i +ethnology.museum=i +meloy.no=i +com.py=i +xn--slat-5na.no=i +z-1.compute-1.amazonaws.com=p +tsuru.yamanashi.jp=i +kh.ua=i +com.ps=i +com.pt=i +com.pr=i +hbo=i +kamikawa.saitama.jp=i +tgory.pl=i +com.pk=i +com.pl=i +com.ph=i +com.pe=i +nfshost.com=p +tagawa.fukuoka.jp=i +izhevsk.ru=i +com.pf=i +net.eu.org=p +cs.it=i +voto=i +okawa.kochi.jp=i +xn--vads-jra.no=i +group.aero=i +pr.it=i +ks.ua=i +inuyama.aichi.jp=i +video=i +com.sy=i +com.sv=i +bus.museum=i +com.st=i +takaharu.miyazaki.jp=i +a.se=i +公益=i +com.sn=i +com.so=i +com.sl=i +k12.nh.us=i +pg.it=i +com.sh=i +vote=i +com.sg=i +com.sd=i +com.se=p +com.sb=i +com.sc=i +xn--6frz82g=i +okoppe.hokkaido.jp=i +com.sa=i +inami.wakayama.jp=i +biz.id=i +olecko.pl=i +com.rw=i +yashiro.hyogo.jp=i +engineer=i +com.ru=i +calvinklein=i +com.ro=i +ch.it=i +xn--mgbab2bd=i +vestvågøy.no=i +ks.us=i +buzz=i +com.re=i +sakata.yamagata.jp=i +lowicz.pl=i +com.et=i +per.la=i +com.es=i +shimokitayama.nara.jp=i +komatsu=i +nishikawa.yamagata.jp=i +tips=i +com.eg=i +karmoy.no=i +com.ee=i +ma.us=i +com.ec=i +flekkefjord.no=i +bozen.it=i +aurskog-høland.no=i +com.dz=i +w.se=i +gouv.ht=i +l.se=i +com.do=i +com.dm=i +kragero.no=i +coop.mv=i +coop.mw=i +osakasayama.osaka.jp=i +firebaseapp.com=p +england.museum=i +com.de=p +webcam=i +from-de.com=p +xn--kprw13d=i +is-a-bookkeeper.com=p +xn--sknit-yqa.no=i +com.cy=i +yn.cn=i +com.cw=i +com.gy=i +com.gt=i +lugansk.ua=i +com.gr=i +mátta-várjjat.no=i +com.gp=i +com.gn=i +cc.ar.us=i +com.gl=i +nc.us=i +smart=i +muika.niigata.jp=i +vs.it=i +oryol.ru=i +com.gh=i +com.gi=i +frogans=i +corsica=i +com.ge=i +gouv.km=i +xn--srfold-bya.no=i +commbank=i +for-our.info=p +hamburg=i +coop.km=i +ao.it=i +yamamoto.miyagi.jp=i +from-co.net=p +xn--stjrdalshalsen-sqb.no=i +is-very-good.org=p +com.fr=i +kawakami.nagano.jp=i +nc.tr=i +trentinoa-adige.it=i +ambulance.aero=i +tennis=i +higashiosaka.osaka.jp=i +gushikami.okinawa.jp=i +yazu.tottori.jp=i +ad.jp=i +chat=i +sci.eg=i +garden=i +from-oh.com=p +cc.ut.us=i +from-tn.com=p +aejrie.no=i +com.is=i +koshu.yamanashi.jp=i +kayabe.hokkaido.jp=i +com.iq=i +spjelkavik.no=i +com.io=i +oldnavy=i +praxi=i +com.im=i +bo.telemark.no=i +coop.ht=i +xn--klty5x.jp=i +in-the-band.net=p +snz.ru=i +honjyo.akita.jp=i +hiv=i +higashishirakawa.gifu.jp=i +k12.pa.us=i +maniwa.okayama.jp=i +lib.az.us=i +kusatsu.gunma.jp=i +medical.museum=i +北海道.jp=i +minakami.gunma.jp=i +com.ht=i +com.hr=i +xn--rht3d.jp=i +ichinohe.iwate.jp=i +com.hn=i +tsushima.aichi.jp=i +com.hk=i +wakkanai.hokkaido.jp=i +pokrovsk.su=i +xn--8y0a063a=i +condos=i +art.br=i +xn--fiq64b=i +selbu.no=i +biz.bb=i +com.kz=i +sdn.gov.pl=i +ginowan.okinawa.jp=i +bio.br=i +com.ky=i +rightathome=i +xn--lhppi-xqa.no=i +from-mn.com=p +com.kp=i +biz.az=i +hkt=i +com.km=i +biz.at=p +com.ki=i +com.kg=i +afamilycompany=i +xn--mgbai9azgqp6j=i +nichinan.miyazaki.jp=i +mishima.fukushima.jp=i +長野.jp=i +auto.pl=i +akkeshi.hokkaido.jp=i +gouv.fr=i +ny.us=i +nowaruda.pl=i +to.it=i +com.jo=i +xn--mlselv-iua.no=i +xn--fiqz9s=i +xn--54b7fta0cc=i +ashiya.fukuoka.jp=i +ebino.miyazaki.jp=i +art.do=i +kochi.kochi.jp=i +新加坡=i +k12.ks.us=i +asda=i +vik.no=i +aostavalley.it=i +coop.tt=i +finearts.museum=i +bashkiria.su=i +gs.hl.no=i +tushu=i +lutsk.ua=i +tas.gov.au=i +xn--cg4bki=i +asso.re=i +id.au=i +virtuel.museum=i +glogow.pl=i +trentino-alto-adige.it=i +dental=i +coupons=i +beer=i +you=i +mobi.tt=i +ål.no=i +olkusz.pl=i +bashkiria.ru=i +xn--kvnangen-k0a.no=i +leasing.aero=i +active=i +fukushima.hokkaido.jp=i +cn.it=i +from-id.com=p +kids.museum=i +shisui.chiba.jp=i +mobi.tz=i +deals=i +xn--bievt-0qa.no=i +larvik.no=i +hb.cn=i +ohtawara.tochigi.jp=i +kitagata.saga.jp=i +is-a-guru.com=p +rishiri.hokkaido.jp=i +xn--xkc2al3hye2a=i +yukuhashi.fukuoka.jp=i +is-a-blogger.com=p +how=i +kyotanabe.kyoto.jp=i +hikawa.shimane.jp=i +sakuragawa.ibaraki.jp=i +does-it.net=p +is-an-anarchist.com=p +组織.hk=i +volgograd.ru=i +gv.ao=i +gv.at=i +cyber.museum=i +meet=i +stor-elvdal.no=i +tagajo.miyagi.jp=i +sa.gov.pl=i +inc.hk=p +com.ar=i +com.an=i +theater.museum=i +com.al=i +daito.osaka.jp=i +coop.py=i +com.ai=i +com.af=i +kråanghke.no=i +com.ag=i +k12.ny.us=i +com.ac=i +porsanger.no=i +oji.nara.jp=i +koeln.museum=i +endofinternet.org=p +reggio-calabria.it=i +xn--w4rs40l=i +comunicações.museum=i +ba.it=i +minokamo.gifu.jp=i +sciencehistory.museum=i +higashi.okinawa.jp=i +vision=i +lavangen.no=i +k12.nc.us=i +fi.cr=i +suwalki.pl=i +mup.gov.pl=i +k12.ma.us=i +com.cu=i +com.cn=i +snaase.no=i +com.co=i +com.cm=i +com.ci=i +kyotamba.kyoto.jp=i +yun=i +cc.fl.us=i +judygarland.museum=i +htc=i +stargard.pl=i +exposed=i +bl.it=i +com.by=i +com.bz=i +com.bs=i +com.bt=i +com.br=i +com.bo=i +komagane.nagano.jp=i +com.bm=i +asia=i +com.bi=i +com.bh=i +shimoda.shizuoka.jp=i +ushiku.ibaraki.jp=i +jus.br=i +com.ba=i +cyou=i +com.bb=i +com.az=i +sande.vestfold.no=i +omaezaki.shizuoka.jp=i +xn--tckwe=i +com.aw=i +xn--frde-gra.no=i +com.au=i +haram.no=i +meme=i +brasil.museum=i +sande.møre-og-romsdal.no=i +kmpsp.gov.pl=i +ibara.okayama.jp=i +xn--hgebostad-g3a.no=i +yugawara.kanagawa.jp=i +aioi.hyogo.jp=i +москва=i +emergency.aero=i +steinkjer.no=i +catering.aero=i +jor.br=i +yamashina.kyoto.jp=i +heguri.nara.jp=i +hiphop=i +tienda=i +xn--od0alg.cn=i +ogori.fukuoka.jp=i +itano.tokushima.jp=i +bearalvahki.no=i +otoyo.kochi.jp=i +kembuchi.hokkaido.jp=i +higashiomi.shiga.jp=i +okuma.fukushima.jp=i +ontario.museum=i +jaguar=i +ozora.hokkaido.jp=i +is-a-democrat.com=p +sandcats.io=p +from-pr.com=p +snoasa.no=i +政务=i +yolasite.com=p +kalmykia.ru=i +montblanc=i +mobi.ng=i +xn--correios-e-telecomunicaes-ghc29a.museum=i +toray=i +hanno.saitama.jp=i +from.hr=i +mobi.na=i +frogn.no=i +kalmykia.su=i +bando.ibaraki.jp=i +menu=i +able=i +from-vt.com=p +vegårshei.no=i +sarufutsu.hokkaido.jp=i +shintomi.miyazaki.jp=i +asahi.toyama.jp=i +codes=i +vologda.ru=i +midatlantic.museum=i +chijiwa.nagasaki.jp=i +minami.tokushima.jp=i +kpmg=i +seika.kyoto.jp=i +odesa.ua=i +lu.it=i +international=i +vologda.su=i +strand.no=i +compute.amazonaws.cn=p +xn--od0alg.hk=i +dentist=i +ryuoh.shiga.jp=i +muroran.hokkaido.jp=i +tara.saga.jp=i +հայ=i +spb.ru=i +weber=i +aoki.nagano.jp=i +games.hu=i +nexus=i +tos.it=i +xn--xhq521b=i +best=i +rodoy.no=i +kumatori.osaka.jp=i +ostrowwlkp.pl=i +xn--hyanger-q1a.no=i +ise.mie.jp=i +piaget=i +is-slick.com=p +spb.su=i +k12.pr.us=i +matsushige.tokushima.jp=i +iitate.fukushima.jp=i +pubol.museum=i +togitsu.nagasaki.jp=i +miyakonojo.miyazaki.jp=i +kanuma.tochigi.jp=i +pl.eu.org=p +福井.jp=i +xn--io0a7i=i +kurate.fukuoka.jp=i +ogawa.ibaraki.jp=i +qld.edu.au=i +dellogliastra.it=i +kisarazu.chiba.jp=i +xn--ntso0iqx3a.jp=i +røyken.no=i +lib.de.us=i +county.museum=i +blockbuster=i +gets-it.net=p +theatre=i +mitane.akita.jp=i +ru.eu.org=p +lund.no=i +name.tj=i +tranøy.no=i +oyodo.nara.jp=i +whaling.museum=i +shiroishi.miyagi.jp=i +bryansk.su=i +xn--lgbbat1ad8j=i +asso.bj=i +siracusa.it=i +shibetsu.hokkaido.jp=i +cc.oh.us=i +fujiyoshida.yamanashi.jp=i +nakayama.yamagata.jp=i +onomichi.hiroshima.jp=i +szkola.pl=i +owani.aomori.jp=i +bryansk.ru=i +torino.museum=i +xn--mgba3a4fra.ir=i +asso.ci=i +cartoonart.museum=i +高知.jp=i +setouchi.okayama.jp=i +ninomiya.kanagawa.jp=i +bill.museum=i +settsu.osaka.jp=i +granvin.no=i +ru.com=p +name.vn=i +toyooka.hyogo.jp=i +gob.sv=i +nakhodka.ru=i +name.tt=i +paragliding.aero=i +barclays=i +takino.hyogo.jp=i +name.tr=i +sugito.saitama.jp=i +berlevåg.no=i +kani.gifu.jp=i +tank.museum=i +hvaler.no=i +hembygdsforbund.museum=i +trolley.museum=i +ibm=i +aogashima.tokyo.jp=i +alsace=i +ath.cx=p +onga.fukuoka.jp=i +asso.dz=i +guge=i +yoshimi.saitama.jp=i +de.us=i +chikuho.fukuoka.jp=i +sex.pl=i +arida.wakayama.jp=i +ice=i +chungbuk.kr=i +コム=i +cc.md.us=i +vda.it=i +cc.mo.us=i +mypets.ws=p +dp.ua=i +kawaminami.miyazaki.jp=i +icu=i +tiffany=i +york.museum=i +magazine.aero=i +vc.it=i +hm.no=i +mutuelle=i +kamimine.saga.jp=i +nkz.ru=i +vegarshei.no=i +plumbing=i +aizumisato.fukushima.jp=i +asso.fr=i +gob.ve=i +一号店=i +kakegawa.shizuoka.jp=i +id.ly=i +ruhr=i +sakaiminato.tottori.jp=i +maritime.museum=i +christiansburg.museum=i +ven.it=i +asso.gp=i +id.lv=i +обр.срб=i +imageandsound.museum=i +judaica.museum=i +koeln=i +العليان=i +q.bg=i +haebaru.okinawa.jp=i +likescandy.com=p +f.bg=i +lyngdal.no=i +oppegård.no=i +name.pr=i +grozny.ru=i +jinsekikogen.hiroshima.jp=i +usuki.oita.jp=i +asso.ht=i +aure.no=i +name.qa=i +ifm=i +isumi.chiba.jp=i +telecity=i +fage=i +yamagata.nagano.jp=i +space.museum=i +oppdal.no=i +be.eu.org=p +resistance.museum=i +oregon.museum=i +ami.ibaraki.jp=i +snåase.no=i +kinko.kagoshima.jp=i +aetna=i +moka.tochigi.jp=i +ono.hyogo.jp=i +konsulat.gov.pl=i +riodejaneiro.museum=i +safety=i +iz.hr=i +grozny.su=i +mari-el.ru=i +archaeology.museum=i +hámmárfeasta.no=i +zip=i +surgeonshall.museum=i +sør-aurdal.no=i +bunkyo.tokyo.jp=i +嘉里大酒店=i +yoshioka.gunma.jp=i +educator.aero=i +ishikawa.jp=i +id.ir=i +corvette.museum=i +forsale=i +fail=i +xn--gls-elac.no=i +vn.ua=i +divttasvuotna.no=i +amusement.aero=i +nakano.nagano.jp=i +asso.km=i +tono.iwate.jp=i +cheltenham.museum=i +church=i +higashiizumo.shimane.jp=i +viking.museum=i +cn.ua=i +pictures=i +towada.aomori.jp=i +sodegaura.chiba.jp=i +nico=i +blogsite.org=p +chiyoda.gunma.jp=i +tomika.gifu.jp=i +ina.nagano.jp=i +lv.eu.org=p +xn--koluokta-7ya57h.no=i +kiev.ua=i +noshiro.akita.jp=i +steiermark.museum=i +kawara.fukuoka.jp=i +xn--s-1fa.no=i +yokoshibahikari.chiba.jp=i +asso.mc=i +pohl=i +childrens.museum=i +lib.id.us=i +settlers.museum=i +三重.jp=i +guru=i +kawazu.shizuoka.jp=i +saga.jp=i +makinohara.shizuoka.jp=i +murmansk.ru=i +meiwa.mie.jp=i +rsc.cdn77.org=p +yahaba.iwate.jp=i +asso.nc=i +xn--p1acf=i +teo.br=i +امارات=i +theater=i +saiki.oita.jp=i +ham-radio-op.net=p +cc.na=i +s3-eu-central-1.amazonaws.com=p +isa-geek.org=p +murmansk.su=i +from-nj.com=p +takashima.shiga.jp=i +sa.gov.au=i +ibaraki.osaka.jp=i +bryne.no=i +repair=i +xn--mkru45i.jp=i +citic=i +yamato.kumamoto.jp=i +schlesisches.museum=i +食品=i +inatsuki.fukuoka.jp=i +val-daosta.it=i +computer.museum=i +shimane.shimane.jp=i +yusui.kagoshima.jp=i +ing=i +ink=i +coloradoplateau.museum=i +ogi.saga.jp=i +lesja.no=i +trainer.aero=i +shibata.niigata.jp=i +삼성=i +int=i +网络.hk=i +fans=i +homelinux.org=p +sakhalin.ru=i +hgtv=i +ichikawamisato.yamanashi.jp=i +ayabe.kyoto.jp=i +shitara.aichi.jp=i +xn--mtta-vrjjat-k7af.no=i +newhampshire.museum=i +fi.it=i +xn--zfr164b=i +移动=i +bv.nl=i +is-a-libertarian.com=p +saarland=i +leka.no=i +tokuyama.yamaguchi.jp=i +simple-url.com=p +statoil=i +midsund.no=i +ollo=i +lib.ct.us=i +farm=i +aarborte.no=i +yamanakako.yamanashi.jp=i +motorcycles=i +nanae.hokkaido.jp=i +toyotomi.hokkaido.jp=i +okagaki.fukuoka.jp=i +ind.gt=i +ltda=i +namegawa.saitama.jp=i +ørsta.no=i +kita.osaka.jp=i +nose.osaka.jp=i +porn=i +kazan.ru=i +mining.museum=i +winners=i +fuettertdasnetz.de=p +de.eu.org=p +nike=i +ha.cn=i +okaya.nagano.jp=i +xn--fjq720a=i +fuchu.hiroshima.jp=i +fast=i +kustanai.ru=i +hl.cn=i +台湾=i +ist=i +it.eu.org=p +post=i +hitachinaka.ibaraki.jp=i +网絡.hk=i +kami.miyagi.jp=i +yuzhno-sakhalinsk.ru=i +kiyokawa.kanagawa.jp=i +kids.us=i +rauma.no=i +leirfjord.no=i +higashiagatsuma.gunma.jp=i +sevastopol.ua=i +kasugai.aichi.jp=i +itv=i +dolls.museum=i +fujimino.saitama.jp=i +certification.aero=i +mutsuzawa.chiba.jp=i +shell=i +kunisaki.oita.jp=i +pharmacy=i +nm.cn=i +cb.it=i +funahashi.toyama.jp=i +sandøy.no=i +eastcoast.museum=i +directory=i +kvanangen.no=i +集团=i +iide.yamagata.jp=i +yuzawa.niigata.jp=i +bale.museum=i +kherson.ua=i +k12.nm.us=i +nb.ca=i +suginami.tokyo.jp=i +宮崎.jp=i +bronnoysund.no=i +siljan.no=i +shima.mie.jp=i +avellino.it=i +yabuki.fukushima.jp=i +iwc=i +f.se=i +lundbeck=i +iwama.ibaraki.jp=i +dyndns-at-home.com=p +airforce=i +wallonie.museum=i +ind.in=i +siena.it=i +college=i +xn--bjarky-fya.no=i +elb.amazonaws.com=p +kr.it=i +hasami.nagasaki.jp=i +wassamu.hokkaido.jp=i +cargo.aero=i +toyako.hokkaido.jp=i +satosho.okayama.jp=i +cc.al.us=i +sandefjord.no=i +property=i +li.it=i +botany.museum=i +kg.kr=i +embetsu.hokkaido.jp=i +ae.org=p +americana.museum=i +nikko.tochigi.jp=i +vibo-valentia.it=i +futsu.nagasaki.jp=i +sec.ps=i +hirogawa.wakayama.jp=i +chikuma.nagano.jp=i +gaivuotna.no=i +capebreton.museum=i +klabu.no=i +oz.au=i +city=i +moroyama.saitama.jp=i +nx.cn=i +rana.no=i +at.it=i +rimini.it=i +healthcare=i +k12.tn.us=i +sa.au=i +hikimi.shimane.jp=i +xn--7t0a264c.jp=i +wanggou=i +mex.com=p +cadaques.museum=i +shimane.jp=i +fin.ec=i +press.museum=i +kawaguchi.saitama.jp=i +愛知.jp=i +yodobashi=i +coal.museum=i +costume.museum=i +xn--muost-0qa.no=i +citi=i +lt.it=i +inzai.chiba.jp=i +toride.ibaraki.jp=i +shaw=i +is-a-knight.org=p +gildeskal.no=i +database.museum=i +ptz.ru=i +matsumae.hokkaido.jp=i +chocolate.museum=i +tajiri.osaka.jp=i +oharu.aichi.jp=i +ikeda.gifu.jp=i +shiroishi.saga.jp=i +mombetsu.hokkaido.jp=i +read=i +takayama.gifu.jp=i +xn--ostery-fya.no=i +xn--jlster-bya.no=i +parma.it=i +versicherung=i +xn--uuwu58a.jp=i +midori.chiba.jp=i +trentinoalto-adige.it=i +is-a-teacher.com=p +muko.kyoto.jp=i +nakanojo.gunma.jp=i +unzen.nagasaki.jp=i +parliament.nz=i +is-a-conservative.com=p +mizusawa.iwate.jp=i +qh.cn=i +shiksha=i +iki.fi=p +from-ok.com=p +kamioka.akita.jp=i +shirakawa.fukushima.jp=i +ind.br=i +zgrad.ru=i +today=i +construction=i +imb.br=i +frankfurt.museum=i +høyanger.no=i +tsk.ru=i +vermögensberatung=i +xn--node=i +hå.no=i +sera.hiroshima.jp=i +nannestad.no=i +kagawa.jp=i +xn--srum-gra.no=i +qc.com=p +motobu.okinawa.jp=i +global.ssl.fastly.net=p +statebank=i +from-ms.com=p +francaise.museum=i +gratangen.no=i +mizunami.gifu.jp=i +samukawa.kanagawa.jp=i +est-a-la-maison.com=p +detroit.museum=i +media.pl=i +nowruz=i +sayama.saitama.jp=i +itami.hyogo.jp=i +toyota.aichi.jp=i +in.ua=i +sunagawa.hokkaido.jp=i +shia=i +minnesota.museum=i +ap.gov.pl=i +johana.toyama.jp=i +rollag.no=i +nishigo.fukushima.jp=i +in.th=i +british.museum=i +usantiques.museum=i +toyohashi.aichi.jp=i +showa.yamanashi.jp=i +tamano.okayama.jp=i +withgoogle.com=p +plantation.museum=i +krodsherad.no=i +ono.fukui.jp=i +gliding.aero=i +photography=i +გე=i +ama.shimane.jp=i +xn--mix082f=i +shimada.shizuoka.jp=i +pippu.hokkaido.jp=i +jcb=i +nakai.kanagawa.jp=i +xn--p1ai=i +xn--2m4a15e.jp=i +in.rs=i +xn--rdy-0nab.no=i +jcp=i +联通=i +vega.no=i +iraq.museum=i +friuli-v-giulia.it=i +lenug.su=i +unjarga.no=i +itako.ibaraki.jp=i +stuff-4-sale.org=p +xn--sknland-fxa.no=i +undersea.museum=i +broker=i +tarui.gifu.jp=i +lib.ok.us=i +nachikatsuura.wakayama.jp=i +aomori.jp=i +sigdal.no=i +台灣=i +xn--vgsy-qoa0j.no=i +chikuzen.fukuoka.jp=i +hashimoto.wakayama.jp=i +k12.wi.us=i +gu.us=i +lib.ky.us=i +macys=i +hoyanger.no=i +nanyo.yamagata.jp=i +bizen.okayama.jp=i +pacific.museum=i +kyiv.ua=i +beiarn.no=i +realtor=i +cc.or.us=i +plo.ps=i +us-west-2.compute.amazonaws.com=p +otsuka=i +watches=i +courses=i +クラウド=i +is-not-certified.com=p +kumano.hiroshima.jp=i +reit=i +northwesternmutual=i +indianmarket.museum=i +dnipropetrovsk.ua=i +gripe=i +utsunomiya.tochigi.jp=i +elvendrell.museum=i +hidaka.wakayama.jp=i +budapest=i +novara.it=i +sortland.no=i +kopervik.no=i +show=i +savona.it=i +pizza=i +aisho.shiga.jp=i +rns.tn=i +takamori.nagano.jp=i +somna.no=i +esp.br=i +dyrøy.no=i +takinoue.hokkaido.jp=i +norton=i +al.eu.org=p +in.na=i +sálat.no=i +civilwar.museum=i +xn--dyry-ira.no=i +ro.com=p +masoy.no=i +gyokuto.kumamoto.jp=i +hiranai.aomori.jp=i +land-4-sale.us=p +buyshouses.net=p +wielun.pl=i +健康=i +pisz.pl=i +sopot.pl=p +cc.ne.us=i +hl.no=i +assn.lk=i +chita.aichi.jp=i +mod.gi=i +yosemite.museum=i +xn--rmskog-bya.no=i +rent=i +herokussl.com=p +higashiyoshino.nara.jp=i +broadcast.museum=i +moriguchi.osaka.jp=i +jio=i +xn--tjme-hra.no=i +shinjo.yamagata.jp=i +cc.la.us=i +gent=i +andøy.no=i +kvitsøy.no=i +trentinoaltoadige.it=i +durban=i +engine.aero=i +namikata.ehime.jp=i +valle-d-aosta.it=i +ha.no=i +chrysler=i +asnes.no=i +fh.se=i +xn--yfro4i67o=i +cc.mn.us=i +kursk.ru=i +fin.tn=i +style=i +astronomy.museum=i +notodden.no=i +berg.no=i +værøy.no=i +epost=i +nishiizu.shizuoka.jp=i +monticello.museum=i +kåfjord.no=i +xn--qxam=i +xn--jlq61u9w7b=i +jlc=i +xn--risr-ira.no=i +other.nf=i +assedic.fr=i +jll=i +yoshida.saitama.jp=i +koka.shiga.jp=i +higashikawa.hokkaido.jp=i +trapani.it=i +kasumigaura.ibaraki.jp=i +ind.tn=i +vadsø.no=i +xn--lten-gra.no=i +honefoss.no=i +starhub=i +xn--42c2d9a=i +is-a-chef.com=p +xn--ngbe9e0a=i +ushistory.museum=i +jmp=i +cloud=i +stranda.no=i +audi=i +isteingeek.de=p +here-for-more.info=p +isa.kagoshima.jp=i +za.com=p +jnj=i +adac=i +nosegawa.nara.jp=i +nishiarita.saga.jp=i +aizuwakamatsu.fukushima.jp=i +bájddar.no=i +lodi.it=i +syzran.ru=i +gs.sf.no=i +rifu.miyagi.jp=i +xn--aroport-bya.ci=i +soja.okayama.jp=i +tatsuno.nagano.jp=i +hughes=i +pulawy.pl=i +jot=i +xn--aurskog-hland-jnb.no=i +for-some.biz=p +tsuruoka.yamagata.jp=i +sakai.ibaraki.jp=i +joy=i +control.aero=i +rest=i +zhitomir.ua=i +hidaka.hokkaido.jp=i +tromsø.no=i +yasugi.shimane.jp=i +kakogawa.hyogo.jp=i +xn--mgbqly7cvafr=i +yandex=i +from-ga.com=p +flights=i +jefferson.museum=i +softbank=i +gub.uy=i +is-very-bad.org=p +公司.cn=i +tomobe.ibaraki.jp=i +rankoshi.hokkaido.jp=i +nesodden.no=i +rovigo.it=i +ارامكو=i +tobishima.aichi.jp=i +hk.org=p +juedisches.museum=i +info.vn=i +canon=i +fusa.no=i +semboku.akita.jp=i +aoste.it=i +chita.ru=i +vet.br=i +dagestan.ru=i +info.ve=i +pa.it=i +shimizu.shizuoka.jp=i +和歌山.jp=i +recipes=i +stuff-4-sale.us=p +澳門=i +onjuku.chiba.jp=i +ivanovo.ru=i +4.bg=i +per.sg=i +ishikawa.okinawa.jp=i +mishima.shizuoka.jp=i +vladikavkaz.su=i +kotohira.kagawa.jp=i +eun.eg=i +xn--mjndalen-64a.no=i +isehara.kanagawa.jp=i +info.tt=i +lib.nd.us=i +info.tz=i +nerima.tokyo.jp=i +yamanashi.jp=i +gojome.akita.jp=i +nara.nara.jp=i +drive=i +portlligat.museum=i +isshiki.aichi.jp=i +ivanovo.su=i +info.tn=i +vladikavkaz.ru=i +hemsedal.no=i +info.tr=i +pa.gov.pl=i +ppg.br=i +xn--1lqs71d.jp=i +gen.tr=i +xn--flw351e=i +xn--1ck2e1b=i +github.io=p +campidano-medio.it=i +skjervoy.no=i +unazuki.toyama.jp=i +info.sd=i +togura.nagano.jp=i +jobs=i +餐厅=i +yachimata.chiba.jp=i +mashiki.kumamoto.jp=i +dubai=i +urbino-pesaro.it=i +lincoln.museum=i +kred=i +chuo.osaka.jp=i +gda.pl=p +xn--fzc2c9e2c=i +murayama.yamagata.jp=i +shijonawate.osaka.jp=i +agano.niigata.jp=i +mosjøen.no=i +jewelry.museum=i +anan.tokushima.jp=i +v.bg=i +km.ua=i +info.ro=i +kiso.nagano.jp=i +kikuchi.kumamoto.jp=i +furano.hokkaido.jp=i +nrw.museum=i +cc.nv.us=i +contact=i +k.bg=i +lajolla.museum=i +jar.ru=i +澳门=i +mesaverde.museum=i +tires=i +xn--o1ach.xn--90a3ac=i +gen.nz=i +info.pr=i +re.it=i +parts=i +pila.pl=i +passenger-association.aero=i +kawahara.tottori.jp=i +consulting.aero=i +party=i +accountants=i +psse.gov.pl=i +sør-fron.no=i +info.pk=i +ainan.ehime.jp=i +per.nf=i +info.pl=i +mckinsey=i +tomisato.chiba.jp=i +fujimi.nagano.jp=i +cc.mt.us=i +goodyear=i +puglia.it=i +cc.mi.us=i +for-more.biz=p +shimoichi.nara.jp=i +dagestan.su=i +info.nr=i +baikal.ru=i +friuliveneziagiulia.it=i +info.na=i +zj.cn=i +vb.it=i +omachi.saga.jp=i +ginan.gifu.jp=i +expert=i +gen.in=i +cri.nz=i +info.nf=i +karumai.iwate.jp=i +taishi.hyogo.jp=i +works.aero=i +is-a-republican.com=p +seljord.no=i +glas.museum=i +higashimurayama.tokyo.jp=i +xn--lt-liac.no=i +info.mv=i +sokndal.no=i +neustar=i +philips=i +teshikaga.hokkaido.jp=i +crown=i +nagaokakyo.kyoto.jp=i +bestbuy=i +tsukumi.oita.jp=i +autos=i +dni.us=i +os.hordaland.no=i +慈善=i +gálsá.no=i +kitamoto.saitama.jp=i +од.срб=i +닷넷=i +and.museum=i +jobs.tt=i +niihama.ehime.jp=i +works=i +info.la=i +hemnes.no=i +higashichichibu.saitama.jp=i +airtraffic.aero=i +world=i +s3-external-1.amazonaws.com=p +guardian=i +uppo.gov.pl=i +ap-southeast-2.compute.amazonaws.com=p +nsn.us=i +sanjo.niigata.jp=i +go.dyndns.org=p +shiiba.miyazaki.jp=i +serveftp.org=p +si.eu.org=p +restaurant=i +xn--mgbc0a9azcg=i +fukushima.jp=i +nordkapp.no=i +nh.us=i +info.ki=i +hdfcbank=i +yokawa.hyogo.jp=i +re.kr=i +report=i +gol.no=i +høylandet.no=i +pilot.aero=i +homesense=i +nakamura.kochi.jp=i +点看=i +forlicesena.it=i +is-into-anime.com=p +haga.tochigi.jp=i +auto=i +coldwar.museum=i +gonohe.aomori.jp=i +download=i +xn--3e0b707e=i +aisai.aichi.jp=i +sr.it=i +sor-fron.no=i +nowtv=i +cancerresearch=i +ogawara.miyagi.jp=i +cloudapp.net=p +ar.com=p +poker=i +vana=i +tirol=i +cc.il.us=i +ltd.co.im=i +组织.hk=i +xn--4it168d.jp=i +cc.ia.us=i +hara.nagano.jp=i +sumida.tokyo.jp=i +inderøy.no=i +info.hu=i +info.ht=i +naka.hiroshima.jp=i +leitungsen.de=p +vefsn.no=i +hirara.okinawa.jp=i +tt.im=i +benevento.it=i +githubusercontent.com=p +xn--mely-ira.no=i +total=i +cam.it=i +figueres.museum=i +shishikui.tokushima.jp=i +khabarovsk.ru=i +ge.it=i +pl.ua=i +babia-gora.pl=i +social=i +is-an-entertainer.com=p +principe.st=i +pa.us=i +xn--rholt-mra.no=i +kagoshima.kagoshima.jp=i +ski.no=i +basilicata.it=i +idv.hk=i +hitachi=i +earth=i +meråker.no=i +mediocampidano.it=i +columbia.museum=i +az.us=i +biratori.hokkaido.jp=i +luxe=i +shibuya.tokyo.jp=i +vlaanderen=i +circus.museum=i +xn--osyro-wua.no=i +iyo.ehime.jp=i +azure-mobile.net=p +pilots.museum=i +bjugn.no=i +dyndns-ip.com=p +frana.no=i +lipsy=i +ragusa.it=i +info.et=i +genkai.saga.jp=i +parliament.cy=i +alt.za=i +xn--eckvdtc9d=i +shimamaki.hokkaido.jp=i +rexroth=i +info.ec=i +amot.no=i +cagliari.it=i +kaminoyama.yamagata.jp=i +chirurgiens-dentistes.fr=i +ichiba.tokushima.jp=i +sumoto.hyogo.jp=i +stv.ru=i +みんな=i +asahi.chiba.jp=i +misaki.osaka.jp=i +samara.ru=i +stjordalshalsen.no=i +kaluga.ru=i +audio=i +info.co=i +silk=i +show.aero=i +mw.gov.pl=i +amber.museum=i +ירושלים.museum=i +omasvuotna.no=i +baidar.no=i +sasaguri.fukuoka.jp=i +bytom.pl=i +sauherad.no=i +kawanishi.nara.jp=i +kfh=i +is-into-cartoons.com=p +mobara.chiba.jp=i +cupcake.is=p +idv.tw=i +info.bb=i +grong.no=i +cricket=i +shibukawa.gunma.jp=i +utazas.hu=i +akita.akita.jp=i +usa.museum=i +kaluga.su=i +kolobrzeg.pl=i +hannan.osaka.jp=i +info.at=p +k.se=i +xn--1lqs03n.jp=i +paris.museum=i +info.au=i +info.az=i +trentinosudtirol.it=i +quest=i +oum.gov.pl=i +karasjohka.no=i +kitayama.wakayama.jp=i +paleo.museum=i +joyo.kyoto.jp=i +lib.vi.us=i +athleta=i +ee.eu.org=p +narvik.no=i +int.la=i +udono.mie.jp=i +degree=i +zp.ua=i +int.lk=i +daegu.kr=i +sor-varanger.no=i +lib.vt.us=i +sina=i +kia=i +sumoto.kumamoto.jp=i +swatch=i +xn--ses554g=i +caltanissetta.it=i +kim=i +walbrzych.pl=i +sherbrooke.museum=i +shintoku.hokkaido.jp=i +nord-fron.no=i +shibecha.hokkaido.jp=i +hu.com=p +barletta-trani-andria.it=i +kraanghke.no=i +technology=i +crafts.museum=i +mihara.kochi.jp=i +australia.museum=i +k12.gu.us=i +notaires.fr=i +int.is=i +hareid.no=i +engineer.aero=i +ryokami.saitama.jp=i +nanporo.hokkaido.jp=i +akashi.hyogo.jp=i +lib.sc.us=i +feedback=i +canada.museum=i +swinoujscie.pl=i +biella.it=i +opoczno.pl=i +企业=i +baghdad.museum=i +kamiichi.toyama.jp=i +滋賀.jp=i +k12.in.us=i +yuasa.wakayama.jp=i +hønefoss.no=i +sabae.fukui.jp=i +notaires.km=i +hagebostad.no=i +ujiie.tochigi.jp=i +gliwice.pl=p +tosashimizu.kochi.jp=i +finnøy.no=i +historical.museum=i +solund.no=i +homeip.net=p +shouji=i +lo.it=i +kautokeino.no=i +hichiso.gifu.jp=i +グーグル=i +place=i +showa.fukushima.jp=i +ns.ca=i +is-a-landscaper.com=p +mykolaiv.ua=i +skoczow.pl=i +site=i +ski.museum=i +ota.tokyo.jp=i +lierne.no=i +nagaoka.niigata.jp=i +hidaka.saitama.jp=i +kozagawa.wakayama.jp=i +hirokawa.fukuoka.jp=i +ueno.gunma.jp=i +tr.eu.org=p +gs.fm.no=i +kawaba.gunma.jp=i +is-a-nascarfan.com=p +elasticbeanstalk.com=p +is-into-cars.com=p +is-a-player.com=p +loans=i +公司.hk=i +kpn=i +pasadena.museum=i +servebbs.com=p +corporation.museum=i +int.ci=i +au.eu.org=p +toyota.yamaguchi.jp=i +int.co=i +rennebu.no=i +lucca.it=i +paris=i +immobilien=i +qc.ca=i +krd=i +int.bo=i +dyndns.org=p +sømna.no=i +valleaosta.it=i +int.ar=i +ipiranga=i +limited=i +jelenia-gora.pl=i +int.az=i +flight.aero=i +kyowa.akita.jp=i +creditcard=i +katsushika.tokyo.jp=i +turystyka.pl=i +sauda.no=i +soundandvision.museum=i +nyny.museum=i +xn--blt-elab.no=i +ro.it=i +udine.it=i +minato.osaka.jp=i +ishigaki.okinawa.jp=i +desa.id=i +lego=i +chiyoda.tokyo.jp=i +harstad.no=i +www.ro=i +yamagata.yamagata.jp=i +kawanishi.hyogo.jp=i +oketo.hokkaido.jp=i +latino=i +morimachi.shizuoka.jp=i +masaki.ehime.jp=i +hobby-site.org=p +crotone.it=i +redstone=i +nankoku.kochi.jp=i +weir=i +hosting=i +theguardian=i +xn--ogbpf8fl=i +infiniti=i +is-with-theband.com=p +島根.jp=i +kurgan.su=i +leaŋgaviika.no=i +wanouchi.gifu.jp=i +matta-varjjat.no=i +k12.wy.us=i +ug.gov.pl=i +taishi.osaka.jp=i +atami.shizuoka.jp=i +ts.it=i +pv.it=i +kharkov.ua=i +cincinnati.museum=i +joshkar-ola.ru=i +airguard.museum=i +surgery=i +me.uk=i +go.ug=i +station.museum=i +go.tz=i +imdb=i +hattfjelldal.no=i +nanbu.yamanashi.jp=i +me.us=i +kaizuka.osaka.jp=i +viajes=i +oxford.museum=i +oslo.no=i +lasalle=i +me.tz=i +motosu.gifu.jp=i +akabira.hokkaido.jp=i +ryugasaki.ibaraki.jp=i +nagano.jp=i +hamamatsu.shizuoka.jp=i +yufu.oita.jp=i +hamar.no=i +k12.va.us=i +appspot.com=p +xn--unjrga-rta.no=i +venice.it=i +oshima.yamaguchi.jp=i +sakawa.kochi.jp=i +xn--mix891f=i +maserati=i +olsztyn.pl=i +go.tj=i +go.th=i +zagan.pl=i +katowice.pl=i +hakone.kanagawa.jp=i +fjell.no=i +mjondalen.no=i +reggiocalabria.it=i +mzansimagic=i +simbirsk.ru=i +tjmaxx=i +higashiyodogawa.osaka.jp=i +ardal.no=i +naruto.tokushima.jp=i +kainan.wakayama.jp=i +palace.museum=i +tohma.hokkaido.jp=i +lancome=i +carrier.museum=i +xn--klbu-woa.no=i +plc.co.im=i +date.fukushima.jp=i +radio.br=i +xn--45brj9c=i +kitami.hokkaido.jp=i +xerox=i +vossevangen.no=i +pioneer=i +熊本.jp=i +schmidt=i +airtel=i +go.pw=i +xn--smla-hra.no=i +worse-than.tv=p +toga.toyama.jp=i +vadso.no=i +law.pro=i +intel=i +政府=i +orkdal.no=i +uonuma.niigata.jp=i +xn--vre-eiker-k8a.no=i +المغرب=i +is-a-geek.org=p +mattel=i +naustdal.no=i +wajima.ishikawa.jp=i +oishida.yamagata.jp=i +shakotan.hokkaido.jp=i +club.tw=i +aero=i +nagawa.nagano.jp=i +va.no=i +lier.no=i +kimitsu.chiba.jp=i +chuo.chiba.jp=i +kaga.ishikawa.jp=i +interactive.museum=i +idrett.no=i +iris.arpa=i +walter=i +andasuolo.no=i +from-mt.com=p +cheap=i +tenkawa.nara.jp=i +stjørdalshalsen.no=i +kitadaito.okinawa.jp=i +industries=i +oga.akita.jp=i +商業.tw=i +ac.vn=i +råde.no=i +mihara.hiroshima.jp=i +capital=i +kodaira.tokyo.jp=i +xn--zf0avx.hk=i +seirou.niigata.jp=i +fm.no=i +yoshino.nara.jp=i +hirado.nagasaki.jp=i +mopar=i +stjørdal.no=i +is.it=i +maori.nz=i +jetzt=i +krakow.pl=p +fuchu.tokyo.jp=i +lat=i +law=i +tec.ve=i +xn--mot-tla.no=i +gran.no=i +rennesoy.no=i +yamaguchi.jp=i +veneto.it=i +space-to-rent.com=p +wales=i +jevnaker.no=i +hurdal.no=i +ggee=i +gáŋgaviika.no=i +sr.gov.pl=i +numata.gunma.jp=i +anpachi.gifu.jp=i +gucci=i +bingo=i +sf.no=i +fr.eu.org=p +bonn.museum=i +kizu.kyoto.jp=i +university=i +cr.ua=i +gs.tm.no=i +stavanger.no=i +go.jp=i +kitagawa.kochi.jp=i +imamat=i +kurogi.fukuoka.jp=i +tjeldsund.no=i +southwest.museum=i +se.net=p +va.it=i +carbonia-iglesias.it=i +پاكستان=i +nieruchomosci.pl=i +from-ma.com=p +swidnica.pl=i +tomigusuku.okinawa.jp=i +go.kr=i +ac.za=i +immo=i +lds=i +nov.ru=i +xn--laheadju-7ya.no=i +9.bg=i +kobierzyce.pl=i +jewishart.museum=i +ikeda.osaka.jp=i +toei.aichi.jp=i +edu.ws=i +xn--d5qv7z876c.jp=i +skole.museum=i +fie.ee=i +xn--pssy2u=i +eidsberg.no=i +fermo.it=i +supply=i +gs.vf.no=i +xn--kltx9a.jp=i +shinto.gunma.jp=i +ah.cn=i +docs=i +yotsukaido.chiba.jp=i +press=i +perso.tn=i +tateyama.chiba.jp=i +transport.museum=i +gwangju.kr=i +ohira.tochigi.jp=i +rishirifuji.hokkaido.jp=i +tanabe.kyoto.jp=i +off.ai=i +asago.hyogo.jp=i +lib.co.us=i +jprs=i +hamada.shimane.jp=i +valle-aosta.it=i +labor.museum=i +austevoll.no=i +e.bg=i +kariwa.niigata.jp=i +bj.cn=i +suzu.ishikawa.jp=i +musashino.tokyo.jp=i +mp.br=i +edu.za=i +olbiatempio.it=i +fuchu.toyama.jp=i +sykkylven.no=i +karate.museum=i +perso.sn=i +medecin.km=i +lib.ak.us=i +lubin.pl=i +dodge=i +komatsu.ishikawa.jp=i +gorge.museum=i +aurland.no=i +tottori.jp=i +ono.fukushima.jp=i +xn--8ltr62k.jp=i +higashiyamato.tokyo.jp=i +cc.me.us=i +gs.ah.no=i +gon.pk=i +ln.cn=i +sld.do=i +xn--45q11c=i +egersund.no=i +taiwa.miyagi.jp=i +cc.wi.us=i +medizinhistorisches.museum=i +tateshina.nagano.jp=i +roan.no=i +masuda.shimane.jp=i +urakawa.hokkaido.jp=i +ilawa.pl=i +fashion=i +insurance=i +california.museum=i +newport.museum=i +komoro.nagano.jp=i +a.ssl.fastly.net=p +пр.срб=i +nagara.chiba.jp=i +durham.museum=i +herøy.nordland.no=i +teaches-yoga.com=p +homes=i +agdenes.no=i +takasaki.gunma.jp=i +kanagawa.jp=i +chattanooga.museum=i +gs.bu.no=i +norilsk.ru=i +iveland.no=i +fnd.br=i +kuchinotsu.nagasaki.jp=i +stalbans.museum=i +soeda.fukuoka.jp=i +ايران=i +คอม=i +lomza.pl=i +namerikawa.toyama.jp=i +p.bg=i +etajima.hiroshima.jp=i +aramco=i +kalisz.pl=i +shingu.fukuoka.jp=i +xn--rdal-poa.no=i +page=i +suldal.no=i +augustow.pl=i +cards=i +nalchik.su=i +b.ssl.fastly.net=p +xn--fct429k=i +communication.museum=i +padova.it=i +קום=i +lexus=i +jessheim.no=i +irkutsk.ru=i +niepce.museum=i +xn--mgbtx2b=i +nalchik.ru=i +center=i +oyer.no=i +doha=i +medecin.fr=i +at.eu.org=p +kushima.miyazaki.jp=i +fund=i +cuneo.it=i +compute.amazonaws.com=p +áltá.no=i +is-a-nurse.com=p +wakasa.fukui.jp=i +urausu.hokkaido.jp=i +studio=i +fujieda.shizuoka.jp=i +komaki.aichi.jp=i +erni=i +takasago.hyogo.jp=i +nhs.uk=i +siellak.no=i +rahkkeravju.no=i +iwanai.hokkaido.jp=i +xn--vermgensberater-ctb=i +nakamichi.yamanashi.jp=i +saitama.jp=i +lc.it=i +xn--lcvr32d.hk=i +caa.aero=i +xn--comunicaes-v6a2o.museum=i +roma.museum=i +castres.museum=i +soka.saitama.jp=i +law.za=i +kicks-ass.net=p +hurum.no=i +adm.br=i +homeftp.org=p +herøy.møre-og-romsdal.no=i +seiyo.ehime.jp=i +vanylven.no=i +kamigori.hyogo.jp=i +massacarrara.it=i +winb.gov.pl=i +lib.ga.us=i +kawanishi.yamagata.jp=i +lol=i +solar=i +xn--3oq18vl8pn36a=i +xn--90azh.xn--90a3ac=i +utashinai.hokkaido.jp=i +baseball.museum=i +komae.tokyo.jp=i +xn--vhquv=i +xn--80asehdb=i +yoka.hyogo.jp=i +岡山.jp=i +computer=i +ferrero=i +sassari.it=i +aéroport.ci=i +tosu.saga.jp=i +blackfriday=i +yamatokoriyama.nara.jp=i +ogawa.nagano.jp=i +hawaii.museum=i +ens.tn=i +accident-investigation.aero=i +tsurugashima.saitama.jp=i +global.prod.fastly.net=p +prod=i +services.aero=i +heimatunduhren.museum=i +lpl=i +prof=i +virtual.museum=i +chintai=i +me.it=i +br.com=p +airbus=i +kuban.ru=i +savannahga.museum=i +løten.no=i +xn--bjddar-pta.no=i +scienceandhistory.museum=i +vindafjord.no=i +ot.it=i +zone=i +maison=i +koenig.ru=i +jondal.no=i +yanaizu.fukushima.jp=i +jewelry=i +takaoka.toyama.jp=i +jeonnam.kr=i +helsinki=i +is-a-chef.net=p +yamagata.jp=i +kvænangen.no=i +gjovik.no=i +karikatur.museum=i +kurgan.ru=i +珠宝=i +kutno.pl=i +chonan.chiba.jp=i +pars=i +dontexist.net=p +eti.br=i +sld.pa=i +za.net=p +satte.saitama.jp=i +ishikawa.fukushima.jp=i +vikna.no=i +bike=i +educational.museum=i +wif.gov.pl=i +yurihonjo.akita.jp=i +ltd=i +ishikari.hokkaido.jp=i +vercelli.it=i +tsuruga.fukui.jp=i +mochizuki.nagano.jp=i +kashiwa.chiba.jp=i +ලංකා=i +belau.pw=i +is-certified.com=p +ureshino.mie.jp=i +akita.jp=i +etc.br=i +tn.us=i +office-on-the.net=p +ac.be=i +photography.museum=i +xn--3ds443g=i +numazu.shizuoka.jp=i +tokorozawa.saitama.jp=i +tomi.nagano.jp=i +isa-geek.com=p +is-by.us=p +ranzan.saitama.jp=i +estate.museum=i +rivne.ua=i +toshima.tokyo.jp=i +yokote.akita.jp=i +ac.cn=i +ac.ci=i +yura.wakayama.jp=i +abbott=i +aikawa.kanagawa.jp=i +adult=i +cymru.museum=i +ac.cr=i +kamiamakusa.kumamoto.jp=i +carrara-massa.it=i +mitake.gifu.jp=i +flanders.museum=i +ac.cy=i +xn--4gbrim=i +missile.museum=i +us.gov.pl=i +mint=i +gs.va.no=i +e.se=i +xn--mgbx4cd0ab=i +kunst.museum=i +s3-ap-southeast-1.amazonaws.com=p +catania.it=i +usui.fukuoka.jp=i +pomorze.pl=i +bo.nordland.no=i +yokohama=i +koga.fukuoka.jp=i +mini=i +xn--kpu716f=i +ujitawara.kyoto.jp=i +sells-for-less.com=p +hatsukaichi.hiroshima.jp=i +maritimo.museum=i +trentino-aadige.it=i +xn--risa-5na.no=i +tarnobrzeg.pl=i +bing=i +game-host.org=p +presse.fr=i +viterbo.it=i +williamhill=i +stryn.no=i +society.museum=i +lom.no=i +ch.eu.org=p +tokyo.jp=i +london=i +lipetsk.ru=i +xn--nqv7fs00ema=i +buzen.fukuoka.jp=i +青森.jp=i +olbia-tempio.it=i +from-nh.com=p +sener=i +dontexist.org=p +beardu.no=i +abogado=i +incheon.kr=i +sanfrancisco.museum=i +nabari.mie.jp=i +club=i +legal=i +hokksund.no=i +cesena-forli.it=i +adygeya.ru=i +presse.ci=i +info=i +p.se=i +red.sv=i +koryo.nara.jp=i +ozu.kumamoto.jp=i +cherkasy.ua=i +kamitsue.oita.jp=i +adygeya.su=i +soma.fukushima.jp=i +portal.museum=i +smøla.no=i +owariasahi.aichi.jp=i +campania.it=i +iida.nagano.jp=i +valled-aosta.it=i +toyosato.shiga.jp=i +xn--estv75g=i +gb.com=p +amami.kagoshima.jp=i +xn--hxt814e=i +lom.it=i +symantec=i +భారత్=i +agriculture.museum=i +jfk.museum=i +mibu.tochigi.jp=i +ustka.pl=i +tagami.niigata.jp=i +kuromatsunai.hokkaido.jp=i +nagareyama.chiba.jp=i +sør-varanger.no=i +ac.ae=i +cranbrook.museum=i +skjak.no=i +payu=i +military.museum=i +ac.at=i +homeftp.net=p +عراق=i +xn--sr-aurdal-l8a.no=i +from-mi.com=p +gyeongnam.kr=i +discount=i +supplies=i +historyofscience.museum=i +nnov.ru=i +otsuki.kochi.jp=i +alesund.no=i +bamble.no=i +berlin.museum=i +anquan=i +starachowice.pl=i +malopolska.pl=i +burghof.museum=i +minamiuonuma.niigata.jp=i +is-very-evil.org=p +hyundai=i +skin=i +botanicgarden.museum=i +hu.eu.org=p +man=i +seaport.museum=i +chikuhoku.nagano.jp=i +minami.kyoto.jp=i +localhistory.museum=i +ком=i +ap-northeast-1.compute.amazonaws.com=p +kosei.shiga.jp=i +mba=i +cc.in.us=i +kerryhotels=i +goldpoint=i +tychy.pl=i +ikusaka.nagano.jp=i +bialowieza.pl=i +xn--vgu402c.jp=i +nisshin.aichi.jp=i +oksnes.no=i +wi.us=i +casino=i +watchandclock.museum=i +mcd=i +omitama.ibaraki.jp=i +skánit.no=i +grosseto.it=i +habikino.osaka.jp=i +art.museum=i +网站=i +equipment=i +game.tw=i +mormon=i +dyndns-server.com=p +telefonica=i +cc.tn.us=i +minamiboso.chiba.jp=i +troandin.no=i +bjarkøy.no=i +kawakami.nara.jp=i +toyonaka.osaka.jp=i +toki.gifu.jp=i +yomitan.okinawa.jp=i +med=i +is-very-sweet.org=p +tonsberg.no=i +eng.pro=i +exchange.aero=i +safety.aero=i +xn--leagaviika-52b.no=i +meo=i +men=i +skierva.no=i +ostroda.pl=i +shioya.tochigi.jp=i +谷歌=i +turen.tn=i +servegame.org=p +xn--3pxu8k=i +fortworth.museum=i +kicks-ass.org=p +takko.aomori.jp=i +embroidery.museum=i +cc.gu.us=i +orange=i +takikawa.hokkaido.jp=i +jorpeland.no=i +moscow.museum=i +accenture=i +in.us=i +shinichi.hiroshima.jp=i +zippo=i +ac.rs=i +wegrow.pl=i +drangedal.no=i +ac.ru=i +onojo.fukuoka.jp=i +k12.dc.us=i +ac.rw=i +katano.osaka.jp=i +ac.se=i +annefrank.museum=i +nov.su=i +melbourne=i +sandvik=i +izena.okinawa.jp=i +lib.ms.us=i +miharu.fukushima.jp=i +go.it=i +starostwo.gov.pl=i +is-a-painter.com=p +xn--vg-yiab.no=i +富山.jp=i +xn--xkc2dl3a5ee0h=i +ac.sz=i +xn--bdddj-mrabd.no=i +kashiwazaki.niigata.jp=i +saltdal.no=i +go.id=i +niigata.niigata.jp=i +poznan.pl=p +groundhandling.aero=i +ac.th=i +xn--od0aq3b.hk=i +ac.tj=i +xn--mli-tla.no=i +k12.ca.us=i +新闻=i +sa.it=i +k12.me.us=i +yalta.ua=i +mil=i +østre-toten.no=i +ac.tz=i +ariake.saga.jp=i +boots=i +mit=i +ac.ug=i +porsáŋgu.no=i +xperia=i +associates=i +ac.uk=i +tachikawa.tokyo.jp=i +cn.com=p +losangeles.museum=i +kr.ua=i +svizzera.museum=i +tozsde.hu=i +nore-og-uvdal.no=i +matsuura.nagasaki.jp=i +xn--yer-zna.no=i +happou.akita.jp=i +travel.tt=i +is-found.org=p +xn--rady-ira.no=i +lib.nj.us=i +fm.it=i +open=i +lebesby.no=i +tyumen.ru=i +nid.io=p +shacknet.nu=p +naie.hokkaido.jp=i +katsuura.chiba.jp=i +bharti=i +sa.cr=i +k12.as.us=i +lt.ua=i +shiraoi.hokkaido.jp=i +ac.nz=i +in.eu.org=p +kurume.fukuoka.jp=i +mlb=i +gz.cn=i +ibaraki.ibaraki.jp=i +solutions=i +ino.kochi.jp=i +cr.it=i +higashi.fukuoka.jp=i +aerodrome.aero=i +mls=i +ac.pa=i +sinaapp.com=p +xn--4it797k.jp=i +cartier=i +mma=i +jpmorgan=i +minamidaito.okinawa.jp=i +hu.net=p +tozawa.yamagata.jp=i +marche.it=i +gd.cn=i +ouchi.saga.jp=i +lib.hi.us=i +nyc.museum=i +livorno.it=i +tsumagoi.gunma.jp=i +ac.pr=i +fst.br=i +mallorca.museum=i +bajddar.no=i +aquila.it=i +gamvik.no=i +civilaviation.aero=i +illustration.museum=i +organic=i +porsgrunn.no=i +from-pa.com=p +hægebostad.no=i +beats=i +go.cr=i +rentals=i +sosa.chiba.jp=i +mk.ua=i +travel.pl=i +go.ci=i +money=i +square.museum=i +vaga.no=i +catering=i +from-sc.com=p +isla.pr=i +moe=i +an.it=i +moi=i +mom=i +工行=i +od.ua=i +nanjo.okinawa.jp=i +mov=i +murakami.niigata.jp=i +ac.jp=i +fm.br=i +horonobe.hokkaido.jp=i +azure=i +groks-this.info=p +kvam.no=i +ابوظبي=i +भारत=i +xn--c1avg=i +ruovat.no=i +carboniaiglesias.it=i +obninsk.su=i +sado.niigata.jp=i +vr.it=i +cechire.com=p +xn--90a3ac=i +ac.kr=i +sakura.chiba.jp=i +zlg.br=i +rome.it=i +konan.aichi.jp=i +geology.museum=i +lgbt=i +valledaosta.it=i +presse.ml=i +republican=i +atsugi.kanagawa.jp=i +miho.ibaraki.jp=i +kosa.kumamoto.jp=i +kiho.mie.jp=i +ac.lk=i +bologna.it=i +ulsan.kr=i +academy=i +ac.me=i +charter.aero=i +hockey=i +ac.ma=i +nm.us=i +guitars=i +ac.mu=i +ac.mw=i +xn--gildeskl-g0a.no=i +ballooning.aero=i +msd=i +yashio.saitama.jp=i +hino.tottori.jp=i +presse.km=i +karatsu.saga.jp=i +机构=i +aremark.no=i +unsa.ba=i +tn.it=i +alabama.museum=i +tours=i +joso.ibaraki.jp=i +ogasawara.tokyo.jp=i +nakagyo.kyoto.jp=i +онлайн=i +øvre-eiker.no=i +haboro.hokkaido.jp=i +discover=i +ਭਾਰਤ=i +lotte=i +bardu.no=i +monash=i +mtn=i +lotto=i +mtr=i +gs.nl.no=i +kddi=i +quebec.museum=i +masfjorden.no=i +tomari.hokkaido.jp=i +pagespeedmobilizer.com=p +britishcolumbia.museum=i +ac.gn=i +kumamoto.kumamoto.jp=i +shiriuchi.hokkaido.jp=i +cc.nm.us=i +ssl.origin.cdn77-secure.org=p +yasuda.kochi.jp=i +taito.tokyo.jp=i +asker.no=i +hakusan.ishikawa.jp=i +xn--c1avg.xn--90a3ac=i +lacaixa=i +lorenskog.no=i +palana.ru=i +miyoshi.aichi.jp=i +sic.it=i +fjaler.no=i +tsuno.miyazaki.jp=i +xn--skjk-soa.no=i +ac.id=i +cloudcontrolled.com=p +xn--mgbai9a5eva00b=i +kvalsund.no=i +friuliv-giulia.it=i +sakura.tochigi.jp=i +ac.ir=i +ac.im=i +ac.in=i +3.bg=i +xn--tysvr-vra.no=i +ralingen.no=i +locus=i +pro.vn=i +miyawaka.fukuoka.jp=i +cruises=i +ogaki.gifu.jp=i +is-a-cubicle-slave.com=p +mat.br=i +barcelona=i +tochio.niigata.jp=i +rep.kp=i +hokuto.hokkaido.jp=i +izumi.osaka.jp=i +naka.ibaraki.jp=i +tado.mie.jp=i +tm.za=i +skedsmokorset.no=i +gob.ar=i +google=i +sakurai.nara.jp=i +versailles.museum=i +xn--rsta-fra.no=i +atsuma.hokkaido.jp=i +k-uralsk.ru=i +hasuda.saitama.jp=i +science=i +latina.it=i +fujixerox=i +kusu.oita.jp=i +hioki.kagoshima.jp=i +箇人.hk=i +kiwa.mie.jp=i +hobøl.no=i +imakane.hokkaido.jp=i +iglesiascarbonia.it=i +koshigaya.saitama.jp=i +yamato.fukushima.jp=i +xn--b4w605ferd=i +sa.com=p +cq.cn=i +gob.bo=i +gob.cl=i +furniture.museum=i +kchr.ru=i +ostre-toten.no=i +rotorcraft.aero=i +naroy.no=i +freight.aero=i +website=i +giessen.museum=i +blogspot.co.at=p +u.bg=i +lib.mn.us=i +norddal.no=i +lavagis.no=i +toyoura.hokkaido.jp=i +abo.pa=i +utazu.kagawa.jp=i +gob.do=i +minamiaiki.nagano.jp=i +panasonic=i +miyake.nara.jp=i +ama.aichi.jp=i +j.bg=i +kuji.iwate.jp=i +tønsberg.no=i +avianca=i +lib.ne.us=i +taku.saga.jp=i +blogspot.com=p +fitness=i +farm.museum=i +everbank=i +hiroo.hokkaido.jp=i +friuli-veneziagiulia.it=i +gob.ec=i +xn--asky-ira.no=i +westfalen.museum=i +gob.es=i +cc.ri.us=i +archaeological.museum=i +is-saved.org=p +rindal.no=i +čáhcesuolo.no=i +film.hu=i +linz.museum=i +søndre-land.no=i +ws.na=i +epson=i +xn--9dbhblg6di.museum=i +miura.kanagawa.jp=i +lib.or.us=i +dnsalias.org=p +ab.ca=i +oto.fukuoka.jp=i +dnepropetrovsk.ua=i +nobeoka.miyazaki.jp=i +hangout=i +tx.us=i +creation.museum=i +dating=i +fujisato.akita.jp=i +am.br=i +unjárga.no=i +uz.ua=i +xn--o1ac.xn--90a3ac=i +town.museum=i +sayo.hyogo.jp=i +kemerovo.ru=i +cody.museum=i +xn--kltp7d.jp=i +rich=i +māori.nz=i +gob.gt=i +chieti.it=i +xn--frya-hra.no=i +emerck=i +gjerdrum.no=i +florida.museum=i +gob.hn=i +pro.tt=i +family=i +gift=i +oceanographique.museum=i +career=i +leangaviika.no=i +pro.na=i +cc.tx.us=i +encyclopedic.museum=i +online.museum=i +pro.mv=i +blogspot.co.ke=p +genoa.it=i +srv.br=i +sue.fukuoka.jp=i +project.museum=i +gokase.miyazaki.jp=i +pro.om=i +molde.no=i +nba=i +time.no=i +pavia.it=i +higashiura.aichi.jp=i +tomakomai.hokkaido.jp=i +messina.it=i +chikusei.ibaraki.jp=i +naturbruksgymn.se=i +drøbak.no=i +诺基亚=i +kameoka.kyoto.jp=i +tamatsukuri.ibaraki.jp=i +duckdns.org=p +holiday=i +kitagata.gifu.jp=i +jørpeland.no=i +blogspot.co.id=p +risor.no=i +blogspot.co.il=p +froya.no=i +ibaraki.jp=i +vyatka.ru=i +daiwa.hiroshima.jp=i +kamisato.saitama.jp=i +aichi.jp=i +tsubame.niigata.jp=i +微博=i +is-a-chef.org=p +friuli-vegiulia.it=i +schweiz.museum=i +dazaifu.fukuoka.jp=i +flog.br=i +bømlo.no=i +chuo.yamanashi.jp=i +lurøy.no=i +pro.pr=i +takazaki.miyazaki.jp=i +toho.fukuoka.jp=i +kumiyama.kyoto.jp=i +okuizumo.shimane.jp=i +bato.tochigi.jp=i +blogspot.co.nz=p +us.org=p +nec=i +pccw=i +gob.mx=i +k12.mo.us=i +omi.nagano.jp=i +xn--sgne-gra.no=i +z-2.compute-1.amazonaws.com=p +net=i +sochi.su=i +jur.pro=i +sakura=i +new=i +kusatsu.shiga.jp=i +网络=i +domains=i +intelligence.museum=i +nomi.ishikawa.jp=i +kamishihoro.hokkaido.jp=i +ashoro.hokkaido.jp=i +nfl=i +g12.br=i +emp.br=i +prd.km=i +youth.museum=i +godo.gifu.jp=i +kamitonda.wakayama.jp=i +vaapste.no=i +msk.ru=i +media=i +goodhands=i +ito.shizuoka.jp=i +ngo=i +yono.saitama.jp=i +nyc.mn=p +kvinesdal.no=i +firenze.it=i +trentino-sued-tirol.it=i +frontier=i +gob.pk=i +gob.pe=i +paris.eu.org=p +minamifurano.hokkaido.jp=i +marriott=i +gob.pa=i +k12.md.us=i +鹿児島.jp=i +xn--sr-fron-q1a.no=i +msk.su=i +kiwi.nz=i +nhk=i +giske.no=i +lib.la.us=i +prd.mg=i +indian.museum=i +urbinopesaro.it=i +eng.br=i +世界=i +hotmail=i +山梨.jp=i +birdart.museum=i +gallo=i +higashitsuno.kochi.jp=i +lib.wa.us=i +java=i +murata.miyagi.jp=i +lodingen.no=i +gallery.museum=i +sv.it=i +dance=i +tmall=i +mima.tokushima.jp=i +omsk.ru=i +wazuka.kyoto.jp=i +minato.tokyo.jp=i +xj.cn=i +hadsel.no=i +xn--90ais=i +shoo.okayama.jp=i +flakstad.no=i +zappos=i +ri.it=i +سورية=i +tm.hu=i +سوريا=i +xn--mgbb9fbpob=i +báhcavuotna.no=i +k12.oh.us=i +nakanoto.ishikawa.jp=i +pittsburgh.museum=i +tone.ibaraki.jp=i +weibo=i +fuji.shizuoka.jp=i +pe.kr=i +bokn.no=i +u.se=i +tm.km=i +newmexico.museum=i +xn--stjrdal-s1a.no=i +for-the.biz=p +vågsøy.no=i +ullensaker.no=i +miyota.nagano.jp=i +fudai.iwate.jp=i +ud.it=i +lincoln=i +doesntexist.com=p +from-ne.com=p +yamaga.kumamoto.jp=i +force.museum=i +kyoto.jp=i +mitsubishi=i +kotoura.tottori.jp=i +tm.cy=i +town=i +kadena.okinawa.jp=i +lapy.pl=i +bloomberg=i +from-ks.com=p +tsuchiura.ibaraki.jp=i +engineering=i +boats=i +americanfamily=i +pp.ru=i +karelia.su=i +taa.it=i +from-hi.com=p +kerrylogistics=i +memorial.museum=i +heroy.more-og-romsdal.no=i +joboji.iwate.jp=i +mashiko.tochigi.jp=i +osaki.miyagi.jp=i +karelia.ru=i +orkanger.no=i +oiso.kanagawa.jp=i +stange.no=i +xihuan=i +komvux.se=i +now=i +iiyama.nagano.jp=i +fujioka.gunma.jp=i +fukuyama.hiroshima.jp=i +ohi.fukui.jp=i +delivery=i +toys=i +voss.no=i +kosaka.akita.jp=i +sk.eu.org=p +phoenix.museum=i +shirako.chiba.jp=i +bmoattachments.org=p +naspers=i +kainan.tokushima.jp=i +bd.se=i +mitsuke.niigata.jp=i +xn--krehamn-dxa.no=i +tm.fr=i +wakuya.miyagi.jp=i +overhalla.no=i +bel.tr=i +xn--imr513n=i +tm.pl=i +noda.chiba.jp=i +air.museum=i +nishitosa.kochi.jp=i +vf.no=i +kawakita.ishikawa.jp=i +he.cn=i +omachi.nagano.jp=i +chino.nagano.jp=i +averøy.no=i +recreation.aero=i +nra=i +bz.it=i +aquarelle=i +citadel=i +skodje.no=i +nokia=i +manchester.museum=i +fredrikstad.no=i +shinonsen.hyogo.jp=i +freemasonry.museum=i +imabari.ehime.jp=i +mukawa.hokkaido.jp=i +nrw=i +nakasatsunai.hokkaido.jp=i +omihachiman.shiga.jp=i +ginoza.okinawa.jp=i +apartments=i +商城=i +bahcavuotna.no=i +søgne.no=i +tm.ro=i +direct=i +virginia.museum=i +hokkaido.jp=i +azurewebsites.net=p +hitoyoshi.kumamoto.jp=i +nikolaev.ua=i +is-a-candidate.org=p +karlsoy.no=i +pp.se=i +oita.jp=i +iruma.saitama.jp=i +nirasaki.yamanashi.jp=i +pp.ua=p +gc.ca=i +ntt=i +tonaki.okinawa.jp=i +bargains=i +tm.se=i +tatebayashi.gunma.jp=i +xn--kpry57d=i +nahari.kochi.jp=i +hachirogata.akita.jp=i +homegoods=i +k12.ut.us=i +marshalls=i +komatsushima.tokushima.jp=i +nagasu.kumamoto.jp=i +vantaa.museum=i +baidu=i +sciencesnaturelles.museum=i +niikappu.hokkaido.jp=i +ryukyu=i +forgot.her.name=p +koto.shiga.jp=i +oppegard.no=i +埼玉.jp=i +wolomin.pl=i +funagata.yamagata.jp=i +market=i +lindas.no=i +dnsdojo.com=p +tm.mg=i +oskol.ru=i +tm.mc=i +rakkestad.no=i +esashi.hokkaido.jp=i +tm.no=i +ri.us=i +from-ut.com=p +panerai=i +ichinomiya.chiba.jp=i +kamchatka.ru=i +correios-e-telecomunicações.museum=i +rocks=i +museumvereniging.museum=i +hakata.fukuoka.jp=i +xn--troms-zua.no=i +police.uk=i +uk.net=p +reklam.hu=i +inabe.mie.jp=i +kita.kyoto.jp=i +lib.sd.us=i +taiki.mie.jp=i +sannan.hyogo.jp=i +bo.it=i +vuelos=i +nyc=i +national.museum=i +suzuka.mie.jp=i +nishihara.kumamoto.jp=i +fortmissoula.museum=i +kunimi.fukushima.jp=i +biz.tt=i +biz.tr=i +art.dz=i +xn--bearalvhki-y4a.no=i +org.do=i +biz.tj=i +org.dm=i +الاردن=i +stalowa-wola.pl=i +padua.it=i +storfjord.no=i +omi.niigata.jp=i +tom.ru=i +org.cy=i +chiryu.aichi.jp=i +lukow.pl=i +aero.tt=i +org.et=i +org.es=i +balat.no=i +gos.pk=i +usculture.museum=i +hyllestad.no=i +fineart.museum=i +xn--tnsberg-q1a.no=i +org.eg=i +store.ro=i +org.ee=i +vacations=i +org.ec=i +ddr.museum=i +brother=i +hongo.hiroshima.jp=i +org.dz=i +org.bw=i +tydal.no=i +org.bt=i +org.bs=i +org.br=i +vic.gov.au=i +org.bo=i +software.aero=i +biz.vn=i +xn--mgb2ddes=i +org.bm=i +org.bi=i +org.bh=i +as.us=i +gouv.sn=i +åsnes.no=i +org.bb=i +sanofi=i +org.ba=i +網路.tw=i +xn--i1b6b1a6a2e=i +sch.ae=i +servebbs.org=p +fr.it=i +org.az=i +midori.gunma.jp=i +org.cw=i +readmyblog.org=p +org.cu=i +lezajsk.pl=i +мкд=i +org.co=i +org.cn=i +yatsuka.shimane.jp=i +تونس=i +org.ci=i +actor=i +kagamiishi.fukushima.jp=i +新潟.jp=i +yamaxun=i +cookingchannel=i +tamba.hyogo.jp=i +art.ht=i +gouv.rw=i +obira.hokkaido.jp=i +chukotka.ru=i +wlocl.pl=i +cc.wy.us=i +biz.ua=p +umig.gov.pl=i +frøya.no=i +org.bz=i +sunndal.no=i +on.ca=i +store.ve=i +emerson=i +иком.museum=i +perso.ht=i +graphics=i +gorlice.pl=i +網絡.cn=i +ine.kyoto.jp=i +nishikata.tochigi.jp=i +gouv.ml=i +vinnica.ua=i +org.au=i +org.ar=i +history.museum=i +bu.no=i +yugawa.fukushima.jp=i +org.an=i +org.al=i +org.ai=i +broke-it.net=p +pe.ca=i +org.ag=i +org.af=i +nl.ca=i +org.ae=i +org.ac=i +fg.it=i +sells-for-u.com=p +xn--frna-woa.no=i +pp.az=i +garden.museum=i +ukiha.fukuoka.jp=i +homelinux.net=p +cuisinella=i +wsa.gov.pl=i +skiptvet.no=i +luroy.no=i +lifeinsurance=i +suli.hu=i +yamanouchi.nagano.jp=i +xfinity=i +forgot.his.name=p +risør.no=i +målselv.no=i +microlight.aero=i +stcgroup=i +store.st=i +izumozaki.niigata.jp=i +ca.na=i +kharkiv.ua=i +hk.cn=i +comcast=i +georgia.museum=i +kppsp.gov.pl=i +мон=i +ファッション=i +allfinanz=i +management=i +otaki.saitama.jp=i +obi=i +niiza.saitama.jp=i +kanra.gunma.jp=i +alstom=i +leirvik.no=i +jolster.no=i +etne.no=i +kashima.kumamoto.jp=i +nishinoshima.shimane.jp=i +trieste.it=i +e-burg.ru=i +gamagori.aichi.jp=i +kunohe.iwate.jp=i +yonabaru.okinawa.jp=i +biz.ki=i +ca.it=i +網絡.hk=i +ポイント=i +from-ny.net=p +sk.ca=i +jgora.pl=i +skanit.no=i +web.id=i +fetsund.no=i +hokuryu.hokkaido.jp=i +no.eu.org=p +xn--rst-0na.no=i +gdansk.pl=p +zaporizhzhe.ua=i +miyoshi.tokushima.jp=i +xn--9krt00a=i +biz.nr=i +xn--vry-yla5g.no=i +xn--pbt977c=i +historicalsociety.museum=i +lig.it=i +yuu.yamaguchi.jp=i +leclerc=i +repbody.aero=i +camera=i +shirahama.wakayama.jp=i +biz.mv=i +bungoono.oita.jp=i +biz.mw=i +k12.tx.us=i +art.pl=p +ugim.gov.pl=i +off=i +ah.no=i +stat.no=i +kouhoku.saga.jp=i +xn--h-2fa.no=i +cl.it=i +oguni.yamagata.jp=i +scotland.museum=i +from-ar.com=p +na.it=i +trondheim.no=i +xn--nyqy26a=i +biz.pr=i +countryestate.museum=i +web.co=i +biz.pk=i +biz.pl=i +saintlouis.museum=i +inawashiro.fukushima.jp=i +dyn-o-saur.com=p +shikabe.hokkaido.jp=i +engerdal.no=i +association.aero=i +edogawa.tokyo.jp=i +pe.it=i +ie.eu.org=p +lancaster=i +mytis.ru=i +8.bg=i +eu.com=p +xn--trany-yua.no=i +mt.eu.org=p +web.do=i +shimodate.ibaraki.jp=i +north.museum=i +k12.ri.us=i +suzaka.nagano.jp=i +臺灣=i +academy.museum=i +hagi.yamaguchi.jp=i +art.sn=i +port.fr=i +bnpparibas=i +atm.pl=i +nl.no=i +kurobe.toyama.jp=i +raholt.no=i +katori.chiba.jp=i +uda.nara.jp=i +oygarden.no=i +otaki.nagano.jp=i +oseto.nagasaki.jp=i +iwatsuki.saitama.jp=i +res.aero=i +tomsk.ru=i +træna.no=i +contractors=i +ebiz.tw=i +panama.museum=i +nz.eu.org=p +z.bg=i +za.org=p +friuli-ve-giulia.it=i +sch.sa=i +dnsalias.net=p +mielno.pl=i +maizuru.kyoto.jp=i +lecce.it=i +prd.fr=i +basel.museum=i +conference.aero=i +cc.dc.us=i +kasuya.fukuoka.jp=i +usa.oita.jp=i +o.bg=i +web.pk=i +holtålen.no=i +cherkassy.ua=i +cc.ca.us=i +d.bg=i +bauhaus=i +matsubara.osaka.jp=i +xn--rhkkervju-01af.no=i +sel.no=i +xn--clchc0ea0b2g2a9gcd=i +mine.nu=p +no.com=p +halden.no=i +store.bb=i +izunokuni.shizuoka.jp=i +ueda.nagano.jp=i +tokashiki.okinawa.jp=i +saves-the-whales.com=p +kamikitayama.nara.jp=i +madrid=i +bandai.fukushima.jp=i +iwakuni.yamaguchi.jp=i +wzmiuw.gov.pl=i +opole.pl=i +sch.qa=i +yoga=i +echizen.fukui.jp=i +taishin.fukushima.jp=i +静岡.jp=i +george=i +homeunix.net=p +tokamachi.niigata.jp=i +web.nf=i +maebashi.gunma.jp=i +one=i +kunitachi.tokyo.jp=i +ong=i +va.us=i +la-spezia.it=i +onl=i +nl.eu.org=p +mihama.wakayama.jp=i +osakikamijima.hiroshima.jp=i +岐阜.jp=i +pro.ht=i +dyndns-home.com=p +dealer=i +itabashi.tokyo.jp=i +kunstunddesign.museum=i +tobetsu.hokkaido.jp=i +makeup=i +country=i +daejeon.kr=i +builders=i +pro.az=i +trøgstad.no=i +dyndns.ws=p +ooo=i +matsusaka.mie.jp=i +xn--mgberp4a5d4a87g=i +webhop.biz=p +web.lk=i +kami.kochi.jp=i +oystre-slidre.no=i +selfip.com=p +xn--w4r85el8fhu5dnra=i +xn--mgbbh1a71e=i +pro.br=i +sch.ng=i +handa.aichi.jp=i +parti.se=i +surnadal.no=i +大拿=i +raisa.no=i +support=i +luzern.museum=i +yoichi.hokkaido.jp=i +plc.uk=i +sch.ly=i +donostia.museum=i +khmelnitskiy.ua=i +xn--fhbei=i +collection.museum=i +wien=i +life=i +pro.cy=i +is-a-hunter.com=p +izu.shizuoka.jp=i +tajimi.gifu.jp=i +dyndns.tv=p +vennesla.no=i +org=i +pro.ec=i +kishiwada.osaka.jp=i +kanie.aichi.jp=i +honda=i +gotsu.shimane.jp=i +sch.lk=i +from-nm.com=p +lidl=i +pomorskie.pl=i +bentley=i +muroto.kochi.jp=i +lib.al.us=i +orenburg.ru=i +software=i +salat.no=i +misasa.tottori.jp=i +lørenskog.no=i +xn--fl-zia.no=i +etnedal.no=i +tsuga.tochigi.jp=i +intl.tn=i +aukra.no=i +est-le-patron.com=p +valley.museum=i +wloclawek.pl=i +fhs.no=i +getmyip.com=p +boehringer=i +liguria.it=i +xn--o3cw4h=i +zt.ua=i +stjohn.museum=i +xn--80adxhks=i +valle-daosta.it=i +ott=i +game-server.cc=p +kazuno.akita.jp=i +multichoice=i +nationwide=i +sch.jo=i +aigo=i +songdalen.no=i +isesaki.gunma.jp=i +web.ve=i +zuerich=i +santafe.museum=i +handson.museum=i +kaho.fukuoka.jp=i +hirono.fukushima.jp=i +pictet=i +asahi.ibaraki.jp=i +sch.id=i +八卦=i +shobara.hiroshima.jp=i +beeldengeluid.museum=i +agrar.hu=i +aero.mv=i +ninohe.iwate.jp=i +sch.ir=i +ovh=i +chikushino.fukuoka.jp=i +coop=i +cool=i +shonai.fukuoka.jp=i +on-the-web.tv=p +手表=i +xn--tn0ag.hk=i +date=i +xn--unup4y=i +gausdal.no=i +tochigi.tochigi.jp=i +seki.gifu.jp=i +wiki=i +yame.fukuoka.jp=i +blog=i +karmøy.no=i +piacenza.it=i +emilia-romagna.it=i +yorii.saitama.jp=i +web.tr=i +aosta.it=i +yasuoka.nagano.jp=i +sologne.museum=i +web.tj=i +arboretum.museum=i +flora.no=i +awaji.hyogo.jp=i +is-a-celticsfan.org=p +like=i +roros.no=i +xn--5rtq34k.jp=i +trading.aero=i +zara=i +travelchannel=i +ichihara.chiba.jp=i +xn--ldingen-q1a.no=i +båtsfjord.no=i +ca.us=i +cahcesuolo.no=i +gs.tr.no=i +kashima.ibaraki.jp=i +store.nf=i +arezzo.it=i +youtube=i +dn.ua=i +dc.us=i +shimoji.okinawa.jp=i +link=i +taiki.hokkaido.jp=i +plc.ly=i +sncf=i +school.museum=i +im.it=i +nesna.no=i +kvinnherad.no=i +kui.hiroshima.jp=i +k12.fl.us=i +from-tx.com=p +cc.as.us=i +tsukigata.hokkaido.jp=i +fishing=i +chel.ru=i +limo=i +wy.us=i +eidfjord.no=i +bievát.no=i +statefarm=i +cc.va.us=i +yamada.toyama.jp=i +kisosaki.mie.jp=i +o.se=i +alto-adige.it=i +express.aero=i +公司=i +takasu.hokkaido.jp=i +ed.pw=i +gunma.jp=i +lunner.no=i +inf.br=i +grandrapids.museum=i +läns.museum=i +andebu.no=i +rec.ro=i +trustee.museum=i +conf.lv=i +kuroiso.tochigi.jp=i +elburg.museum=i +cremona.it=i +toba.mie.jp=i +noto.ishikawa.jp=i +d.se=i +sanagochi.tokushima.jp=i +odda.no=i +wine=i +noheji.aomori.jp=i +kagoshima.jp=i +1kapp.com=p +ráisa.no=i +chungnam.kr=i +operaunite.com=p +sorfold.no=i +cv.ua=i +uw.gov.pl=i +farmers.museum=i +machida.tokyo.jp=i +heritage.museum=i +aeroclub.aero=i +mihama.aichi.jp=i +kuroishi.aomori.jp=i +shimizu.hokkaido.jp=i +indiana.museum=i +cc.ks.us=i +vic.edu.au=i +xn--vgan-qoa.no=i +creditunion=i +qld.au=i +vestnes.no=i +yamatotakada.nara.jp=i +blue=i +katsuyama.fukui.jp=i +ck.ua=i +gulen.no=i +vågå.no=i +z.se=i +愛媛.jp=i +taipei=i +broadway=i +aip.ee=i +rec.ve=i +xn--kvfjord-nxa.no=i +mi.it=i +vestre-slidre.no=i +miami=i +jamal.ru=i +nanmoku.gunma.jp=i +varoy.no=i +il.eu.org=p +okayama.okayama.jp=i +surrey.museum=i +xn--berlevg-jxa.no=i +saotome.st=i +arendal.no=i +blog.br=i +kamikawa.hokkaido.jp=i +from-al.com=p +bydgoszcz.pl=i +como.it=i +ventures=i +philately.museum=i +toscana.it=i +lib.ar.us=i +lg.jp=i +claims=i +bremanger.no=i +kinokawa.wakayama.jp=i +podzone.net=p +sicilia.it=i +krasnoyarsk.ru=i +rade.no=i +بھارت=i +fuso.aichi.jp=i +hatoyama.saitama.jp=i +istmein.de=p +hinohara.tokyo.jp=i +cc.id.us=i +liaison=i +higashiizu.shizuoka.jp=i +live=i +hayashima.okayama.jp=i +historisches.museum=i +大众汽车=i +spy.museum=i +hoteles=i +realty=i +friulivegiulia.it=i +mt.it=i +schwarz=i +s3-sa-east-1.amazonaws.com=p +doosan=i +xn--io0a7i.hk=i +asn.au=i +ar.us=i +xn--msy-ula0h.no=i +inf.cu=i +coffee=i +kochi.jp=i +tarama.okinawa.jp=i +brescia.it=i +åmot.no=i +aerobatic.aero=i +workinggroup.aero=i +sennan.osaka.jp=i +shinkamigoto.nagasaki.jp=i +oristano.it=i +xn--uc0ay4a.hk=i +uzs.gov.pl=i +nature.museum=i +jewish.museum=i +far.br=i +pet=i +africa=i +po.it=i +pharmacien.fr=i +aso.kumamoto.jp=i +tw.cn=i +sakai.fukui.jp=i +svalbard.no=i +usgarden.museum=i +copenhagen.museum=i +oki.fukuoka.jp=i +rikubetsu.hokkaido.jp=i +cy.eu.org=p +kameyama.mie.jp=i +pd.it=i +ed.jp=i +ota.gunma.jp=i +oshima.tokyo.jp=i +inagawa.hyogo.jp=i +gs.cn=i +xn--kbrq7o.jp=i +cloudcontrolapp.com=p +is-a-hard-worker.com=p +xn--ntsq17g.jp=i +xn--6orx2r.jp=i +cc.de.us=i +lancia=i +sogne.no=i +iwamizawa.hokkaido.jp=i +bærum.no=i +denmark.museum=i +froland.no=i +delaware.museum=i +amur.ru=i +xn--czrs0t=i +championship.aero=i +tosa.kochi.jp=i +rs.ba=i +flesberg.no=i +dunlop=i +kawajima.saitama.jp=i +okayama.jp=i +xn--mgbayh7gpa=i +xn--io0a7i.cn=i +ar.it=i +xn--nnx388a=i +pid=i +narita.chiba.jp=i +banamex=i +wroc.pl=p +bmd.br=i +narashino.chiba.jp=i +sakahogi.gifu.jp=i +kin.okinawa.jp=i +pin=i +bristol.museum=i +hammerfest.no=i +skydiving.aero=i +ed.cr=i +bi.it=i +ed.ci=i +org.za=i +pordenone.it=i +orland.no=i +roma.it=i +våler.hedmark.no=i +marburg.museum=i +soc.lk=i +nesset.no=i +gs.aa.no=i +afjord.no=i +iwaizumi.iwate.jp=i +is-a-soxfan.org=p +kitaura.miyazaki.jp=i +org.vu=i +design=i +arts.museum=i +org.vn=i +bt.it=i +kosuge.yamanashi.jp=i +portland.museum=i +aguni.okinawa.jp=i +pz.it=i +from-ri.com=p +sorreisa.no=i +org.ws=i +lg.ua=i +agrinet.tn=i +org.ug=i +rnu.tn=i +blogspot.ae=p +org.ua=i +shriram=i +家電=i +org.tw=i +組织.hk=i +kamikawa.hyogo.jp=i +org.tt=i +org.tr=i +xn--lns-qla.museum=i +kuzbass.ru=i +org.to=i +org.tn=i +商店=i +blogspot.am=p +org.tm=i +sakuho.nagano.jp=i +org.tj=i +blogspot.al=p +bible=i +org.vi=i +pnc=i +org.ve=i +ternopil.ua=i +org.vc=i +org.uz=i +asn.lv=i +org.uy=i +takatsuki.shiga.jp=i +trani-barletta-andria.it=i +selfip.biz=p +dallas.museum=i +org.uk=i +org.sh=i +org.sg=i +blogspot.ca=p +org.se=i +org.sd=i +org.sc=i +blogspot.ch=p +badajoz.museum=i +org.sb=i +org.sa=i +blogspot.cf=p +xn--mgb9awbf=i +int.eu.org=p +giving=i +tanagura.fukushima.jp=i +entomology.museum=i +org.ru=i +org.rs=i +richardli=i +blogspot.cv=p +org.ro=i +blogspot.cl=p +lib.ia.us=i +drobak.no=i +gives=i +blogspot.ba=p +blogspot.bg=p +consulting=i +blogspot.be=p +lewismiller.museum=i +org.sz=i +org.sy=i +ismaili=i +org.sv=i +упр.срб=i +org.st=i +lib.il.us=i +sano.tochigi.jp=i +ashibetsu.hokkaido.jp=i +blogspot.bj=p +org.so=i +org.sn=i +org.sl=i +school.na=i +org.qa=i +lib.mi.us=i +org.py=i +c.cdn77.org=p +tanabe.wakayama.jp=i +lib.mt.us=i +sondrio.it=i +plaza.museum=i +org.pt=i +izumisano.osaka.jp=i +org.ps=i +org.pr=i +school.nz=i +hobby-site.com=p +org.pn=i +org.pl=i +org.pk=i +org.ph=i +org.pf=i +homebuilt.aero=i +betainabox.com=p +colonialwilliamsburg.museum=i +finland.museum=i +verran.no=i +oita.oita.jp=i +pro=i +futbol=i +il.us=i +pru=i +kirovograd.ua=i +kounosu.saitama.jp=i +automotive.museum=i +film.museum=i +xn--czr694b=i +ia.us=i +shinyoshitomi.fukuoka.jp=i +fauske.no=i +natori.miyagi.jp=i +org.nz=i +washingtondc.museum=i +kms.ru=i +unbi.ba=i +leikanger.no=i +experts-comptables.fr=i +org.nr=i +dynalias.com=p +davvesiida.no=i +shiranuka.hokkaido.jp=i +civilization.museum=i +tsuruta.aomori.jp=i +org.ng=i +lixil=i +org.pe=i +org.pa=i +perugia.it=i +gjemnes.no=i +群馬.jp=i +lib.ee=i +kihoku.ehime.jp=i +ut.us=i +windows=i +hashikami.aomori.jp=i +tamamura.gunma.jp=i +xn--gckr3f0f=i +org.om=i +lib.nv.us=i +pub=i +ohira.miyagi.jp=i +ladbrokes=i +org.ma=i +xn--s9brj9c=i +blogspot.in=p +trysil.no=i +consulado.st=i +cc.pr.us=i +org.ly=i +org.lv=i +blogspot.ie=p +org.ls=i +org.lr=i +potenza.it=i +homeunix.org=p +izumi.kagoshima.jp=i +org.lk=i +xn--b-5ga.nordland.no=i +bananarepublic=i +org.lc=i +blogspot.is=p +org.lb=i +blogspot.it=p +ايران.ir=i +nissedal.no=i +nagasaki.jp=i +org.na=i +nat.tn=i +blogspot.hk=p +austrheim.no=i +org.my=i +homedns.org=p +herad.no=i +org.mx=i +org.mw=i +beauxarts.museum=i +ullensvang.no=i +org.mv=i +org.mu=i +org.mt=i +xn--bidr-5nac.no=i +org.ms=i +bir.ru=i +kirov.ru=i +org.mo=i +trentino-s-tirol.it=i +org.mn=i +assabu.hokkaido.jp=i +org.ml=i +sardegna.it=i +yakutia.ru=i +org.mk=i +org.mg=i +nittedal.no=i +org.me=i +yasaka.nagano.jp=i +blogspot.hu=p +blogspot.hr=p +xn--brnnysund-m8ac.no=i +naha.okinawa.jp=i +hanamigawa.chiba.jp=i +television.museum=i +balestrand.no=i +nuernberg.museum=i +org.jo=i +monzaebrianza.it=i +tree.museum=i +yachts=i +minamiashigara.kanagawa.jp=i +org.je=i +kitakata.fukushima.jp=i +otsuki.yamanashi.jp=i +blogspot.kr=p +rocher=i +org.la=i +trentinoaadige.it=i +nic.in=i +oshino.yamanashi.jp=i +org.kz=i +org.ky=i +k12.ar.us=i +porsangu.no=i +xn--mgba7c0bbn0a=i +is-gone.com=p +org.kp=i +cc.ny.us=i +org.kn=i +org.km=i +org.ki=i +org.kg=i +blogspot.jp=p +travel=i +dstv=i +southcarolina.museum=i +slupsk.pl=i +uk.com=p +org.hu=i +pvt.k12.ma.us=i +org.ht=i +kawasaki.miyagi.jp=i +дети=i +org.hn=i +org.hk=i +2.bg=i +group=i +blogspot.de=p +inf.mk=i +is-a-financialadvisor.com=p +zao.miyagi.jp=i +music.museum=i +org.is=i +blogspot.cz=p +lu.eu.org=p +org.ir=i +org.iq=i +lanbib.se=i +org.in=i +org.im=i +cc.nc.us=i +vallee-aoste.it=i +pesaro-urbino.it=i +adachi.tokyo.jp=i +blogspot.dk=p +nadex=i +ballangen.no=i +xn--merker-kua.no=i +عمان=i +t.bg=i +hol.no=i +bayern=i +nagasaki.nagasaki.jp=i +xn--hnefoss-q1a.no=i +yakumo.hokkaido.jp=i +suisse.museum=i +exeter.museum=i +stord.no=i +futaba.fukushima.jp=i +gyeonggi.kr=i +shizuoka.jp=i +i.bg=i +florence.it=i +sogndal.no=i +blogspot.gr=p +levanger.no=i +cc.ma.us=i +ebina.kanagawa.jp=i +blogspot.fi=p +pt.eu.org=p +mjøndalen.no=i +org.gt=i +org.gr=i +org.gp=i +org.gn=i +org.gl=i +school.za=i +org.gi=i +org.gh=i +finnoy.no=i +org.gg=i +org.ge=i +miyoshi.saitama.jp=i +so.gov.pl=i +blogspot.fr=p +bugatti=i +xn--c3s14m.jp=i +news.hu=i +mikasa.hokkaido.jp=i +xz.cn=i +tr.it=i +xn--gmqw5a.hk=i +mail.pl=i +takata.fukuoka.jp=i +fiat=i +prime=i +us.com=p +tama.tokyo.jp=i +ikoma.nara.jp=i +nanbu.tottori.jp=i +wajiki.tokushima.jp=i +act.au=i +nagiso.nagano.jp=i +texas.museum=i +is-a-green.com=p +tawaramoto.nara.jp=i +ålgård.no=i +outsystemscloud.com=p +kamikoani.akita.jp=i +xn--sandnessjen-ogb.no=i +trana.no=i +adv.br=i +hr.eu.org=p +odawara.kanagawa.jp=i +mo.us=i +書籍=i +wedding=i +us.eu.org=p +xn--fiqs8s=i +moskenes.no=i +sohu=i +rnrt.tn=i +massa-carrara.it=i +muenster.museum=i +rn.it=i +torsken.no=i +kanazawa.ishikawa.jp=i +kamo.niigata.jp=i +brumunddal.no=i +hamburg.museum=i +xn--q9jyb4c=i +md.us=i +rc.it=i +hanawa.fukushima.jp=i +xn--mxtq1m.hk=i +chigasaki.kanagawa.jp=i +accountant=i +settlement.museum=i +moscow=i +ochi.kochi.jp=i +koga.ibaraki.jp=i +netbank=i +sirdal.no=i +xn--rht27z.jp=i +بازار=i +talk=i +defense.tn=i +childrensgarden.museum=i +oh.us=i +fido=i +орг=i +lib.md.us=i +preservation.museum=i +artsandcrafts.museum=i +matsudo.chiba.jp=i +xn--wcvs22d.hk=i +guovdageaidnu.no=i +yamakita.kanagawa.jp=i +lib.fl.us=i +xn--ehqz56n.jp=i +xn--krager-gya.no=i +abudhabi=i +s3-ap-southeast-2.amazonaws.com=p +ikeda.fukui.jp=i +aogaki.hyogo.jp=i +xn--f6qx53a.jp=i +s3-fips-us-gov-west-1.amazonaws.com=p +research.museum=i +ayase.kanagawa.jp=i +fujiidera.osaka.jp=i +dyndns-work.com=p +xn--bmlo-gra.no=i +labour.museum=i +lib.mo.us=i +pri.ee=i +sp.it=i +ayagawa.kagawa.jp=i +hizen.saga.jp=i +abr.it=i +sagamihara.kanagawa.jp=i +contemporaryart.museum=i +dyndns-web.com=p +leksvik.no=i +stjordal.no=i +fujisawa.iwate.jp=i +insurance.aero=i +时尚=i +ichinomiya.aichi.jp=i +honbetsu.hokkaido.jp=i +modum.no=i +nikaho.akita.jp=i +nakama.fukuoka.jp=i +oshu.iwate.jp=i +ora.gunma.jp=i +rehab=i +himi.toyama.jp=i +cieszyn.pl=i +fl.us=i +cc.ok.us=i +álaheadju.no=i +sony=i +s3-ap-northeast-1.amazonaws.com=p +gloppen.no=i +grp.lk=i +song=i +boutique=i +k12.al.us=i +bosch=i +reggio-emilia.it=i +shinshinotsu.hokkaido.jp=i +måsøy.no=i +xn--t60b56a=i +xn--vler-qoa.xn--stfold-9xa.no=i +shimonita.gunma.jp=i +xn--rny31h.jp=i +xn--flor-jra.no=i +koebenhavn.museum=i +nantan.kyoto.jp=i +fujinomiya.shizuoka.jp=i +koriyama.fukushima.jp=i +bolzano.it=i +mordovia.su=i +nysa.pl=i +uki.kumamoto.jp=i +podlasie.pl=i +mansion.museum=i +health=i +chiba.jp=i +trentinosud-tirol.it=i +furudono.fukushima.jp=i +grondar.za=i +from-me.org=p +bahn.museum=i +dyndns-office.com=p +nara.jp=i +artdeco.museum=i +res.in=i +skedsmo.no=i +xn--kvitsy-fya.no=i +kiyose.tokyo.jp=i +tachiarai.fukuoka.jp=i +miasta.pl=i +xn--rennesy-v1a.no=i +perm.ru=i +smile=i +cn-north-1.compute.amazonaws.cn=p +skiervá.no=i +ricoh=i +nextdirect=i +reviews=i +xn--rskog-uua.no=i +host=i +kongsberg.no=i +sagae.yamagata.jp=i +foundation.museum=i +molise.it=i +kita.tokyo.jp=i +fujitsu=i +cmw.ru=i +logistics.aero=i +osaka.jp=i +tainai.niigata.jp=i +tr.no=i +cc.ky.us=i +bronnoy.no=i +movistar=i +film=i +franziskaner.museum=i +vv.it=i +i.ph=i +mordovia.ru=i +nic.tj=i +austin.museum=i +xn--6btw5a.jp=i +xn--5js045d.jp=i +nakagawa.nagano.jp=i +mazury.pl=i +i.se=i +civilisation.museum=i +årdal.no=i +minamiise.mie.jp=i +nikon=i +laspezia.it=i +kunitomi.miyazaki.jp=i +kyotango.kyoto.jp=i +mitsue.nara.jp=i +inashiki.ibaraki.jp=i +scrapping.cc=p +xn--moreke-jua.no=i +xn--l1acc=i +davvenjárga.no=i +wuoz.gov.pl=i +okawa.fukuoka.jp=i +matera.it=i +yokosuka.kanagawa.jp=i +caravan=i +online=i +blanco=i +eu-central-1.compute.amazonaws.com=p +xn--mori-qsa.nz=i +gs.jan-mayen.no=i +namegata.ibaraki.jp=i +numata.hokkaido.jp=i +mo.cn=i +ritto.shiga.jp=i +se.com=p +edunet.tn=i +aga.niigata.jp=i +ingatlan.hu=i +tananger.no=i +susono.shizuoka.jp=i +dclk=i +nsw.au=i +fyresdal.no=i +shichikashuku.miyagi.jp=i +fire=i +santacruz.museum=i +akdn=i +kawatana.nagasaki.jp=i +shinanomachi.nagano.jp=i +alibaba=i +mnet=i +lib.ut.us=i +sorum.no=i +taxi=i +kasuga.hyogo.jp=i +kitahata.saga.jp=i +md.ci=i +homedepot=i +sydney.museum=i +tysnes.no=i +neues.museum=i +ryazan.ru=i +t.se=i +神奈川.jp=i +brandywinevalley.museum=i +rælingen.no=i +yokkaichi.mie.jp=i +sasebo.nagasaki.jp=i +emr.it=i +andriatranibarletta.it=i +global=i +dyroy.no=i +ehime.jp=i +xn--zbx025d.jp=i +helsinki.museum=i +yaizu.shizuoka.jp=i +ed.ao=i +iwata.shizuoka.jp=i +hashima.gifu.jp=i +nf.ca=i +fed.us=i +kommunalforbund.se=i +bievat.no=i +castle.museum=i +संगठन=i +ag.it=i +mk.eu.org=p +supersport=i +fish=i +xn--mgbt3dhd=i +boldlygoingnowhere.org=p +nome.pt=i +wa.edu.au=i +gifu.jp=i +k12.il.us=i +averoy.no=i +xn--indery-fya.no=i +k12.ia.us=i +ecn.br=i +tsuno.kochi.jp=i +moareke.no=i +higashi.fukushima.jp=i +sund.no=i +higashine.yamagata.jp=i +tinn.no=i +arakawa.saitama.jp=i +sellsyourhome.org=p +samnanger.no=i +eigersund.no=i +aya.miyazaki.jp=i +xn--rland-uua.no=i +ελ=i +dreamhosters.com=p +zoological.museum=i +joetsu.niigata.jp=i +茨城.jp=i +misato.akita.jp=i +lebork.pl=i +mielec.pl=i +himeshima.oita.jp=i +barreau.bj=i +geelvinck.museum=i +nakijin.okinawa.jp=i +sakae.nagano.jp=i +mo.it=i +rodeo=i +kasukabe.saitama.jp=i +blogspot.co.uk=p +otsuchi.iwate.jp=i +hoylandet.no=i +alvdal.no=i +hasama.oita.jp=i +koshimizu.hokkaido.jp=i +eu-west-1.compute.amazonaws.com=p +miasa.nagano.jp=i +nakaniikawa.toyama.jp=i +qvc=i +aki.kochi.jp=i +palmsprings.museum=i +موقع=i +fi.eu.org=p +kawamata.fukushima.jp=i +xn--kcrx77d1x4a=i +xn--langevg-jxa.no=i +sciencecenter.museum=i +網络.hk=i +telekommunikation.museum=i +trentinosued-tirol.it=i +lib.oh.us=i +club.aero=i +saka.hiroshima.jp=i +jeep=i +games=i +imari.saga.jp=i +elverum.no=i +umbria.it=i +staples=i +komi.ru=i +bible.museum=i +okinawa=i +tabuse.yamaguchi.jp=i +tysvar.no=i +ogimi.okinawa.jp=i +kakinoki.shimane.jp=i +onagawa.miyagi.jp=i +s3-us-west-1.amazonaws.com=p +evenes.no=i +tatarstan.ru=i +osaka=i +kamagaya.chiba.jp=i +hisamitsu=i +topology.museum=i +andria-barletta-trani.it=i +dyndns-blog.com=p +antiques.museum=i +vicenza.it=i +ákŋoluokta.no=i +obihiro.hokkaido.jp=i +blogspot.co.za=p +mad.museum=i +coupon=i +agency=i +kuju.oita.jp=i +hino.tokyo.jp=i +uri.arpa=i +zp.gov.pl=i +báidár.no=i +minano.saitama.jp=i +dyndns-pics.com=p +k12.nv.us=i +space=i +chesapeakebay.museum=i +nakagawa.fukuoka.jp=i +servebbs.net=p +4u.com=p +xn--sr-odal-q1a.no=i +tromsa.no=i +xn--g2xx48c=i +kirkenes.no=i +aseral.no=i +viking=i +monza-e-della-brianza.it=i +wios.gov.pl=i +kijo.miyazaki.jp=i +sardinia.it=i +åseral.no=i +travelers=i +umi.fukuoka.jp=i +khmelnytskyi.ua=i +from-la.net=p +kunstsammlung.museum=i +lardal.no=i +tranby.no=i +ena.gifu.jp=i +oirm.gov.pl=i +東京.jp=i +nichinan.tottori.jp=i +modena.it=i +network=i +issmarterthanyou.com=p +k12.mi.us=i +pu.it=i +jp.net=p +jogasz.hu=i +catanzaro.it=i +tokushima.tokushima.jp=i +holmestrand.no=i +xn--mxtq1m=i +xn--lgrd-poac.no=i +vladivostok.ru=i +snåsa.no=i +herokuapp.com=p +trade=i +kv.ua=i +nango.fukushima.jp=i +trentinos-tirol.it=i +loppa.no=i +otoineppu.hokkaido.jp=i +muosat.no=i +shinshiro.aichi.jp=i +k12.mt.us=i +mizuho.tokyo.jp=i +arkhangelsk.ru=i +atlanta.museum=i +ne.us=i +erotika.hu=i +uhren.museum=i +or.th=i +osen.no=i +seihi.nagasaki.jp=i +ne.ug=i +kashima.saga.jp=i +fla.no=i +onna.okinawa.jp=i +pisa.it=i +kakuda.miyagi.jp=i +ne.tz=i +or.tz=i +gs.ol.no=i +arkhangelsk.su=i +asuke.aichi.jp=i +webhop.info=p +lighting=i +or.ug=i +minamiechizen.fukui.jp=i +press.se=i +esan.hokkaido.jp=i +columbus.museum=i +mansions.museum=i +mn.us=i +institute=i +altoadige.it=i +aaa=i +friulivgiulia.it=i +tarumizu.kagoshima.jp=i +notogawa.shiga.jp=i +gjerstad.no=i +natuurwetenschappen.museum=i +togane.chiba.jp=i +fylkesbibl.no=i +niki.hokkaido.jp=i +magnitka.ru=i +from-md.com=p +tabayama.yamanashi.jp=i +lib.ri.us=i +abb=i +eid.no=i +s3-eu-west-1.amazonaws.com=p +abc=i +muosát.no=i +xn--11b4c3d=i +dep.no=i +xn--drbak-wua.no=i +photo=i +fujimi.saitama.jp=i +sling=i +mincom.tn=i +val-d-aosta.it=i +schule=i +cd.eu.org=p +sayama.osaka.jp=i +trentino-altoadige.it=i +佛山=i +es.eu.org=p +ngo.ph=i +xn--l-1fa.no=i +組織.hk=i +me.eu.org=p +tatar=i +marine.ru=i +aco=i +red=i +kumakogen.ehime.jp=i +hasvik.no=i +長崎.jp=i +duck=i +sørum.no=i +omuta.fukuoka.jp=i +novosibirsk.ru=i +ren=i +museumcenter.museum=i +is-a-llama.com=p +gifts=i +kunneppu.hokkaido.jp=i +oumu.hokkaido.jp=i +otobe.hokkaido.jp=i +takahata.yamagata.jp=i +tadotsu.kagawa.jp=i +valle.no=i +tsugaru.aomori.jp=i +gaular.no=i +samegawa.fukushima.jp=i +marnardal.no=i +dali.museum=i +goshiki.hyogo.jp=i +ads=i +nedre-eiker.no=i +mutual=i +ninja=i +bolt.hu=i +blogspot.com.mt=p +øystre-slidre.no=i +minami.fukuoka.jp=i +nagato.yamaguchi.jp=i +exchange=i +yao.osaka.jp=i +trust=i +aeg=i +nagano.nagano.jp=i +motegi.tochigi.jp=i +is-leet.com=p +railroad.museum=i +skjervøy.no=i +doesntexist.org=p +stordal.no=i +chizu.tottori.jp=i +栃木.jp=i +fujikawa.yamanashi.jp=i +blogspot.com.ng=p +yoshida.shizuoka.jp=i +or.us=i +misaki.okayama.jp=i +xn--elqq16h.jp=i +toyo.kochi.jp=i +r.cdn77.net=p +obanazawa.yamagata.jp=i +gs.st.no=i +snillfjord.no=i +design.aero=i +afl=i +turin.it=i +lib.tx.us=i +nishinoomote.kagoshima.jp=i +press.ma=i +koori.fukushima.jp=i +brønnøysund.no=i +wake.okayama.jp=i +tokke.no=i +movie=i +ngo.lk=i +組織.tw=i +scienceandindustry.museum=i +tsushima.nagasaki.jp=i +if.ua=i +accident-prevention.aero=i +student.aero=i +ril=i +xn--vegrshei-c0a.no=i +silk.museum=i +rip=i +rio=i +abira.hokkaido.jp=i +doomdns.org=p +vdonsk.ru=i +woodside=i +swiss=i +is-a-designer.com=p +gjøvik.no=i +suifu.ibaraki.jp=i +otsu.shiga.jp=i +kanoya.kagoshima.jp=i +spot=i +venezia.it=i +xn--czru2d=i +firm.ve=i +aig=i +7.bg=i +bilbao.museum=i +agakhan=i +akaiwa.okayama.jp=i +xn--davvenjrga-y4a.no=i +nishinomiya.hyogo.jp=i +nakagawa.tokushima.jp=i +kawachinagano.osaka.jp=i +satsumasendai.kagoshima.jp=i +xn--h2brj9c=i +inazawa.aichi.jp=i +flå.no=i +its.me=i +mobi=i +narusawa.yamanashi.jp=i +cc.pa.us=i +motorcycle.museum=i +blogdns.org=p +rødøy.no=i +uruma.okinawa.jp=i +ogliastra.it=i +fet.no=i +kviteseid.no=i +qsl.br=i +xn--fpcrj9c3d=i +shizukuishi.iwate.jp=i +k12.vi=i +is-a-bruinsfan.org=p +oe.yamagata.jp=i +xn--mgba3a4fra=i +matsuyama.ehime.jp=i +bas.it=i +vic.au=i +xn--mgba3a4f16a=i +himeji.hyogo.jp=i +act.edu.au=i +kudoyama.wakayama.jp=i +asahi.nagano.jp=i +misawa.aomori.jp=i +horten.no=i +naganohara.gunma.jp=i +science.museum=i +minamisanriku.miyagi.jp=i +c.bg=i +blogspot.com.tr=p +cc.nh.us=i +barlettatraniandria.it=i +us.na=i +chippubetsu.hokkaido.jp=i +reggioemilia.it=i +sowa.ibaraki.jp=i +mcdonalds=i +moda=i +salem.museum=i +орг.срб=i +warmia.pl=i +øygarden.no=i +cafe=i +akrehamn.no=i +villas=i +blogspot.com.uy=p +åfjord.no=i +ogata.akita.jp=i +is-an-accountant.com=p +gjesdal.no=i +xn--ggaviika-8ya47h.no=i +shibata.miyagi.jp=i +鳥取.jp=i +sosnowiec.pl=i +sd.us=i +mitou.yamaguchi.jp=i +tunes=i +sucks=i +skien.no=i +bofa=i +science-fiction.museum=i +xn--4gq48lf9j=i +omura.nagasaki.jp=i +szczecin.pl=i +press.cy=i +anz=i +y.bg=i +misato.wakayama.jp=i +jan-mayen.no=i +singles=i +mus.br=i +n.bg=i +or.ci=i +biei.hokkaido.jp=i +shinjo.okayama.jp=i +or.cr=i +halloffame.museum=i +shiga.jp=i +xn--hylandet-54a.no=i +førde.no=i +app=i +aq.it=i +technology.museum=i +juniper=i +webhop.net=p +arteducation.museum=i +anthro.museum=i +environment.museum=i +娱乐=i +教育.hk=i +abiko.chiba.jp=i +huissier-justice.fr=i +duns=i +kvafjord.no=i +xn--trna-woa.no=i +is-uberleet.com=p +yamagata.ibaraki.jp=i +lecco.it=i +niigata.jp=i +or.at=i +og.ao=i +algard.no=i +uchinomi.kagawa.jp=i +bar.pro=i +karasuyama.tochigi.jp=i +or.bi=i +takanezawa.tochigi.jp=i +xn--lury-ira.no=i +sandnessjøen.no=i +call=i +toyone.aichi.jp=i +bs.it=i +partners=i +is-a-geek.net=p +秋田.jp=i +okegawa.saitama.jp=i +naoshima.kagawa.jp=i +friuli-vgiulia.it=i +run=i +cymru=i +from-in.com=p +celtic.museum=i +doshi.yamanashi.jp=i +bodo.no=i +is-an-artist.com=p +veterinaire.fr=i +casadelamoneda.museum=i +voagat.no=i +my.id=i +from-ky.com=p +mn.it=i +dønna.no=i +higashimatsushima.miyagi.jp=i +communications.museum=i +mc.it=i +itau=i +camp=i +name=i +riik.ee=i +salvadordali.museum=i +voronezh.ru=i +konin.pl=i +rwe=i +tambov.ru=i +matsumoto.nagano.jp=i +ne.kr=i +belluno.it=i +togo.aichi.jp=i +android=i +mandal.no=i +닷컴=i +c.la=p +ikaruga.nara.jp=i +playstation=i +luster.no=i +hidaka.kochi.jp=i +ally=i +kibichuo.okayama.jp=i +pamperedchef=i +hyogo.jp=i +游戏=i +xn--nry-yla5g.no=i +government.aero=i +vagsoy.no=i +ne.jp=i +forsand.no=i +avoues.fr=i +icbc=i +bond=i +amfam=i +target=i +tokyo=i +imperia.it=i +ак.срб=i +svelvik.no=i +xn--0trq7p7nn.jp=i +knowsitall.info=p +aws=i +xn--6qq986b3xl=i +pup.gov.pl=i +gosen.niigata.jp=i +krym.ua=i +casa=i +or.kr=i +k12.nj.us=i +wiki.br=i +xn--55qx5d.cn=i +production.aero=i +building.museum=i +satx.museum=i +romsa.no=i +pt.it=i +axa=i +nis.za=i +santabarbara.museum=i +vestre-toten.no=i +iwafune.tochigi.jp=i +amagasaki.hyogo.jp=i +filatelia.museum=i +xn--30rr7y=i +pi.it=i +k12.ms.us=i +fresenius=i +yuki.ibaraki.jp=i +cash=i +yasu.shiga.jp=i +oirase.aomori.jp=i +fukushima.fukushima.jp=i +udm.ru=i +cars=i +iwanuma.miyagi.jp=i +kashiwara.osaka.jp=i +monzaedellabrianza.it=i +kariya.aichi.jp=i +nishiaizu.fukushima.jp=i +care=i +stockholm=i +blogspot.com.ar=p +blogspot.com.au=p +or.it=i +katsuragi.nara.jp=i +berkeley.museum=i +skånland.no=i +og.it=i +louvre.museum=i +cityeats=i +tsaritsyn.ru=i +not.br=i +astrakhan.ru=i +iizuka.fukuoka.jp=i +kawanehon.shizuoka.jp=i +mino.gifu.jp=i +ina.saitama.jp=i +audnedaln.no=i +sande.xn--mre-og-romsdal-qqb.no=i +or.id=i +sanuki.kagawa.jp=i +cooking=i +from-nc.com=p +or.jp=i +wa.au=i +radom.pl=i +reliance=i +blogspot.com.by=p +tjøme.no=i +blogspot.com.br=p +chofu.tokyo.jp=i +sa.edu.au=i +glass.museum=i +nissan=i +so.it=i +oyabe.toyama.jp=i +hinode.tokyo.jp=i +diamonds=i +blogspot.com.cy=p +navy=i +blogspot.com.co=p +商标=i +fairwinds=i +iwade.wakayama.jp=i +c.se=i +assassination.museum=i +xn--sandy-yua.no=i +abashiri.hokkaido.jp=i +yamal.ru=i +bø.telemark.no=i +miyazaki.jp=i +orientexpress=i +kamaishi.iwate.jp=i +higashimatsuyama.saitama.jp=i +fukuchi.fukuoka.jp=i +odate.akita.jp=i +ne.pw=i +or.pw=i +amsterdam=i +la.us=i +sd.cn=i +kozaki.chiba.jp=i +calabria.it=i +ngo.za=i +moto=i +gangwon.kr=i +chihayaakasaka.osaka.jp=i +blogspot.com.eg=p +kamiizumi.saitama.jp=i +from-mo.com=p +yabu.hyogo.jp=i +blogspot.com.ee=p +harvestcelebration.museum=i +or.mu=i +gujo.gifu.jp=i +rm.it=i +langevag.no=i +blogspot.com.es=p +or.na=i +y.se=i +chtr.k12.ma.us=i +moriya.ibaraki.jp=i +lviv.ua=i +miyama.mie.jp=i +n.se=i +xn--55qx5d.hk=i +sap=i +hanggliding.aero=i +sas=i +nishi.osaka.jp=i +اليمن=i +itoigawa.niigata.jp=i +飞利浦=i +shimogo.fukushima.jp=i +clinic=i +sbi=i +stuttgart.museum=i +ikata.ehime.jp=i +waw.pl=i +kadoma.osaka.jp=i +sbs=i +in.net=p +edu.ge=i +orsta.no=i +edu.gh=i +kostroma.ru=i +edu.gi=i +edu.gl=i +edu.gn=i +sca=i +kr.eu.org=p +bar=i +scb=i +yokoze.saitama.jp=i +dvag=i +verona.it=i +kanmaki.nara.jp=i +stadt.museum=i +敎育.hk=i +bbc=i +xn--vler-qoa.hedmark.no=i +globo=i +nuoro.it=i +edu.hk=i +edu.hn=i +edu.gp=i +edu.gr=i +americanantiques.museum=i +bbt=i +edu.gt=i +deal=i +al.it=i +bcg=i +qa2.com=p +steam.museum=i +bcn=i +hazu.aichi.jp=i +edu.in=i +cc.az.us=i +reisen=i +erimo.hokkaido.jp=i +edu.ht=i +heroy.nordland.no=i +naples.it=i +gotdns.com=p +ses=i +sew=i +bg.eu.org=p +xn--nttery-byae.no=i +amursk.ru=i +sex=i +matsuno.ehime.jp=i +edu.jo=i +forli-cesena.it=i +edu.iq=i +edu.is=i +edu.it=i +visa=i +bergen.no=i +suita.osaka.jp=i +koge.tottori.jp=i +noda.iwate.jp=i +kiwi=i +edu.kg=i +edu.ki=i +edu.km=i +edu.kn=i +edu.kp=i +cbg.ru=i +xn--hobl-ira.no=i +рус=i +bet=i +xn--mgbtf8fl=i +erotica.hu=i +zama.kanagawa.jp=i +vivo=i +বাংলা=i +chrome=i +edu.lk=i +koza.wakayama.jp=i +viva=i +birkenes.no=i +brussels.museum=i +ringsaker.no=i +mashike.hokkaido.jp=i +edu.lr=i +hayakawa.yamanashi.jp=i +sandoy.no=i +edu.ky=i +edu.kz=i +bc.ca=i +edu.la=i +edu.lb=i +yamatsuri.fukushima.jp=i +edu.lc=i +edu.me=i +edu.mg=i +ap-southeast-1.compute.amazonaws.com=p +minamioguni.kumamoto.jp=i +edu.mk=i +xn--ksnes-uua.no=i +takatori.nara.jp=i +edu.ml=i +laquila.it=i +edu.mn=i +taketa.oita.jp=i +isernia.it=i +edu.mo=i +takamatsu.kagawa.jp=i +fukui.jp=i +edu.ms=i +edu.mt=i +shimotsuke.tochigi.jp=i +edu.lv=i +xn--5tzm5g=i +uslivinghistory.museum=i +福岡.jp=i +edu.ly=i +selfip.info=p +holtalen.no=i +edu.ng=i +kouyama.kagoshima.jp=i +living=i +american.museum=i +computerhistory.museum=i +嘉里=i +football=i +edu.nr=i +bindal.no=i +edu.mv=i +edu.mw=i +meiwa.gunma.jp=i +edu.mx=i +edu.my=i +hachinohe.aomori.jp=i +oceanographic.museum=i +kyuragi.saga.jp=i +shiojiri.nagano.jp=i +bid=i +from-nv.com=p +sveio.no=i +mie.jp=i +fukaya.saitama.jp=i +edu.om=i +xn--hery-ira.xn--mre-og-romsdal-qqb.no=i +kristiansund.no=i +bio=i +katagami.akita.jp=i +is.gov.pl=i +endoftheinternet.org=p +togliatti.su=i +ski=i +biz=i +kushimoto.wakayama.jp=i +osøyro.no=i +ulm.museum=i +edu.ph=i +sky=i +lib.wy.us=i +edu.pk=i +edu.pl=i +histoire.museum=i +edu.pn=i +nord-aurdal.no=i +isleofman.museum=i +komforb.se=i +røst.no=i +edu.pr=i +edu.ps=i +edu.pt=i +altai.ru=i +yatomi.aichi.jp=i +budejju.no=i +hachioji.tokyo.jp=i +xn--d1at.xn--90a3ac=i +k12.mn.us=i +veterinaire.km=i +vestby.no=i +jeju.kr=i +várggát.no=i +edu.pa=i +js.cn=i +edu.pe=i +edu.pf=i +air-traffic-control.aero=i +royrvik.no=i +ashiya.hyogo.jp=i +psp.gov.pl=i +xn--trgstad-r1a.no=i +lib.dc.us=i +kitahiroshima.hokkaido.jp=i +edu.py=i +center.museum=i +bellevue.museum=i +lib.as.us=i +rieti.it=i +epilepsy.museum=i +edu.qa=i +grainger=i +research.aero=i +k12.la.us=i +aviation.museum=i +xn--d1acj3b=i +xn--d1alf=i +edu.rs=i +edu.ru=i +is-a-socialist.com=p +izumiotsu.osaka.jp=i +edu.rw=i +sells-it.net=p +geisei.kochi.jp=i +norfolk.museum=i +motoyama.kochi.jp=i +edu.sl=i +edu.sn=i +yonago.tottori.jp=i +bms=i +edu.st=i +office=i +edu.sv=i +manx.museum=i +xn--9et52u=i +bmw=i +edu.sy=i +石川.jp=i +tsukuba.ibaraki.jp=i +edu.sa=i +edu.sb=i +barefoot=i +edu.sc=i +xn--uist22h.jp=i +edu.sd=i +edu.sg=i +tana.no=i +k12.or.us=i +soy=i +zakopane.pl=p +fuel.aero=i +xn--fjord-lra.no=i +osoyro.no=i +lib.va.us=i +bnl=i +edu.tm=i +ráhkkerávju.no=i +nishio.aichi.jp=i +cipriani=i +podhale.pl=i +xn--slt-elab.no=i +edu.to=i +edu.tr=i +edu.tt=i +edu.tw=i +berlevag.no=i +chanel=i +ohda.shimane.jp=i +zgora.pl=i +tur.ar=i +stokke.no=i +edu.tj=i +nationalfirearms.museum=i +donna.no=i +bom=i +boo=i +cnt.br=i +bot=i +hisayama.fukuoka.jp=i +kutchan.hokkaido.jp=i +arao.kumamoto.jp=i +uzhgorod.ua=i +miyazu.kyoto.jp=i +edu.uy=i +maryland.museum=i +tokoname.aichi.jp=i +sebastopol.ua=i +edu.ua=i +chelyabinsk.ru=i +council.aero=i +aircraft.aero=i +tur.br=i +xn--qcka1pmc=i +edu.vn=i +natura=i +edu.vu=i +balashov.su=i +toon.ehime.jp=i +srl=i +hiraya.nagano.jp=i +dell=i +ishinomaki.miyagi.jp=i +tsukiyono.gunma.jp=i +amex=i +komono.mie.jp=i +k12.ne.us=i +srt=i +edu.vc=i +tra.kp=i +edu.ve=i +sklep.pl=i +odessa.ua=i +lib.ca.us=i +mi.us=i +is-lost.org=p +delta=i +firm.nf=i +higashihiroshima.hiroshima.jp=i +hof.no=i +sanda.hyogo.jp=i +oguni.kumamoto.jp=i +jamison.museum=i +grue.no=i +stc=i +gose.nara.jp=i +caserta.it=i +africa.com=p +xn--brnny-wuac.no=i +test.ru=i +rec.br=i +metlife=i +horse=i +xn--ciqpn.hk=i +zgorzelec.pl=i +royken.no=i +همراه=i +mi.th=i +xn--loabt-0qa.no=i +røyrvik.no=i +kitaakita.akita.jp=i +furubira.hokkaido.jp=i +cci.fr=i +sasayama.hyogo.jp=i +eu.org=p +okazaki.aichi.jp=i +kumagaya.saitama.jp=i +trentino.it=i +netflix=i +clock.museum=i +democrat=i +niyodogawa.kochi.jp=i +moss.no=i +fujisawa.kanagawa.jp=i +yawatahama.ehime.jp=i +lyngen.no=i +yk.ca=i +seiro.niigata.jp=i +milano.it=i +test.tj=i +ichikawa.hyogo.jp=i +ve.it=i +karuizawa.nagano.jp=i +store=i +dr.tr=i +ashikaga.tochigi.jp=i +mt.us=i +depot.museum=i +xn--mgba3a3ejt=i +uryu.hokkaido.jp=i +tcm.museum=i +ina.ibaraki.jp=i +kerryproperties=i +stateofdelaware.museum=i +buy=i +neyagawa.osaka.jp=i +express=i +firm.ro=i +miyama.fukuoka.jp=i +zero=i +sobetsu.hokkaido.jp=i +upow.gov.pl=i +minoh.osaka.jp=i +paroch.k12.ma.us=i +سودان=i +scholarships=i +kumano.mie.jp=i +aibetsu.hokkaido.jp=i +xn--snes-poa.no=i +iki.nagasaki.jp=i +oracle=i +pesarourbino.it=i +dr.na=i +istanbul=i +blogdns.net=p +us-west-1.compute.amazonaws.com=p +tube=i +from-wa.com=p +desi=i +gallup=i +naamesjevuemie.no=i +xn--sndre-land-0cb.no=i +hanyu.saitama.jp=i +yamada.iwate.jp=i +kamifurano.hokkaido.jp=i +nv.us=i +mol.it=i +ta.it=i +iselect=i +al.us=i +samsclub=i +rec.co=i +vao.it=i +luxembourg.museum=i +arakawa.tokyo.jp=i +nakagawa.hokkaido.jp=i +hida.gifu.jp=i +de.com=p +shimamoto.osaka.jp=i +from-or.com=p +bern.museum=i +jpn.com=p +takamori.kumamoto.jp=i +kamakura.kanagawa.jp=i +cbre=i +geometre-expert.fr=i +sport.hu=i +hachijo.tokyo.jp=i +higashiyama.kyoto.jp=i +bzh=i +smola.no=i +agr.br=i +review=i +wiih.gov.pl=i +frosta.no=i +weather=i +malselv.no=i +obama.fukui.jp=i +reise=i +guide=i +nagoya=i +nogi.tochigi.jp=i +akiruno.tokyo.jp=i +state.museum=i +journalism.museum=i +1.bg=i +avocat.fr=i +moriyoshi.akita.jp=i +gmail=i +fujishiro.ibaraki.jp=i +racing=i +szex.hu=i +k12.wa.us=i +mantova.it=i +pistoia.it=i +nakatane.kagoshima.jp=i +edu.ac=i +rawa-maz.pl=i +edu.af=i +ålesund.no=i +sciences.museum=i +kota.aichi.jp=i +anjo.aichi.jp=i +archi=i +edu.az=i +webhop.org=p +starnberg.museum=i +edu.ba=i +edu.bb=i +gx.cn=i +dovre.no=i +edu.bh=i +edu.bi=i +susaki.kochi.jp=i +edu.al=i +kvitsoy.no=i +edu.an=i +广东=i +powiat.pl=i +kuriyama.hokkaido.jp=i +edu.ar=i +edu.au=i +firm.co=i +ostroleka.pl=i +edu.bz=i +vevelstad.no=i +edu.ci=i +conf.au=i +xn--nvuotna-hwa.no=i +rec.nf=i +edu.bm=i +tab=i +s.bg=i +edu.bo=i +choshi.chiba.jp=i +ce.it=i +cologne=i +edu.br=i +tsubetsu.hokkaido.jp=i +edu.bs=i +edu.bt=i +at-band-camp.net=p +bn.it=i +likes-pie.com=p +kumejima.okinawa.jp=i +h.bg=i +matsushima.miyagi.jp=i +uchihara.ibaraki.jp=i +mitaka.tokyo.jp=i +tax=i +cdn77-ssl.net=p +kasahara.gifu.jp=i +edu.cn=i +firm.in=i +edu.co=i +safe=i +foggia.it=i +edu.cu=i +sko.gov.pl=i +edu.cw=i +asahikawa.hokkaido.jp=i +cab=i +edu.ec=i +edu.ee=i +edu.eg=i +museet.museum=i +lanxess=i +aa.no=i +kurotaki.nara.jp=i +cal=i +capetown=i +edu.dm=i +edu.do=i +ikeda.hokkaido.jp=i +car=i +cat=i +lærdal.no=i +is-a-personaltrainer.com=p +tci=i +niimi.okayama.jp=i +firm.ht=i +gr.com=p +edu.dz=i +paderborn.museum=i +cleaning=i +kepno.pl=i +срб=i +cba=i +nes.akershus.no=i +al.no=i +cbn=i +ストア=i +pharmaciens.km=i +cbs=i +boston.museum=i +edu.es=i +edu.et=i +tdk=i \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionRequestTests.java new file mode 100644 index 00000000000..3ff165ff872 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionRequestTests.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.ml.action.CloseJobAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class CloseJobActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setCloseTimeout(TimeValue.timeValueMillis(randomNonNegativeLong())); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionResponseTests.java new file mode 100644 index 00000000000..0cdcb2d068d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CloseJobActionResponseTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.CloseJobAction.Response; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class CloseJobActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + return new Response(randomBoolean()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CreateFilterActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CreateFilterActionRequestTests.java new file mode 100644 index 00000000000..6ba252ddf83 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/CreateFilterActionRequestTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.PutFilterAction.Request; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class CreateFilterActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + int size = randomInt(10); + List items = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + items.add(randomAsciiOfLengthBetween(1, 20)); + } + MlFilter filter = new MlFilter(randomAsciiOfLengthBetween(1, 20), items); + return new PutFilterAction.Request(filter); + } + + @Override + protected Request createBlankInstance() { + return new PutFilterAction.Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return PutFilterAction.Request.parseRequest(parser); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java new file mode 100644 index 00000000000..ecef03d16cc --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DatafeedJobsIT.java @@ -0,0 +1,291 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodeHotThreads; +import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsResponse; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.persistent.PersistentActionResponse; +import org.elasticsearch.xpack.persistent.RemovePersistentTaskAction; +import org.junit.After; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.ml.integration.TooManyJobsIT.ensureClusterStateConsistencyWorkAround; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +@ESIntegTestCase.ClusterScope(numDataNodes = 1) +public class DatafeedJobsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MlPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @After + public void clearMlMetadata() throws Exception { + clearMlMetadata(client()); + } + + public void testLookbackOnly() throws Exception { + client().admin().indices().prepareCreate("data-1") + .addMapping("type", "time", "type=date") + .get(); + long numDocs = randomIntBetween(32, 2048); + long now = System.currentTimeMillis(); + long oneWeekAgo = now - 604800000; + long twoWeeksAgo = oneWeekAgo - 604800000; + indexDocs("data-1", numDocs, twoWeeksAgo, oneWeekAgo); + + client().admin().indices().prepareCreate("data-2") + .addMapping("type", "time", "type=date") + .get(); + long numDocs2 = randomIntBetween(32, 2048); + indexDocs("data-2", numDocs2, oneWeekAgo, now); + + Job.Builder job = createJob("lookback-job"); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build(true, job.getId())); + PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); + assertTrue(putJobResponse.isAcknowledged()); + client().execute(InternalOpenJobAction.INSTANCE, new InternalOpenJobAction.Request(job.getId())); + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); + assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED); + }); + + DatafeedConfig datafeedConfig = createDatafeed(job.getId() + "-datafeed", job.getId(), Collections.singletonList("data-*")); + PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(datafeedConfig); + PutDatafeedAction.Response putDatafeedResponse = client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).get(); + assertTrue(putDatafeedResponse.isAcknowledged()); + + StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedConfig.getId(), 0L); + startDatafeedRequest.setEndTime(now); + PersistentActionResponse startDatafeedResponse = + client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get(); + assertBusy(() -> { + DataCounts dataCounts = getDataCounts(job.getId()); + assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs + numDocs2)); + assertThat(dataCounts.getOutOfOrderTimeStampCount(), equalTo(0L)); + + GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedConfig.getId()); + GetDatafeedsStatsAction.Response response = client().execute(GetDatafeedsStatsAction.INSTANCE, request).actionGet(); + assertThat(response.getResponse().results().get(0).getDatafeedState(), equalTo(DatafeedState.STOPPED)); + }); + } + + public void testRealtime() throws Exception { + client().admin().indices().prepareCreate("data") + .addMapping("type", "time", "type=date") + .get(); + long numDocs1 = randomIntBetween(32, 2048); + long now = System.currentTimeMillis(); + long lastWeek = now - 604800000; + indexDocs("data", numDocs1, lastWeek, now); + + Job.Builder job = createJob("realtime-job"); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build(true, job.getId())); + PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); + assertTrue(putJobResponse.isAcknowledged()); + client().execute(InternalOpenJobAction.INSTANCE, new InternalOpenJobAction.Request(job.getId())); + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); + assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED); + }); + + DatafeedConfig datafeedConfig = createDatafeed(job.getId() + "-datafeed", job.getId(), Collections.singletonList("data")); + PutDatafeedAction.Request putDatafeedRequest = new PutDatafeedAction.Request(datafeedConfig); + PutDatafeedAction.Response putDatafeedResponse = client().execute(PutDatafeedAction.INSTANCE, putDatafeedRequest).get(); + assertTrue(putDatafeedResponse.isAcknowledged()); + + StartDatafeedAction.Request startDatafeedRequest = new StartDatafeedAction.Request(datafeedConfig.getId(), 0L); + PersistentActionResponse startDatafeedResponse = + client().execute(StartDatafeedAction.INSTANCE, startDatafeedRequest).get(); + assertBusy(() -> { + DataCounts dataCounts = getDataCounts(job.getId()); + assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1)); + assertThat(dataCounts.getOutOfOrderTimeStampCount(), equalTo(0L)); + }); + + long numDocs2 = randomIntBetween(2, 64); + now = System.currentTimeMillis(); + indexDocs("data", numDocs2, now + 5000, now + 6000); + assertBusy(() -> { + DataCounts dataCounts = getDataCounts(job.getId()); + assertThat(dataCounts.getProcessedRecordCount(), equalTo(numDocs1 + numDocs2)); + assertThat(dataCounts.getOutOfOrderTimeStampCount(), equalTo(0L)); + }, 30, TimeUnit.SECONDS); + + StopDatafeedAction.Request stopDatafeedRequest = new StopDatafeedAction.Request(datafeedConfig.getId()); + try { + RemovePersistentTaskAction.Response stopJobResponse = client().execute(StopDatafeedAction.INSTANCE, stopDatafeedRequest).get(); + assertTrue(stopJobResponse.isAcknowledged()); + } catch (Exception e) { + NodesHotThreadsResponse nodesHotThreadsResponse = client().admin().cluster().prepareNodesHotThreads().get(); + int i = 0; + for (NodeHotThreads nodeHotThreads : nodesHotThreadsResponse.getNodes()) { + logger.info(i++ + ":\n" +nodeHotThreads.getHotThreads()); + } + throw e; + } + assertBusy(() -> { + GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedConfig.getId()); + GetDatafeedsStatsAction.Response response = client().execute(GetDatafeedsStatsAction.INSTANCE, request).actionGet(); + assertThat(response.getResponse().results().get(0).getDatafeedState(), equalTo(DatafeedState.STOPPED)); + }); + } + + private void indexDocs(String index, long numDocs, long start, long end) { + int maxDelta = (int) (end - start - 1); + BulkRequestBuilder bulkRequestBuilder = client().prepareBulk(); + for (int i = 0; i < numDocs; i++) { + IndexRequest indexRequest = new IndexRequest(index, "type"); + long timestamp = start + randomIntBetween(0, maxDelta); + assert timestamp >= start && timestamp < end; + indexRequest.source("time", timestamp); + bulkRequestBuilder.add(indexRequest); + } + BulkResponse bulkResponse = bulkRequestBuilder + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + assertThat(bulkResponse.hasFailures(), is(false)); + logger.info("Indexed [{}] documents", numDocs); + } + + private Job.Builder createJob(String jobId) { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.JSON); + dataDescription.setTimeFormat("yyyy-MM-dd HH:mm:ss"); + + Detector.Builder d = new Detector.Builder("count", null); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(d.build())); + + Job.Builder builder = new Job.Builder(); + builder.setId(jobId); + + builder.setAnalysisConfig(analysisConfig); + builder.setDataDescription(dataDescription); + return builder; + } + + private DatafeedConfig createDatafeed(String datafeedId, String jobId, List indexes) { + DatafeedConfig.Builder builder = new DatafeedConfig.Builder(datafeedId, jobId); + builder.setQueryDelay(1); + builder.setFrequency(2); + builder.setIndexes(indexes); + builder.setTypes(Collections.singletonList("type")); + return builder.build(); + } + + private DataCounts getDataCounts(String jobId) { + GetResponse getResponse = client().prepareGet(AnomalyDetectorsIndex.jobResultsIndexName(jobId), + DataCounts.TYPE.getPreferredName(), jobId + "-data-counts").get(); + if (getResponse.isExists() == false) { + return new DataCounts(jobId); + } + + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, getResponse.getSourceAsBytesRef())) { + return DataCounts.PARSER.apply(parser, null); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static void clearMlMetadata(Client client) throws Exception { + deleteAllDatafeeds(client); + deleteAllJobs(client); + } + + private static void deleteAllDatafeeds(Client client) throws Exception { + MetaData metaData = client.admin().cluster().prepareState().get().getState().getMetaData(); + MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE); + for (DatafeedConfig datafeed : mlMetadata.getDatafeeds().values()) { + String datafeedId = datafeed.getId(); + try { + RemovePersistentTaskAction.Response stopResponse = + client.execute(StopDatafeedAction.INSTANCE, new StopDatafeedAction.Request(datafeedId)).get(); + assertTrue(stopResponse.isAcknowledged()); + } catch (ExecutionException e) { + // CONFLICT is ok, as it means the datafeed has already stopped, which isn't an issue at all. + if (RestStatus.CONFLICT != ExceptionsHelper.status(e.getCause())) { + throw new RuntimeException(e); + } + } + assertBusy(() -> { + try { + GetDatafeedsStatsAction.Request request = new GetDatafeedsStatsAction.Request(datafeedId); + GetDatafeedsStatsAction.Response r = client.execute(GetDatafeedsStatsAction.INSTANCE, request).get(); + assertThat(r.getResponse().results().get(0).getDatafeedState(), equalTo(DatafeedState.STOPPED)); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + DeleteDatafeedAction.Response deleteResponse = + client.execute(DeleteDatafeedAction.INSTANCE, new DeleteDatafeedAction.Request(datafeedId)).get(); + assertTrue(deleteResponse.isAcknowledged()); + } + } + + public static void deleteAllJobs(Client client) throws Exception { + MetaData metaData = client.admin().cluster().prepareState().get().getState().getMetaData(); + MlMetadata mlMetadata = metaData.custom(MlMetadata.TYPE); + for (Map.Entry entry : mlMetadata.getJobs().entrySet()) { + String jobId = entry.getKey(); + try { + CloseJobAction.Response response = + client.execute(CloseJobAction.INSTANCE, new CloseJobAction.Request(jobId)).get(); + assertTrue(response.isClosed()); + } catch (Exception e) { + // ignore + } + DeleteJobAction.Response response = + client.execute(DeleteJobAction.INSTANCE, new DeleteJobAction.Request(jobId)).get(); + assertTrue(response.isAcknowledged()); + } + } + + @Override + protected void ensureClusterStateConsistency() throws IOException { + ensureClusterStateConsistencyWorkAround(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedRequestTests.java new file mode 100644 index 00000000000..0f1b24b0b9c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DeleteDatafeedRequestTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class DeleteDatafeedRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DeleteJobRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DeleteJobRequestTests.java new file mode 100644 index 00000000000..6ba6f584eee --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/DeleteJobRequestTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.DeleteJobAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class DeleteJobRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java new file mode 100644 index 00000000000..723dd56e3ff --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.GetBucketsAction.Request; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class GetBucketActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + GetBucketsAction.Request request = new GetBucketsAction.Request(randomAsciiOfLengthBetween(1, 20)); + + if (randomBoolean()) { + request.setTimestamp(String.valueOf(randomLong())); + } else { + if (randomBoolean()) { + request.setMaxNormalizedProbability(randomDouble()); + } + if (randomBoolean()) { + request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setStart(String.valueOf(randomLong())); + } + if (randomBoolean()) { + request.setEnd(String.valueOf(randomLong())); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + } + if (randomBoolean()) { + request.setExpand(randomBoolean()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new GetBucketsAction.Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return GetBucketsAction.Request.parseRequest(null, parser); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java new file mode 100644 index 00000000000..7d884522373 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetBucketsAction.Response; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; +import org.elasticsearch.xpack.ml.job.results.PartitionScore; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class GetBucketActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int sequenceNum = 0; + + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + String jobId = "foo"; + Bucket bucket = new Bucket(jobId, new Date(randomLong()), randomNonNegativeLong()); + if (randomBoolean()) { + bucket.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + int size = randomInt(10); + List bucketInfluencers = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + BucketInfluencer bucketInfluencer = new BucketInfluencer("foo", bucket.getTimestamp(), bucket.getBucketSpan(), + sequenceNum++); + bucketInfluencer.setAnomalyScore(randomDouble()); + bucketInfluencer.setInfluencerFieldName(randomAsciiOfLengthBetween(1, 20)); + bucketInfluencer.setInitialAnomalyScore(randomDouble()); + bucketInfluencer.setProbability(randomDouble()); + bucketInfluencer.setRawAnomalyScore(randomDouble()); + bucketInfluencers.add(bucketInfluencer); + } + bucket.setBucketInfluencers(bucketInfluencers); + } + if (randomBoolean()) { + bucket.setEventCount(randomNonNegativeLong()); + } + if (randomBoolean()) { + bucket.setInitialAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + bucket.setInterim(randomBoolean()); + } + if (randomBoolean()) { + bucket.setMaxNormalizedProbability(randomDouble()); + } + if (randomBoolean()) { + int size = randomInt(10); + List partitionScores = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + partitionScores.add(new PartitionScore(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomDouble(), randomDouble(), randomDouble())); + } + bucket.setPartitionScores(partitionScores); + } + if (randomBoolean()) { + int size = randomInt(10); + Map perPartitionMaxProbability = new HashMap<>(size); + for (int i = 0; i < size; i++) { + perPartitionMaxProbability.put(randomAsciiOfLengthBetween(1, 20), randomDouble()); + } + bucket.setPerPartitionMaxProbability(perPartitionMaxProbability); + } + if (randomBoolean()) { + bucket.setProcessingTimeMs(randomLong()); + } + if (randomBoolean()) { + bucket.setRecordCount(randomInt()); + } + if (randomBoolean()) { + int size = randomInt(10); + List records = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + AnomalyRecord anomalyRecord = new AnomalyRecord(jobId, new Date(randomLong()), randomNonNegativeLong(), sequenceNum++); + anomalyRecord.setAnomalyScore(randomDouble()); + anomalyRecord.setActual(Collections.singletonList(randomDouble())); + anomalyRecord.setTypical(Collections.singletonList(randomDouble())); + anomalyRecord.setProbability(randomDouble()); + anomalyRecord.setInterim(randomBoolean()); + records.add(anomalyRecord); + } + bucket.setRecords(records); + } + hits.add(bucket); + } + QueryPage buckets = new QueryPage<>(hits, listSize, Bucket.RESULTS_FIELD); + return new Response(buckets); + } + + @Override + protected Response createBlankInstance() { + return new GetBucketsAction.Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesRequestTests.java new file mode 100644 index 00000000000..ce7c870b3c3 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesRequestTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class GetCategoriesRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected GetCategoriesAction.Request createTestInstance() { + String jobId = randomAsciiOfLength(10); + GetCategoriesAction.Request request = new GetCategoriesAction.Request(jobId); + if (randomBoolean()) { + request.setCategoryId(randomAsciiOfLength(10)); + } else { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected GetCategoriesAction.Request createBlankInstance() { + return new GetCategoriesAction.Request(); + } + + @Override + protected GetCategoriesAction.Request parseInstance(XContentParser parser) { + return GetCategoriesAction.Request.parseRequest(null, parser); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesResponseTests.java new file mode 100644 index 00000000000..c3fa85ce8b0 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetCategoriesResponseTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.Collections; + +public class GetCategoriesResponseTests extends AbstractStreamableTestCase { + + @Override + protected GetCategoriesAction.Response createTestInstance() { + CategoryDefinition definition = new CategoryDefinition(randomAsciiOfLength(10)); + QueryPage queryPage = + new QueryPage<>(Collections.singletonList(definition), 1L, CategoryDefinition.RESULTS_FIELD); + return new GetCategoriesAction.Response(queryPage); + } + + @Override + protected GetCategoriesAction.Response createBlankInstance() { + return new GetCategoriesAction.Response(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedStatsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedStatsActionRequestTests.java new file mode 100644 index 00000000000..bb1b612bcbf --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedStatsActionRequestTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetDatafeedsStatsAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class GetDatafeedStatsActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomBoolean() ? Job.ALL : randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedStatsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedStatsActionResponseTests.java new file mode 100644 index 00000000000..9ffbda76f95 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedStatsActionResponseTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetDatafeedsStatsAction.Response; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedState; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class GetDatafeedStatsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final Response result; + + int listSize = randomInt(10); + List datafeedStatsList = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + String datafeedId = randomAsciiOfLength(10); + DatafeedState datafeedState = randomFrom(DatafeedState.values()); + + Response.DatafeedStats datafeedStats = new Response.DatafeedStats(datafeedId, datafeedState); + datafeedStatsList.add(datafeedStats); + } + + result = new Response(new QueryPage<>(datafeedStatsList, datafeedStatsList.size(), DatafeedConfig.RESULTS_FIELD)); + + return result; + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedsActionRequestTests.java new file mode 100644 index 00000000000..add0a9e9115 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedsActionRequestTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetDatafeedsAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class GetDatafeedsActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomBoolean() ? Job.ALL : randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedsActionResponseTests.java new file mode 100644 index 00000000000..4538801486d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetDatafeedsActionResponseTests.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.ml.action.GetDatafeedsAction.Response; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class GetDatafeedsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final Response result; + + int listSize = randomInt(10); + List datafeedList = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + String datafeedId = DatafeedConfigTests.randomValidDatafeedId(); + String jobId = randomAsciiOfLength(10); + DatafeedConfig.Builder datafeedConfig = new DatafeedConfig.Builder(datafeedId, jobId); + datafeedConfig.setIndexes(randomSubsetOf(2, Arrays.asList("index-1", "index-2", "index-3"))); + datafeedConfig.setTypes(randomSubsetOf(2, Arrays.asList("type-1", "type-2", "type-3"))); + datafeedConfig.setFrequency(randomNonNegativeLong()); + datafeedConfig.setQueryDelay(randomNonNegativeLong()); + if (randomBoolean()) { + datafeedConfig.setQuery(QueryBuilders.termQuery(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + int scriptsSize = randomInt(3); + if (randomBoolean()) { + List scriptFields = new ArrayList<>(scriptsSize); + for (int scriptIndex = 0; scriptIndex < scriptsSize; scriptIndex++) { + scriptFields.add(new SearchSourceBuilder.ScriptField(randomAsciiOfLength(10), new Script(randomAsciiOfLength(10)), + randomBoolean())); + } + datafeedConfig.setScriptFields(scriptFields); + } + if (randomBoolean()) { + datafeedConfig.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE)); + } + if (randomBoolean() && scriptsSize == 0) { + AggregatorFactories.Builder aggsBuilder = new AggregatorFactories.Builder(); + aggsBuilder.addAggregator(AggregationBuilders.avg(randomAsciiOfLength(10))); + datafeedConfig.setAggregations(aggsBuilder); + } + + datafeedList.add(datafeedConfig.build()); + } + + result = new Response(new QueryPage<>(datafeedList, datafeedList.size(), DatafeedConfig.RESULTS_FIELD)); + + return result; + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetFiltersActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetFiltersActionRequestTests.java new file mode 100644 index 00000000000..28bfa96178f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetFiltersActionRequestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetFiltersAction.Request; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class GetFiltersActionRequestTests extends AbstractStreamableTestCase { + + + @Override + protected Request createTestInstance() { + Request request = new Request(); + if (randomBoolean()) { + request.setFilterId(randomAsciiOfLengthBetween(1, 20)); + } else { + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetFiltersActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetFiltersActionResponseTests.java new file mode 100644 index 00000000000..c40d5985d16 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetFiltersActionResponseTests.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetFiltersAction.Response; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.MlFilter; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.Collections; + +public class GetFiltersActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final QueryPage result; + + MlFilter doc = new MlFilter( + randomAsciiOfLengthBetween(1, 20), Collections.singletonList(randomAsciiOfLengthBetween(1, 20))); + result = new QueryPage<>(Collections.singletonList(doc), 1, MlFilter.RESULTS_FIELD); + return new Response(result); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionRequestTests.java new file mode 100644 index 00000000000..22dd4476fb7 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionRequestTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.GetInfluencersAction.Request; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class GetInfluencersActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser) { + return GetInfluencersAction.Request.parseRequest(null, parser); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + String start = randomBoolean() ? randomAsciiOfLengthBetween(1, 20) : String.valueOf(randomNonNegativeLong()); + request.setStart(start); + } + if (randomBoolean()) { + String end = randomBoolean() ? randomAsciiOfLengthBetween(1, 20) : String.valueOf(randomNonNegativeLong()); + request.setEnd(end); + } + if (randomBoolean()) { + request.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setSort(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDecending(randomBoolean()); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java new file mode 100644 index 00000000000..352bee43bd6 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetInfluencersAction.Response; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class GetInfluencersActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + Influencer influencer = new Influencer(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomAsciiOfLengthBetween(1, 20), new Date(randomNonNegativeLong()), randomNonNegativeLong(), j + 1); + influencer.setAnomalyScore(randomDouble()); + influencer.setInitialAnomalyScore(randomDouble()); + influencer.setProbability(randomDouble()); + influencer.setInterim(randomBoolean()); + hits.add(influencer); + } + QueryPage buckets = new QueryPage<>(hits, listSize, Influencer.RESULTS_FIELD); + return new Response(buckets); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobStatsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobStatsActionRequestTests.java new file mode 100644 index 00000000000..19fdc305606 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobStatsActionRequestTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class GetJobStatsActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomBoolean() ? Job.ALL : randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobStatsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobStatsActionResponseTests.java new file mode 100644 index 00000000000..877425d4af8 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobStatsActionResponseTests.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetJobsStatsAction.Response; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; + +public class GetJobStatsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final Response result; + + int listSize = randomInt(10); + List jobStatsList = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + String jobId = randomAsciiOfLength(10); + + DataCounts dataCounts = new DataCounts(randomAsciiOfLength(10), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate()); + + ModelSizeStats sizeStats = null; + if (randomBoolean()) { + sizeStats = new ModelSizeStats.Builder("foo").build(); + } + JobState jobState = randomFrom(EnumSet.allOf(JobState.class)); + + Response.JobStats jobStats = new Response.JobStats(jobId, dataCounts, sizeStats, jobState); + jobStatsList.add(jobStats); + } + + result = new Response(new QueryPage<>(jobStatsList, jobStatsList.size(), Job.RESULTS_FIELD)); + + return result; + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsActionRequestTests.java new file mode 100644 index 00000000000..ef8b87e3cec --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsActionRequestTests.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetJobsAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class GetJobsActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomBoolean() ? Job.ALL : randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsActionResponseTests.java new file mode 100644 index 00000000000..8d9da97e018 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsActionResponseTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetJobsAction.Response; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.IgnoreDowntime; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class GetJobsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + final Response result; + + int listSize = randomInt(10); + List jobList = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + String jobId = "job" + j; + String description = randomBoolean() ? randomAsciiOfLength(100) : null; + Date createTime = new Date(randomNonNegativeLong()); + Date finishedTime = randomBoolean() ? new Date(randomNonNegativeLong()) : null; + Date lastDataTime = randomBoolean() ? new Date(randomNonNegativeLong()) : null; + long timeout = randomNonNegativeLong(); + AnalysisConfig analysisConfig = new AnalysisConfig.Builder( + Collections.singletonList(new Detector.Builder("metric", "some_field").build())).build(); + AnalysisLimits analysisLimits = new AnalysisLimits(randomNonNegativeLong(), randomNonNegativeLong()); + DataDescription dataDescription = randomBoolean() ? new DataDescription.Builder().build() : null; + ModelDebugConfig modelDebugConfig = randomBoolean() ? new ModelDebugConfig(randomDouble(), randomAsciiOfLength(10)) : null; + IgnoreDowntime ignoreDowntime = randomFrom(IgnoreDowntime.values()); + Long normalizationWindowDays = randomBoolean() ? Long.valueOf(randomIntBetween(0, 365)) : null; + Long backgroundPersistInterval = randomBoolean() ? Long.valueOf(randomIntBetween(3600, 86400)) : null; + Long modelSnapshotRetentionDays = randomBoolean() ? Long.valueOf(randomIntBetween(0, 365)) : null; + Long resultsRetentionDays = randomBoolean() ? Long.valueOf(randomIntBetween(0, 365)) : null; + Map customConfig = randomBoolean() ? Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10)) + : null; + String modelSnapshotId = randomBoolean() ? randomAsciiOfLength(10) : null; + String indexName = randomBoolean() ? "index" + j : null; + Job job = new Job(jobId, description, createTime, finishedTime, lastDataTime, + analysisConfig, analysisLimits, dataDescription, + modelDebugConfig, ignoreDowntime, normalizationWindowDays, backgroundPersistInterval, + modelSnapshotRetentionDays, resultsRetentionDays, customConfig, modelSnapshotId, indexName); + + jobList.add(job); + } + + result = new Response(new QueryPage<>(jobList, jobList.size(), Job.RESULTS_FIELD)); + + return result; + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsStatsActionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsStatsActionTests.java new file mode 100644 index 00000000000..24b28fbd258 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetJobsStatsActionTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.xpack.ml.action.GetJobsStatsAction.TransportAction.determineJobIdsWithoutLiveStats; + +public class GetJobsStatsActionTests extends ESTestCase { + + public void testDetermineJobIds() { + List result = determineJobIdsWithoutLiveStats(Collections.singletonList("id1"), Collections.emptyList()); + assertEquals(1, result.size()); + assertEquals("id1", result.get(0)); + + result = determineJobIdsWithoutLiveStats(Collections.singletonList("id1"), Collections.singletonList( + new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, JobState.CLOSED))); + assertEquals(0, result.size()); + + result = determineJobIdsWithoutLiveStats( + Arrays.asList("id1", "id2", "id3"), Collections.emptyList()); + assertEquals(3, result.size()); + assertEquals("id1", result.get(0)); + assertEquals("id2", result.get(1)); + assertEquals("id3", result.get(2)); + + result = determineJobIdsWithoutLiveStats( + Arrays.asList("id1", "id2", "id3"), + Collections.singletonList(new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, JobState.CLOSED)) + ); + assertEquals(2, result.size()); + assertEquals("id2", result.get(0)); + assertEquals("id3", result.get(1)); + + result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), Arrays.asList( + new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, JobState.CLOSED), + new GetJobsStatsAction.Response.JobStats("id3", new DataCounts("id3"), null, JobState.CLOSED) + )); + assertEquals(1, result.size()); + assertEquals("id2", result.get(0)); + + result = determineJobIdsWithoutLiveStats(Arrays.asList("id1", "id2", "id3"), + Arrays.asList(new GetJobsStatsAction.Response.JobStats("id1", new DataCounts("id1"), null, JobState.CLOSED), + new GetJobsStatsAction.Response.JobStats("id2", new DataCounts("id2"), null, JobState.CLOSED), + new GetJobsStatsAction.Response.JobStats("id3", new DataCounts("id3"), null, JobState.CLOSED))); + assertEquals(0, result.size()); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsActionRequestTests.java new file mode 100644 index 00000000000..c78b7af4c1a --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsActionRequestTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.GetModelSnapshotsAction.Request; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class GetModelSnapshotsActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser) { + return GetModelSnapshotsAction.Request.parseRequest(null, null, parser); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20), randomBoolean() ? null : randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setDescriptionString(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setStart(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setEnd(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setSort(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDescOrder(randomBoolean()); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsActionResponseTests.java new file mode 100644 index 00000000000..9aeec617eeb --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetModelSnapshotsActionResponseTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetModelSnapshotsAction.Response; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class GetModelSnapshotsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + ModelSnapshot snapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + snapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + hits.add(snapshot); + } + QueryPage snapshots = new QueryPage<>(hits, listSize, ModelSnapshot.RESULTS_FIELD); + return new Response(snapshots); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java new file mode 100644 index 00000000000..3f2cfbb3ead --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.GetRecordsAction.Request; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class GetRecordsActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser) { + return GetRecordsAction.Request.parseRequest(null, parser); + } + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + String start = randomBoolean() ? randomAsciiOfLengthBetween(1, 20) : String.valueOf(randomNonNegativeLong()); + request.setStart(start); + } + if (randomBoolean()) { + String end = randomBoolean() ? randomAsciiOfLengthBetween(1, 20) : String.valueOf(randomNonNegativeLong()); + request.setEnd(end); + } + if (randomBoolean()) { + request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setSort(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setDecending(randomBoolean()); + } + if (randomBoolean()) { + request.setAnomalyScore(randomDouble()); + } + if (randomBoolean()) { + request.setIncludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setMaxNormalizedProbability(randomDouble()); + } + if (randomBoolean()) { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + request.setPageParams(new PageParams(from, size)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionResponseTests.java new file mode 100644 index 00000000000..97046d1063e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionResponseTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.GetRecordsAction.Response; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class GetRecordsActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + int listSize = randomInt(10); + List hits = new ArrayList<>(listSize); + String jobId = randomAsciiOfLengthBetween(1, 20); + for (int j = 0; j < listSize; j++) { + AnomalyRecord record = new AnomalyRecord(jobId, new Date(), 600, j + 1); + hits.add(record); + } + QueryPage snapshots = new QueryPage<>(hits, listSize, AnomalyRecord.RESULTS_FIELD); + return new Response(snapshots); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionRequestTests.java new file mode 100644 index 00000000000..6244e61fb31 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionRequestTests.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.xpack.ml.action.OpenJobAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class OpenJobActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setOpenTimeout(TimeValue.timeValueMillis(randomNonNegativeLong())); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionResponseTests.java new file mode 100644 index 00000000000..a7eb687c1a8 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/OpenJobActionResponseTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.OpenJobAction.Response; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class OpenJobActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + return new Response(randomBoolean()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataActionRequestTests.java new file mode 100644 index 00000000000..0f4544214a8 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataActionRequestTests.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class PostDataActionRequestTests extends AbstractStreamableTestCase { + @Override + protected PostDataAction.Request createTestInstance() { + PostDataAction.Request request = new PostDataAction.Request(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setResetStart(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setResetEnd(randomAsciiOfLengthBetween(1, 20)); + } + return request; + } + + @Override + protected PostDataAction.Request createBlankInstance() { + return new PostDataAction.Request(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataActionResponseTests.java new file mode 100644 index 00000000000..6596c6eee0c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataActionResponseTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; +import org.joda.time.DateTime; + +public class PostDataActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected PostDataAction.Response createTestInstance() { + DataCounts counts = new DataCounts(randomAsciiOfLength(10), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), randomIntBetween(1, 1_000_000), + new DateTime(randomDateTimeZone()).toDate(), new DateTime(randomDateTimeZone()).toDate()); + + return new PostDataAction.Response(counts); + } + + @Override + protected PostDataAction.Response createBlankInstance() { + return new PostDataAction.Response("foo") ; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataFlushRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataFlushRequestTests.java new file mode 100644 index 00000000000..04c904d0f76 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataFlushRequestTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.FlushJobAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class PostDataFlushRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLengthBetween(1, 20)); + request.setCalcInterim(randomBoolean()); + if (randomBoolean()) { + request.setStart(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setEnd(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + request.setAdvanceTime(randomAsciiOfLengthBetween(1, 20)); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataFlushResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataFlushResponseTests.java new file mode 100644 index 00000000000..a0aa5de1b68 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PostDataFlushResponseTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.FlushJobAction.Response; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class PostDataFlushResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + return new Response(randomBoolean()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutDatafeedActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutDatafeedActionRequestTests.java new file mode 100644 index 00000000000..99ee6f4d28c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutDatafeedActionRequestTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.PutDatafeedAction.Request; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; +import org.junit.Before; + +import java.util.Arrays; + +public class PutDatafeedActionRequestTests extends AbstractStreamableXContentTestCase { + + private String datafeedId; + + @Before + public void setUpDatafeedId() { + datafeedId = DatafeedConfigTests.randomValidDatafeedId(); + } + + @Override + protected Request createTestInstance() { + DatafeedConfig.Builder datafeedConfig = new DatafeedConfig.Builder(datafeedId, randomAsciiOfLength(10)); + datafeedConfig.setIndexes(Arrays.asList(randomAsciiOfLength(10))); + datafeedConfig.setTypes(Arrays.asList(randomAsciiOfLength(10))); + return new Request(datafeedConfig.build()); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return Request.parseRequest(datafeedId, parser); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutDatafeedActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutDatafeedActionResponseTests.java new file mode 100644 index 00000000000..c2875952019 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutDatafeedActionResponseTests.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.PutDatafeedAction.Response; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import java.util.Arrays; + +public class PutDatafeedActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + DatafeedConfig.Builder datafeedConfig = new DatafeedConfig.Builder( + DatafeedConfigTests.randomValidDatafeedId(), randomAsciiOfLength(10)); + datafeedConfig.setIndexes(Arrays.asList(randomAsciiOfLength(10))); + datafeedConfig.setTypes(Arrays.asList(randomAsciiOfLength(10))); + return new Response(randomBoolean(), datafeedConfig.build()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutJobActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutJobActionRequestTests.java new file mode 100644 index 00000000000..051017b2983 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutJobActionRequestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.PutJobAction.Request; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; +import static org.elasticsearch.xpack.ml.job.config.JobTests.randomValidJobId; + +public class PutJobActionRequestTests extends AbstractStreamableXContentTestCase { + + private final String jobId = randomValidJobId(); + + @Override + protected Request createTestInstance() { + Job.Builder jobConfiguration = buildJobBuilder(jobId); + return new Request(jobConfiguration.build(true, jobConfiguration.getId())); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return Request.parseRequest(jobId, parser); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutJobActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutJobActionResponseTests.java new file mode 100644 index 00000000000..c1b0d9b8db8 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutJobActionResponseTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.PutJobAction.Response; +import org.elasticsearch.xpack.ml.job.config.IgnoreDowntime; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; +import static org.elasticsearch.xpack.ml.job.config.JobTests.randomValidJobId; + +public class PutJobActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + Job.Builder builder = buildJobBuilder(randomValidJobId()); + builder.setIgnoreDowntime(IgnoreDowntime.NEVER); + return new Response(randomBoolean(), builder.build()); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutModelSnapshotDescriptionActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutModelSnapshotDescriptionActionRequestTests.java new file mode 100644 index 00000000000..2bbc17e0dfd --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutModelSnapshotDescriptionActionRequestTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class PutModelSnapshotDescriptionActionRequestTests + extends AbstractStreamableXContentTestCase { + + @Override + protected Request parseInstance(XContentParser parser) { + return UpdateModelSnapshotAction.Request.parseRequest(null, null, parser); + } + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20)); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutModelSnapshotDescriptionActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutModelSnapshotDescriptionActionResponseTests.java new file mode 100644 index 00000000000..883568bce1f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/PutModelSnapshotDescriptionActionResponseTests.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.UpdateModelSnapshotAction.Response; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class PutModelSnapshotDescriptionActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + ModelSnapshot snapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + snapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + return new Response(snapshot); + } + + @Override + protected Response createBlankInstance() { + return new Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotActionRequestTests.java new file mode 100644 index 00000000000..28facc7cd83 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotActionRequestTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction.Request; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class RevertModelSnapshotActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + RevertModelSnapshotAction.Request request = + new RevertModelSnapshotAction.Request(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + request.setDeleteInterveningResults(randomBoolean()); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new RevertModelSnapshotAction.Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return RevertModelSnapshotAction.Request.parseRequest(null, null, parser); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotActionResponseTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotActionResponseTests.java new file mode 100644 index 00000000000..4da4214de3f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/RevertModelSnapshotActionResponseTests.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.RevertModelSnapshotAction.Response; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class RevertModelSnapshotActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected Response createTestInstance() { + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setDescription(randomAsciiOfLengthBetween(1, 20)); + return new RevertModelSnapshotAction.Response(modelSnapshot); + } + + @Override + protected Response createBlankInstance() { + return new RevertModelSnapshotAction.Response(); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionRequestTests.java new file mode 100644 index 00000000000..d1b6fdd95c9 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/StartDatafeedActionRequestTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction.Request; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class StartDatafeedActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + Request request = new Request(randomAsciiOfLength(10), randomNonNegativeLong()); + if (randomBoolean()) { + request.setEndTime(randomNonNegativeLong()); + } + return request; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return Request.parseRequest(null, parser); + } + + public void testValidate() { + Job job1 = DatafeedJobRunnerTests.createDatafeedJob().build(); + MlMetadata mlMetadata1 = new MlMetadata.Builder() + .putJob(job1, false) + .build(); + Exception e = expectThrows(ResourceNotFoundException.class, + () -> StartDatafeedAction.validate("some-datafeed", mlMetadata1)); + assertThat(e.getMessage(), equalTo("No datafeed with id [some-datafeed] exists")); + + DatafeedConfig datafeedConfig1 = DatafeedJobRunnerTests.createDatafeedConfig("foo-datafeed", "foo").build(); + MlMetadata mlMetadata2 = new MlMetadata.Builder(mlMetadata1) + .putDatafeed(datafeedConfig1) + .build(); + e = expectThrows(ElasticsearchStatusException.class, + () -> StartDatafeedAction.validate("foo-datafeed", mlMetadata2)); + assertThat(e.getMessage(), equalTo("cannot start datafeed, expected job state [opened], but got [closed]")); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java new file mode 100644 index 00000000000..a9ee0609163 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/StopDatafeedActionRequestTests.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.xpack.ml.action.StopDatafeedAction.Request; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedConfig; +import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedJob; +import static org.hamcrest.Matchers.equalTo; + +public class StopDatafeedActionRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + Request r = new Request(randomAsciiOfLengthBetween(1, 20)); + return r; + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + public void testValidate() { + Job job = createDatafeedJob().build(); + MlMetadata mlMetadata1 = new MlMetadata.Builder().putJob(job, false).build(); + Exception e = expectThrows(ResourceNotFoundException.class, + () -> StopDatafeedAction.validate("foo", mlMetadata1)); + assertThat(e.getMessage(), equalTo("No datafeed with id [foo] exists")); + + DatafeedConfig datafeedConfig = createDatafeedConfig("foo", "foo").build(); + MlMetadata mlMetadata2 = new MlMetadata.Builder().putJob(job, false) + .putDatafeed(datafeedConfig) + .build(); + StopDatafeedAction.validate("foo", mlMetadata2); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/UpdateJobStateRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/UpdateJobStateRequestTests.java new file mode 100644 index 00000000000..e23acd51f0b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/UpdateJobStateRequestTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.xpack.ml.action.UpdateJobStateAction.Request; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; + +public class UpdateJobStateRequestTests extends AbstractStreamableTestCase { + + @Override + protected Request createTestInstance() { + return new Request(randomAsciiOfLengthBetween(1, 20), randomFrom(JobState.values())); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/UpdateProcessActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/UpdateProcessActionRequestTests.java new file mode 100644 index 00000000000..e8bc72a2d2d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/UpdateProcessActionRequestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.config.JobUpdate; +import org.elasticsearch.xpack.ml.job.config.ModelDebugConfig; +import org.elasticsearch.xpack.ml.support.AbstractStreamableTestCase; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +import java.util.List; + +public class UpdateProcessActionRequestTests extends AbstractStreamableTestCase { + + + @Override + protected UpdateProcessAction.Request createTestInstance() { + ModelDebugConfig config = null; + if (randomBoolean()) { + config = new ModelDebugConfig(5.0, "debug,config"); + } + List updates = null; + if (randomBoolean()) { + + } + return new UpdateProcessAction.Request(randomAsciiOfLength(10), config, updates); + } + + @Override + protected UpdateProcessAction.Request createBlankInstance() { + return new UpdateProcessAction.Request(); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/ValidateDetectorActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/ValidateDetectorActionRequestTests.java new file mode 100644 index 00000000000..95ad22d5698 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/ValidateDetectorActionRequestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.ValidateDetectorAction.Request; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +public class ValidateDetectorActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + Detector.Builder detector; + if (randomBoolean()) { + detector = new Detector.Builder(randomFrom(Detector.COUNT_WITHOUT_FIELD_FUNCTIONS), null); + } else { + detector = new Detector.Builder(randomFrom(Detector.FIELD_NAME_FUNCTIONS), randomAsciiOfLengthBetween(1, 20)); + } + return new Request(detector.build()); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return Request.parseRequest(parser); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/ValidateJobConfigActionRequestTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/ValidateJobConfigActionRequestTests.java new file mode 100644 index 00000000000..1cba0f92fcc --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/ValidateJobConfigActionRequestTests.java @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.action.ValidateJobConfigAction.Request; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.support.AbstractStreamableXContentTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class ValidateJobConfigActionRequestTests extends AbstractStreamableXContentTestCase { + + @Override + protected Request createTestInstance() { + List detectors = new ArrayList<>(); + detectors.add(new Detector.Builder(randomFrom(Detector.FIELD_NAME_FUNCTIONS), randomAsciiOfLengthBetween(1, 20)).build()); + detectors.add(new Detector.Builder(randomFrom(Detector.COUNT_WITHOUT_FIELD_FUNCTIONS), null).build()); + AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(detectors); + analysisConfigBuilder.setBucketSpan(randomIntBetween(60, 86400)); + if (randomBoolean()) { + analysisConfigBuilder.setLatency(randomIntBetween(0, 12)); + } + if (randomBoolean()) { + analysisConfigBuilder.setCategorizationFieldName(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + analysisConfigBuilder.setSummaryCountFieldName(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + List influencers = new ArrayList<>(); + for (int i = randomIntBetween(1, 5); i > 0; --i) { + influencers.add(randomAsciiOfLengthBetween(1, 20)); + } + analysisConfigBuilder.setInfluencers(influencers); + } + if (randomBoolean()) { + analysisConfigBuilder.setOverlappingBuckets(randomBoolean()); + } + if (randomBoolean()) { + analysisConfigBuilder.setMultivariateByFields(randomBoolean()); + } + Job.Builder job = new Job.Builder("ok"); + job.setAnalysisConfig(analysisConfigBuilder); + if (randomBoolean()) { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + if (randomBoolean()) { + dataDescription.setFormat(DataDescription.DataFormat.DELIMITED); + if (randomBoolean()) { + dataDescription.setFieldDelimiter(';'); + } + if (randomBoolean()) { + dataDescription.setQuoteCharacter('\''); + } + } else { + dataDescription.setFormat(DataDescription.DataFormat.JSON); + } + dataDescription.setTimeField(randomAsciiOfLengthBetween(1, 20)); + if (randomBoolean()) { + dataDescription.setTimeFormat("yyyy-MM-dd HH:mm:ssX"); + } + job.setDataDescription(dataDescription); + } + if (randomBoolean()) { + job.setAnalysisLimits(new AnalysisLimits(randomNonNegativeLong(), randomNonNegativeLong())); + } + return new Request(job.build(true, "ok")); + } + + @Override + protected Request createBlankInstance() { + return new Request(); + } + + @Override + protected Request parseInstance(XContentParser parser) { + return Request.parseRequest(parser); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/util/PageParamsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/util/PageParamsTests.java new file mode 100644 index 00000000000..15181393946 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/util/PageParamsTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action.util; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +public class PageParamsTests extends AbstractSerializingTestCase { + + @Override + protected PageParams parseInstance(XContentParser parser) { + return PageParams.PARSER.apply(parser, null); + } + + @Override + protected PageParams createTestInstance() { + int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); + int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; + int size = randomInt(maxSize); + return new PageParams(from, size); + } + + @Override + protected Reader instanceReader() { + return PageParams::new; + } + + public void testValidate_GivenFromIsMinusOne() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PageParams(-1, 100)); + assertEquals("Parameter [from] cannot be < 0", e.getMessage()); + } + + public void testValidate_GivenFromIsMinusTen() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PageParams(-10, 100)); + assertEquals("Parameter [from] cannot be < 0", e.getMessage()); + } + + public void testValidate_GivenSizeIsMinusOne() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PageParams(0, -1)); + assertEquals("Parameter [size] cannot be < 0", e.getMessage()); + } + + public void testValidate_GivenSizeIsMinusHundred() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PageParams(0, -100)); + assertEquals("Parameter [size] cannot be < 0", e.getMessage()); + } + + public void testValidate_GivenFromAndSizeSumIsMoreThan10000() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new PageParams(0, 10001)); + assertEquals("The sum of parameters [from] and [size] cannot be higher than 10000.", e.getMessage()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/util/QueryPageTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/util/QueryPageTests.java new file mode 100644 index 00000000000..19102c3fa99 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/action/util/QueryPageTests.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action.util; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.support.AbstractWireSerializingTestCase; + +import java.util.ArrayList; +import java.util.Date; + +public class QueryPageTests extends AbstractWireSerializingTestCase> { + + @Override + protected QueryPage createTestInstance() { + int hitCount = randomIntBetween(0, 10); + ArrayList hits = new ArrayList<>(); + for (int i = 0; i < hitCount; i++) { + hits.add(new Influencer(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), + randomAsciiOfLengthBetween(1, 20), new Date(), randomNonNegativeLong(), i + 1)); + } + return new QueryPage<>(hits, hitCount, new ParseField("test")); + } + + @Override + protected Reader> instanceReader() { + return (in) -> new QueryPage<>(in, Influencer::new); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/ChunkingConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/ChunkingConfigTests.java new file mode 100644 index 00000000000..b6456a9128d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/ChunkingConfigTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import static org.hamcrest.Matchers.is; + +public class ChunkingConfigTests extends AbstractSerializingTestCase { + + @Override + protected ChunkingConfig createTestInstance() { + return createRandomizedChunk(); + } + + @Override + protected Writeable.Reader instanceReader() { + return ChunkingConfig::new; + } + + @Override + protected ChunkingConfig parseInstance(XContentParser parser) { + return ChunkingConfig.PARSER.apply(parser, null); + } + + public void testConstructorGivenAutoAndTimeSpan() { + expectThrows(IllegalArgumentException.class, () ->new ChunkingConfig(ChunkingConfig.Mode.AUTO, 1000L)); + } + + public void testConstructorGivenOffAndTimeSpan() { + expectThrows(IllegalArgumentException.class, () ->new ChunkingConfig(ChunkingConfig.Mode.OFF, 1000L)); + } + + public void testConstructorGivenManualAndNoTimeSpan() { + expectThrows(IllegalArgumentException.class, () ->new ChunkingConfig(ChunkingConfig.Mode.MANUAL, null)); + } + + public void testIsEnabled() { + assertThat(ChunkingConfig.newAuto().isEnabled(), is(true)); + assertThat(ChunkingConfig.newManual(1000).isEnabled(), is(true)); + assertThat(ChunkingConfig.newOff().isEnabled(), is(false)); + } + + public static ChunkingConfig createRandomizedChunk() { + ChunkingConfig.Mode mode = randomFrom(ChunkingConfig.Mode.values()); + Long timeSpan = null; + if (mode == ChunkingConfig.Mode.MANUAL) { + timeSpan = randomNonNegativeLong(); + if (timeSpan == 0L) { + timeSpan = 1L; + } + } + return new ChunkingConfig(mode, timeSpan); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigTests.java new file mode 100644 index 00000000000..cc138887655 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedConfigTests.java @@ -0,0 +1,288 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class DatafeedConfigTests extends AbstractSerializingTestCase { + + @Override + protected DatafeedConfig createTestInstance() { + return createRandomizedDatafeedConfig(randomAsciiOfLength(10)); + } + + public static DatafeedConfig createRandomizedDatafeedConfig(String jobId) { + DatafeedConfig.Builder builder = new DatafeedConfig.Builder(randomValidDatafeedId(), jobId); + builder.setIndexes(randomStringList(1, 10)); + builder.setTypes(randomStringList(1, 10)); + if (randomBoolean()) { + builder.setQuery(QueryBuilders.termQuery(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + int scriptsSize = randomInt(3); + List scriptFields = new ArrayList<>(scriptsSize); + for (int scriptIndex = 0; scriptIndex < scriptsSize; scriptIndex++) { + scriptFields.add(new SearchSourceBuilder.ScriptField(randomAsciiOfLength(10), new Script(randomAsciiOfLength(10)), + randomBoolean())); + } + if (randomBoolean() && scriptsSize == 0) { + // can only test with a single agg as the xcontent order gets randomized by test base class and then + // the actual xcontent isn't the same and test fail. + // Testing with a single agg is ok as we don't have special list writeable / xconent logic + AggregatorFactories.Builder aggs = new AggregatorFactories.Builder(); + aggs.addAggregator(AggregationBuilders.avg(randomAsciiOfLength(10)).field(randomAsciiOfLength(10))); + builder.setAggregations(aggs); + } + builder.setScriptFields(scriptFields); + if (randomBoolean()) { + builder.setScrollSize(randomIntBetween(0, Integer.MAX_VALUE)); + } + if (randomBoolean()) { + builder.setFrequency(randomNonNegativeLong()); + } + if (randomBoolean()) { + builder.setQueryDelay(randomNonNegativeLong()); + } + if (randomBoolean()) { + builder.setSource(randomBoolean()); + } + if (randomBoolean()) { + builder.setChunkingConfig(ChunkingConfigTests.createRandomizedChunk()); + } + return builder.build(); + } + + private static List randomStringList(int min, int max) { + int size = scaledRandomIntBetween(min, max); + List list = new ArrayList<>(); + for (int i = 0; i < size; i++) { + list.add(randomAsciiOfLength(10)); + } + return list; + } + + @Override + protected Writeable.Reader instanceReader() { + return DatafeedConfig::new; + } + + @Override + protected DatafeedConfig parseInstance(XContentParser parser) { + return DatafeedConfig.PARSER.apply(parser, null).build(); + } + + public void testFillDefaults() { + DatafeedConfig.Builder expectedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1"); + expectedDatafeedConfig.setIndexes(Arrays.asList("index")); + expectedDatafeedConfig.setTypes(Arrays.asList("type")); + expectedDatafeedConfig.setQueryDelay(60L); + expectedDatafeedConfig.setScrollSize(1000); + DatafeedConfig.Builder defaultedDatafeedConfig = new DatafeedConfig.Builder("datafeed1", "job1"); + defaultedDatafeedConfig.setIndexes(Arrays.asList("index")); + defaultedDatafeedConfig.setTypes(Arrays.asList("type")); + + assertEquals(expectedDatafeedConfig.build(), defaultedDatafeedConfig.build()); + } + + public void testEquals_GivenDifferentQueryDelay() { + DatafeedConfig.Builder b1 = createFullyPopulated(); + DatafeedConfig.Builder b2 = createFullyPopulated(); + b2.setQueryDelay(120L); + + DatafeedConfig sc1 = b1.build(); + DatafeedConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentScrollSize() { + DatafeedConfig.Builder b1 = createFullyPopulated(); + DatafeedConfig.Builder b2 = createFullyPopulated(); + b2.setScrollSize(1); + + DatafeedConfig sc1 = b1.build(); + DatafeedConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentFrequency() { + DatafeedConfig.Builder b1 = createFullyPopulated(); + DatafeedConfig.Builder b2 = createFullyPopulated(); + b2.setFrequency(120L); + + DatafeedConfig sc1 = b1.build(); + DatafeedConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentIndexes() { + DatafeedConfig.Builder sc1 = createFullyPopulated(); + DatafeedConfig.Builder sc2 = createFullyPopulated(); + sc2.setIndexes(Arrays.asList("blah", "di", "blah")); + + assertFalse(sc1.build().equals(sc2.build())); + assertFalse(sc2.build().equals(sc1.build())); + } + + public void testEquals_GivenDifferentTypes() { + DatafeedConfig.Builder sc1 = createFullyPopulated(); + DatafeedConfig.Builder sc2 = createFullyPopulated(); + sc2.setTypes(Arrays.asList("blah", "di", "blah")); + + assertFalse(sc1.build().equals(sc2.build())); + assertFalse(sc2.build().equals(sc1.build())); + } + + public void testEquals_GivenDifferentQuery() { + DatafeedConfig.Builder b1 = createFullyPopulated(); + DatafeedConfig.Builder b2 = createFullyPopulated(); + b2.setQuery(QueryBuilders.termQuery("foo", "bar")); + + DatafeedConfig sc1 = b1.build(); + DatafeedConfig sc2 = b2.build(); + assertFalse(sc1.equals(sc2)); + assertFalse(sc2.equals(sc1)); + } + + public void testEquals_GivenDifferentAggregations() { + DatafeedConfig.Builder sc1 = createFullyPopulated(); + DatafeedConfig.Builder sc2 = createFullyPopulated(); + sc2.setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.count("foo"))); + + assertFalse(sc1.build().equals(sc2.build())); + assertFalse(sc2.build().equals(sc1.build())); + } + + private static DatafeedConfig.Builder createFullyPopulated() { + DatafeedConfig.Builder sc = new DatafeedConfig.Builder("datafeed1", "job1"); + sc.setIndexes(Arrays.asList("myIndex")); + sc.setTypes(Arrays.asList("myType1", "myType2")); + sc.setFrequency(60L); + sc.setScrollSize(5000); + sc.setQuery(QueryBuilders.matchAllQuery()); + sc.setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("foo"))); + sc.setQueryDelay(90L); + return sc; + } + + public void testCheckValid_GivenNullIndexes() throws IOException { + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + expectThrows(IllegalArgumentException.class, () -> conf.setIndexes(null)); + } + + public void testCheckValid_GivenEmptyIndexes() throws IOException { + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + conf.setIndexes(Collections.emptyList()); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indexes", "[]"), e.getMessage()); + } + + public void testCheckValid_GivenIndexesContainsOnlyNulls() throws IOException { + List indexes = new ArrayList<>(); + indexes.add(null); + indexes.add(null); + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + conf.setIndexes(indexes); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indexes", "[null, null]"), e.getMessage()); + } + + public void testCheckValid_GivenIndexesContainsOnlyEmptyStrings() throws IOException { + List indexes = new ArrayList<>(); + indexes.add(""); + indexes.add(""); + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + conf.setIndexes(indexes); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, conf::build); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indexes", "[, ]"), e.getMessage()); + } + + public void testCheckValid_GivenNegativeQueryDelay() throws IOException { + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setQueryDelay(-10L)); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "query_delay", -10L), e.getMessage()); + } + + public void testCheckValid_GivenZeroFrequency() throws IOException { + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(0L)); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "frequency", 0L), e.getMessage()); + } + + public void testCheckValid_GivenNegativeFrequency() throws IOException { + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(-600L)); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "frequency", -600L), e.getMessage()); + } + + public void testCheckValid_GivenNegativeScrollSize() throws IOException { + DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setScrollSize(-1000)); + assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "scroll_size", -1000L), e.getMessage()); + } + + public void testBuild_GivenScriptFieldsAndAggregations() { + DatafeedConfig.Builder datafeed = new DatafeedConfig.Builder("datafeed1", "job1"); + datafeed.setIndexes(Arrays.asList("my_index")); + datafeed.setTypes(Arrays.asList("my_type")); + datafeed.setScriptFields(Arrays.asList(new SearchSourceBuilder.ScriptField(randomAsciiOfLength(10), + new Script(randomAsciiOfLength(10)), randomBoolean()))); + datafeed.setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("foo"))); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> datafeed.build()); + + assertThat(e.getMessage(), equalTo("script_fields cannot be used in combination with aggregations")); + } + + public void testDomainSplitInjection() { + DatafeedConfig.Builder datafeed = new DatafeedConfig.Builder("datafeed1", "job1"); + datafeed.setIndexes(Arrays.asList("my_index")); + datafeed.setTypes(Arrays.asList("my_type")); + + SearchSourceBuilder.ScriptField withoutSplit = new SearchSourceBuilder.ScriptField( + "script1", new Script("return 1+1;"), false); + SearchSourceBuilder.ScriptField withSplit = new SearchSourceBuilder.ScriptField( + "script2", new Script("return domainSplit('foo.com', params);"), false); + datafeed.setScriptFields(Arrays.asList(withoutSplit, withSplit)); + + DatafeedConfig config = datafeed.build(); + List scriptFields = config.getScriptFields(); + + assertThat(scriptFields.size(), equalTo(2)); + assertThat(scriptFields.get(0).fieldName(), equalTo("script1")); + assertThat(scriptFields.get(0).script().getIdOrCode(), equalTo("return 1+1;")); + assertFalse(scriptFields.get(0).script().getParams().containsKey("exact")); + + assertThat(scriptFields.get(1).fieldName(), equalTo("script2")); + assertThat(scriptFields.get(1).script().getIdOrCode(), containsString("List domainSplit(String host, Map params)")); + assertTrue(scriptFields.get(1).script().getParams().containsKey("exact")); + } + + public static String randomValidDatafeedId() { + CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray()); + return generator.ofCodePointsLength(random(), 10, 10); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java new file mode 100644 index 00000000000..bc97f169166 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobRunnerTests.java @@ -0,0 +1,312 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.mock.orig.Mockito; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.FlushJobAction; +import org.elasticsearch.xpack.ml.action.PostDataAction; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationDataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.chunked.ChunkedDataExtractorFactory; +import org.elasticsearch.xpack.ml.datafeed.extractor.scroll.ScrollDataExtractorFactory; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Arrays; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DatafeedJobRunnerTests extends ESTestCase { + + private Client client; + private ActionFuture jobDataFuture; + private ActionFuture flushJobFuture; + private ClusterService clusterService; + private ThreadPool threadPool; + private DataExtractorFactory dataExtractorFactory; + private DatafeedJobRunner datafeedJobRunner; + private long currentTime = 120000; + + @Before + @SuppressWarnings("unchecked") + public void setUpTests() { + client = mock(Client.class); + jobDataFuture = mock(ActionFuture.class); + flushJobFuture = mock(ActionFuture.class); + clusterService = mock(ClusterService.class); + + JobProvider jobProvider = mock(JobProvider.class); + Mockito.doAnswer(invocationOnMock -> { + String jobId = (String) invocationOnMock.getArguments()[0]; + @SuppressWarnings("unchecked") + Consumer handler = (Consumer) invocationOnMock.getArguments()[1]; + handler.accept(new DataCounts(jobId)); + return null; + }).when(jobProvider).dataCounts(any(), any(), any()); + dataExtractorFactory = mock(DataExtractorFactory.class); + Auditor auditor = mock(Auditor.class); + threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).submit(any(Runnable.class)); + when(threadPool.executor(MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME)).thenReturn(executorService); + when(client.execute(same(PostDataAction.INSTANCE), any())).thenReturn(jobDataFuture); + when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture); + + datafeedJobRunner = new DatafeedJobRunner(threadPool, client, clusterService, jobProvider, () -> currentTime) { + @Override + DataExtractorFactory createDataExtractorFactory(DatafeedConfig datafeedConfig, Job job) { + return dataExtractorFactory; + } + }; + + when(jobProvider.audit(anyString())).thenReturn(auditor); + doAnswer(invocationOnMock -> { + @SuppressWarnings("rawtypes") + Consumer consumer = (Consumer) invocationOnMock.getArguments()[3]; + consumer.accept(new ResourceNotFoundException("dummy")); + return null; + }).when(jobProvider).buckets(any(), any(), any(), any()); + } + + public void testStart_GivenNewlyCreatedJobLoopBack() throws Exception { + Job.Builder jobBuilder = createDatafeedJob(); + DatafeedConfig datafeedConfig = createDatafeedConfig("datafeed1", "foo").build(); + DataCounts dataCounts = new DataCounts("foo", 1, 0, 0, 0, 0, 0, 0, new Date(0), new Date(0)); + Job job = jobBuilder.build(); + MlMetadata mlMetadata = new MlMetadata.Builder() + .putJob(job, false) + .putDatafeed(datafeedConfig) + .updateState("foo", JobState.OPENED, null) + .build(); + when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)) + .build()); + + DataExtractor dataExtractor = mock(DataExtractor.class); + when(dataExtractorFactory.newExtractor(0L, 60000L)).thenReturn(dataExtractor); + when(dataExtractor.hasNext()).thenReturn(true).thenReturn(false); + InputStream in = new ByteArrayInputStream("".getBytes(Charset.forName("utf-8"))); + when(dataExtractor.next()).thenReturn(Optional.of(in)); + when(jobDataFuture.get()).thenReturn(new PostDataAction.Response(dataCounts)); + Consumer handler = mockConsumer(); + StartDatafeedAction.DatafeedTask task = mock(StartDatafeedAction.DatafeedTask.class); + datafeedJobRunner.run("datafeed1", 0L, 60000L, task, handler); + + verify(threadPool, times(1)).executor(MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME); + verify(threadPool, never()).schedule(any(), any(), any()); + verify(client).execute(same(PostDataAction.INSTANCE), eq(createExpectedPostDataRequest(job))); + verify(client).execute(same(FlushJobAction.INSTANCE), any()); + } + + private static PostDataAction.Request createExpectedPostDataRequest(Job job) { + DataDescription.Builder expectedDataDescription = new DataDescription.Builder(); + expectedDataDescription.setTimeFormat("epoch_ms"); + expectedDataDescription.setFormat(DataDescription.DataFormat.JSON); + PostDataAction.Request expectedPostDataRequest = new PostDataAction.Request(job.getId()); + expectedPostDataRequest.setDataDescription(expectedDataDescription.build()); + return expectedPostDataRequest; + } + + public void testStart_extractionProblem() throws Exception { + Job.Builder jobBuilder = createDatafeedJob(); + DatafeedConfig datafeedConfig = createDatafeedConfig("datafeed1", "foo").build(); + DataCounts dataCounts = new DataCounts("foo", 1, 0, 0, 0, 0, 0, 0, new Date(0), new Date(0)); + Job job = jobBuilder.build(); + MlMetadata mlMetadata = new MlMetadata.Builder() + .putJob(job, false) + .putDatafeed(datafeedConfig) + .updateState("foo", JobState.OPENED, null) + .build(); + when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)) + .build()); + + DataExtractor dataExtractor = mock(DataExtractor.class); + when(dataExtractorFactory.newExtractor(0L, 60000L)).thenReturn(dataExtractor); + when(dataExtractor.hasNext()).thenReturn(true).thenReturn(false); + when(dataExtractor.next()).thenThrow(new RuntimeException("dummy")); + when(jobDataFuture.get()).thenReturn(new PostDataAction.Response(dataCounts)); + Consumer handler = mockConsumer(); + StartDatafeedAction.DatafeedTask task = mock(StartDatafeedAction.DatafeedTask.class); + datafeedJobRunner.run("datafeed1", 0L, 60000L, task, handler); + + verify(threadPool, times(1)).executor(MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME); + verify(threadPool, never()).schedule(any(), any(), any()); + verify(client, never()).execute(same(PostDataAction.INSTANCE), eq(new PostDataAction.Request("foo"))); + verify(client, never()).execute(same(FlushJobAction.INSTANCE), any()); + } + + public void testStart_GivenNewlyCreatedJobLoopBackAndRealtime() throws Exception { + Job.Builder jobBuilder = createDatafeedJob(); + DatafeedConfig datafeedConfig = createDatafeedConfig("datafeed1", "foo").build(); + DataCounts dataCounts = new DataCounts("foo", 1, 0, 0, 0, 0, 0, 0, new Date(0), new Date(0)); + Job job = jobBuilder.build(); + MlMetadata mlMetadata = new MlMetadata.Builder() + .putJob(job, false) + .putDatafeed(datafeedConfig) + .updateState("foo", JobState.OPENED, null) + .build(); + when(clusterService.state()).thenReturn(ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)) + .build()); + + DataExtractor dataExtractor = mock(DataExtractor.class); + when(dataExtractorFactory.newExtractor(0L, 60000L)).thenReturn(dataExtractor); + when(dataExtractor.hasNext()).thenReturn(true).thenReturn(false); + InputStream in = new ByteArrayInputStream("".getBytes(Charset.forName("utf-8"))); + when(dataExtractor.next()).thenReturn(Optional.of(in)); + when(jobDataFuture.get()).thenReturn(new PostDataAction.Response(dataCounts)); + Consumer handler = mockConsumer(); + boolean cancelled = randomBoolean(); + StartDatafeedAction.DatafeedTask task = new StartDatafeedAction.DatafeedTask(1, "type", "action", null, "datafeed1"); + datafeedJobRunner.run("datafeed1", 0L, null, task, handler); + + verify(threadPool, times(1)).executor(MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME); + if (cancelled) { + task.stop(); + verify(handler).accept(null); + } else { + verify(client).execute(same(PostDataAction.INSTANCE), eq(createExpectedPostDataRequest(job))); + verify(client).execute(same(FlushJobAction.INSTANCE), any()); + verify(threadPool, times(1)).schedule(eq(new TimeValue(480100)), eq(MlPlugin.DATAFEED_RUNNER_THREAD_POOL_NAME), any()); + } + } + + public void testCreateDataExtractorFactoryGivenDefaultScroll() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setTimeField("time"); + Job.Builder jobBuilder = createDatafeedJob(); + jobBuilder.setDataDescription(dataDescription); + DatafeedConfig datafeedConfig = createDatafeedConfig("datafeed1", "foo").build(); + DatafeedJobRunner runner = new DatafeedJobRunner(threadPool, client, clusterService, mock(JobProvider.class), () -> currentTime); + + DataExtractorFactory dataExtractorFactory = runner.createDataExtractorFactory(datafeedConfig, jobBuilder.build()); + + assertThat(dataExtractorFactory, instanceOf(ChunkedDataExtractorFactory.class)); + } + + public void testCreateDataExtractorFactoryGivenScrollWithAutoChunk() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setTimeField("time"); + Job.Builder jobBuilder = createDatafeedJob(); + jobBuilder.setDataDescription(dataDescription); + DatafeedConfig.Builder datafeedConfig = createDatafeedConfig("datafeed1", "foo"); + datafeedConfig.setChunkingConfig(ChunkingConfig.newAuto()); + DatafeedJobRunner runner = new DatafeedJobRunner(threadPool, client, clusterService, mock(JobProvider.class), () -> currentTime); + + DataExtractorFactory dataExtractorFactory = runner.createDataExtractorFactory(datafeedConfig.build(), jobBuilder.build()); + + assertThat(dataExtractorFactory, instanceOf(ChunkedDataExtractorFactory.class)); + } + + public void testCreateDataExtractorFactoryGivenScrollWithOffChunk() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setTimeField("time"); + Job.Builder jobBuilder = createDatafeedJob(); + jobBuilder.setDataDescription(dataDescription); + DatafeedConfig.Builder datafeedConfig = createDatafeedConfig("datafeed1", "foo"); + datafeedConfig.setChunkingConfig(ChunkingConfig.newOff()); + DatafeedJobRunner runner = new DatafeedJobRunner(threadPool, client, clusterService, mock(JobProvider.class), () -> currentTime); + + DataExtractorFactory dataExtractorFactory = runner.createDataExtractorFactory(datafeedConfig.build(), jobBuilder.build()); + + assertThat(dataExtractorFactory, instanceOf(ScrollDataExtractorFactory.class)); + } + + public void testCreateDataExtractorFactoryGivenDefaultAggregation() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setTimeField("time"); + Job.Builder jobBuilder = createDatafeedJob(); + jobBuilder.setDataDescription(dataDescription); + DatafeedConfig.Builder datafeedConfig = createDatafeedConfig("datafeed1", "foo"); + datafeedConfig.setAggregations(AggregatorFactories.builder()); + DatafeedJobRunner runner = new DatafeedJobRunner(threadPool, client, clusterService, mock(JobProvider.class), () -> currentTime); + + DataExtractorFactory dataExtractorFactory = runner.createDataExtractorFactory(datafeedConfig.build(), jobBuilder.build()); + + assertThat(dataExtractorFactory, instanceOf(AggregationDataExtractorFactory.class)); + } + + public void testCreateDataExtractorFactoryGivenDefaultAggregationWithAutoChunk() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setTimeField("time"); + Job.Builder jobBuilder = createDatafeedJob(); + jobBuilder.setDataDescription(dataDescription); + DatafeedConfig.Builder datafeedConfig = createDatafeedConfig("datafeed1", "foo"); + datafeedConfig.setAggregations(AggregatorFactories.builder()); + datafeedConfig.setChunkingConfig(ChunkingConfig.newAuto()); + DatafeedJobRunner runner = new DatafeedJobRunner(threadPool, client, clusterService, mock(JobProvider.class), () -> currentTime); + + DataExtractorFactory dataExtractorFactory = runner.createDataExtractorFactory(datafeedConfig.build(), jobBuilder.build()); + + assertThat(dataExtractorFactory, instanceOf(ChunkedDataExtractorFactory.class)); + } + + public static DatafeedConfig.Builder createDatafeedConfig(String datafeedId, String jobId) { + DatafeedConfig.Builder datafeedConfig = new DatafeedConfig.Builder(datafeedId, jobId); + datafeedConfig.setIndexes(Arrays.asList("myIndex")); + datafeedConfig.setTypes(Arrays.asList("myType")); + return datafeedConfig; + } + + public static Job.Builder createDatafeedJob() { + AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Arrays.asList(new Detector.Builder("metric", "field").build())); + acBuilder.setBucketSpan(3600L); + acBuilder.setDetectors(Arrays.asList(new Detector.Builder("metric", "field").build())); + + Job.Builder builder = new Job.Builder("foo"); + builder.setAnalysisConfig(acBuilder); + return builder; + } + + @SuppressWarnings("unchecked") + private Consumer mockConsumer() { + return mock(Consumer.class); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java new file mode 100644 index 00000000000..cc7e26299be --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobTests.java @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.action.FlushJobAction; +import org.elasticsearch.xpack.ml.action.PostDataAction; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.junit.Before; +import org.mockito.ArgumentCaptor; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.Optional; +import java.util.function.Supplier; + +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.eq; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class DatafeedJobTests extends ESTestCase { + + private Auditor auditor; + private DataExtractorFactory dataExtractorFactory; + private DataExtractor dataExtractor; + private Client client; + private DataDescription.Builder dataDescription; + private ActionFuture flushJobFuture; + + private long currentTime; + + @Before + @SuppressWarnings("unchecked") + public void setup() throws Exception { + auditor = mock(Auditor.class); + dataExtractorFactory = mock(DataExtractorFactory.class); + dataExtractor = mock(DataExtractor.class); + when(dataExtractorFactory.newExtractor(anyLong(), anyLong())).thenReturn(dataExtractor); + client = mock(Client.class); + dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.JSON); + ActionFuture jobDataFuture = mock(ActionFuture.class); + flushJobFuture = mock(ActionFuture.class); + currentTime = 0; + + when(dataExtractor.hasNext()).thenReturn(true).thenReturn(false); + InputStream inputStream = new ByteArrayInputStream("content".getBytes(StandardCharsets.UTF_8)); + when(dataExtractor.next()).thenReturn(Optional.of(inputStream)); + DataCounts dataCounts = new DataCounts("_job_id", 1, 0, 0, 0, 0, 0, 0, new Date(0), new Date(0)); + + PostDataAction.Request expectedRequest = new PostDataAction.Request("_job_id"); + expectedRequest.setDataDescription(dataDescription.build()); + when(client.execute(same(PostDataAction.INSTANCE), eq(expectedRequest))).thenReturn(jobDataFuture); + when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture); + when(jobDataFuture.get()).thenReturn(new PostDataAction.Response(dataCounts)); + } + + public void testLookBackRunWithEndTime() throws Exception { + DatafeedJob datafeedJob = createDatafeedJob(1000, 500, -1, -1); + assertNull(datafeedJob.runLookBack(0L, 1000L)); + + verify(dataExtractorFactory).newExtractor(0L, 1000L); + FlushJobAction.Request flushRequest = new FlushJobAction.Request("_job_id"); + flushRequest.setCalcInterim(true); + verify(client).execute(same(FlushJobAction.INSTANCE), eq(flushRequest)); + } + + public void testLookBackRunWithNoEndTime() throws Exception { + currentTime = 2000L; + long frequencyMs = 1000; + long queryDelayMs = 500; + DatafeedJob datafeedJob = createDatafeedJob(frequencyMs, queryDelayMs, -1, -1); + long next = datafeedJob.runLookBack(0L, null); + assertEquals(2000 + frequencyMs + 100, next); + + verify(dataExtractorFactory).newExtractor(0L, 1500L); + FlushJobAction.Request flushRequest = new FlushJobAction.Request("_job_id"); + flushRequest.setCalcInterim(true); + verify(client).execute(same(FlushJobAction.INSTANCE), eq(flushRequest)); + } + + public void testLookBackRunWithOverrideStartTime() throws Exception { + currentTime = 10000L; + long latestFinalBucketEndTimeMs = -1; + long latestRecordTimeMs = -1; + if (randomBoolean()) { + latestFinalBucketEndTimeMs = 5000; + } else { + latestRecordTimeMs = 5000; + } + + long frequencyMs = 1000; + long queryDelayMs = 500; + DatafeedJob datafeedJob = createDatafeedJob(frequencyMs, queryDelayMs, latestFinalBucketEndTimeMs, latestRecordTimeMs); + long next = datafeedJob.runLookBack(0L, null); + assertEquals(10000 + frequencyMs + 100, next); + + verify(dataExtractorFactory).newExtractor(5000 + 1L, currentTime - queryDelayMs); + FlushJobAction.Request flushRequest = new FlushJobAction.Request("_job_id"); + flushRequest.setCalcInterim(true); + verify(client).execute(same(FlushJobAction.INSTANCE), eq(flushRequest)); + } + + public void testRealtimeRun() throws Exception { + currentTime = 60000L; + long frequencyMs = 100; + long queryDelayMs = 1000; + DatafeedJob datafeedJob = createDatafeedJob(frequencyMs, queryDelayMs, 1000, -1); + long next = datafeedJob.runRealtime(); + assertEquals(currentTime + frequencyMs + 100, next); + + verify(dataExtractorFactory).newExtractor(1000L + 1L, currentTime - queryDelayMs); + FlushJobAction.Request flushRequest = new FlushJobAction.Request("_job_id"); + flushRequest.setCalcInterim(true); + flushRequest.setAdvanceTime("1000"); + verify(client).execute(same(FlushJobAction.INSTANCE), eq(flushRequest)); + } + + public void testEmptyDataCount() throws Exception { + when(dataExtractor.hasNext()).thenReturn(false); + + DatafeedJob datafeedJob = createDatafeedJob(1000, 500, -1, -1); + expectThrows(DatafeedJob.EmptyDataCountException.class, () -> datafeedJob.runLookBack(0L, 1000L)); + } + + public void testExtractionProblem() throws Exception { + when(dataExtractor.hasNext()).thenReturn(true); + when(dataExtractor.next()).thenThrow(new IOException()); + + DatafeedJob datafeedJob = createDatafeedJob(1000, 500, -1, -1); + expectThrows(DatafeedJob.ExtractionProblemException.class, () -> datafeedJob.runLookBack(0L, 1000L)); + + currentTime = 3001; + expectThrows(DatafeedJob.ExtractionProblemException.class, datafeedJob::runRealtime); + + ArgumentCaptor startTimeCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor endTimeCaptor = ArgumentCaptor.forClass(Long.class); + verify(dataExtractorFactory, times(2)).newExtractor(startTimeCaptor.capture(), endTimeCaptor.capture()); + assertEquals(0L, startTimeCaptor.getAllValues().get(0).longValue()); + assertEquals(1000L, startTimeCaptor.getAllValues().get(1).longValue()); + assertEquals(1000L, endTimeCaptor.getAllValues().get(0).longValue()); + assertEquals(2000L, endTimeCaptor.getAllValues().get(1).longValue()); + } + + public void testAnalysisProblem() throws Exception { + client = mock(Client.class); + when(client.execute(same(FlushJobAction.INSTANCE), any())).thenReturn(flushJobFuture); + when(client.execute(same(PostDataAction.INSTANCE), eq(new PostDataAction.Request("_job_id")))).thenThrow(new RuntimeException()); + + DatafeedJob datafeedJob = createDatafeedJob(1000, 500, -1, -1); + expectThrows(DatafeedJob.AnalysisProblemException.class, () -> datafeedJob.runLookBack(0L, 1000L)); + + currentTime = 3001; + expectThrows(DatafeedJob.EmptyDataCountException.class, datafeedJob::runRealtime); + + ArgumentCaptor startTimeCaptor = ArgumentCaptor.forClass(Long.class); + ArgumentCaptor endTimeCaptor = ArgumentCaptor.forClass(Long.class); + verify(dataExtractorFactory, times(2)).newExtractor(startTimeCaptor.capture(), endTimeCaptor.capture()); + assertEquals(0L, startTimeCaptor.getAllValues().get(0).longValue()); + assertEquals(1000L, startTimeCaptor.getAllValues().get(1).longValue()); + assertEquals(1000L, endTimeCaptor.getAllValues().get(0).longValue()); + assertEquals(2000L, endTimeCaptor.getAllValues().get(1).longValue()); + verify(client, times(0)).execute(same(FlushJobAction.INSTANCE), any()); + } + + private DatafeedJob createDatafeedJob(long frequencyMs, long queryDelayMs, long latestFinalBucketEndTimeMs, + long latestRecordTimeMs) { + Supplier currentTimeSupplier = () -> currentTime; + return new DatafeedJob("_job_id", dataDescription.build(), frequencyMs, queryDelayMs, dataExtractorFactory, client, auditor, + currentTimeSupplier, latestFinalBucketEndTimeMs, latestRecordTimeMs); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java new file mode 100644 index 00000000000..9526808b700 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedJobValidatorTests.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.messages.Messages; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; + +public class DatafeedJobValidatorTests extends ESTestCase { + + public void testValidate_GivenNonZeroLatency() { + String errorMessage = Messages.getMessage(Messages.DATAFEED_DOES_NOT_SUPPORT_JOB_WITH_LATENCY); + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setLatency(3600L); + builder.setAnalysisConfig(ac); + Job job = builder.build(); + DatafeedConfig datafeedConfig = createValidDatafeedConfig().build(); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, + () -> DatafeedJobValidator.validate(datafeedConfig, job)); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenZeroLatency() { + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setLatency(0L); + builder.setAnalysisConfig(ac); + Job job = builder.build(); + DatafeedConfig datafeedConfig = createValidDatafeedConfig().build(); + + DatafeedJobValidator.validate(datafeedConfig, job); + } + + public void testVerify_GivenNoLatency() { + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBatchSpan(1800L); + ac.setBucketSpan(100L); + builder.setAnalysisConfig(ac); + Job job = builder.build(); + DatafeedConfig datafeedConfig = createValidDatafeedConfig().build(); + + DatafeedJobValidator.validate(datafeedConfig, job); + } + + public void testVerify_GivenAggsAndCorrectSummaryCountField() throws IOException { + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setSummaryCountFieldName("doc_count"); + builder.setAnalysisConfig(ac); + Job job = builder.build(); + DatafeedConfig datafeedConfig = createValidDatafeedConfigWithAggs().build(); + + DatafeedJobValidator.validate(datafeedConfig, job); + } + + public void testVerify_GivenAggsAndNoSummaryCountField() throws IOException { + String errorMessage = Messages.getMessage(Messages.DATAFEED_AGGREGATIONS_REQUIRES_JOB_WITH_SUMMARY_COUNT_FIELD, + DatafeedConfig.DOC_COUNT); + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + builder.setAnalysisConfig(ac); + Job job = builder.build(); + DatafeedConfig datafeedConfig = createValidDatafeedConfigWithAggs().build(); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, + () -> DatafeedJobValidator.validate(datafeedConfig, job)); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenAggsAndWrongSummaryCountField() throws IOException { + String errorMessage = Messages.getMessage( + Messages.DATAFEED_AGGREGATIONS_REQUIRES_JOB_WITH_SUMMARY_COUNT_FIELD, DatafeedConfig.DOC_COUNT); + Job.Builder builder = buildJobBuilder("foo"); + AnalysisConfig.Builder ac = createAnalysisConfig(); + ac.setBucketSpan(1800L); + ac.setSummaryCountFieldName("wrong"); + builder.setAnalysisConfig(ac); + Job job = builder.build(); + DatafeedConfig datafeedConfig = createValidDatafeedConfigWithAggs().build(); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, + () -> DatafeedJobValidator.validate(datafeedConfig, job)); + + assertEquals(errorMessage, e.getMessage()); + } + + public static Job.Builder buildJobBuilder(String id) { + Job.Builder builder = new Job.Builder(id); + builder.setCreateTime(new Date()); + AnalysisConfig.Builder ac = createAnalysisConfig(); + builder.setAnalysisConfig(ac); + return builder; + } + + public static AnalysisConfig.Builder createAnalysisConfig() { + Detector.Builder d1 = new Detector.Builder("info_content", "domain"); + d1.setOverFieldName("client"); + Detector.Builder d2 = new Detector.Builder("min", "field"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build())); + return ac; + } + + private static DatafeedConfig.Builder createValidDatafeedConfigWithAggs() throws IOException { + DatafeedConfig.Builder datafeedConfig = createValidDatafeedConfig(); + datafeedConfig.setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("foo"))); + return datafeedConfig; + } + + private static DatafeedConfig.Builder createValidDatafeedConfig() { + DatafeedConfig.Builder builder = new DatafeedConfig.Builder("my-datafeed", "my-job"); + builder.setIndexes(Collections.singletonList("myIndex")); + builder.setTypes(Collections.singletonList("myType")); + return builder; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedStateTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedStateTests.java new file mode 100644 index 00000000000..ca538ba2d57 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedStateTests.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.test.ESTestCase; + +public class DatafeedStateTests extends ESTestCase { + + public void testFromString() { + assertEquals(DatafeedState.fromString("started"), DatafeedState.STARTED); + assertEquals(DatafeedState.fromString("stopped"), DatafeedState.STOPPED); + } + + public void testToString() { + assertEquals("started", DatafeedState.STARTED.toString()); + assertEquals("stopped", DatafeedState.STOPPED.toString()); + } + + public void testValidOrdinals() { + assertEquals(0, DatafeedState.STARTED.ordinal()); + assertEquals(1, DatafeedState.STOPPED.ordinal()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java new file mode 100644 index 00000000000..c91c259b1fa --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/ProblemTrackerTests.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.junit.Before; +import org.mockito.Mockito; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class ProblemTrackerTests extends ESTestCase { + + private Auditor auditor; + + private ProblemTracker problemTracker; + + @Before + public void setUpTests() { + auditor = mock(Auditor.class); + problemTracker = new ProblemTracker(() -> auditor); + } + + public void testReportExtractionProblem() { + problemTracker.reportExtractionProblem("foo"); + + verify(auditor).error("Datafeed is encountering errors extracting data: foo"); + assertTrue(problemTracker.hasProblems()); + } + + public void testReportAnalysisProblem() { + problemTracker.reportAnalysisProblem("foo"); + + verify(auditor).error("Datafeed is encountering errors submitting data for analysis: foo"); + assertTrue(problemTracker.hasProblems()); + } + + public void testReportProblem_GivenSameProblemTwice() { + problemTracker.reportExtractionProblem("foo"); + problemTracker.reportAnalysisProblem("foo"); + + verify(auditor, times(1)).error("Datafeed is encountering errors extracting data: foo"); + assertTrue(problemTracker.hasProblems()); + } + + public void testReportProblem_GivenSameProblemAfterFinishReport() { + problemTracker.reportExtractionProblem("foo"); + problemTracker.finishReport(); + problemTracker.reportExtractionProblem("foo"); + + verify(auditor, times(1)).error("Datafeed is encountering errors extracting data: foo"); + assertTrue(problemTracker.hasProblems()); + } + + public void testUpdateEmptyDataCount_GivenEmptyNineTimes() { + for (int i = 0; i < 9; i++) { + problemTracker.updateEmptyDataCount(true); + } + + Mockito.verifyNoMoreInteractions(auditor); + } + + public void testUpdateEmptyDataCount_GivenEmptyTenTimes() { + for (int i = 0; i < 10; i++) { + problemTracker.updateEmptyDataCount(true); + } + + verify(auditor).warning("Datafeed has been retrieving no data for a while"); + } + + public void testUpdateEmptyDataCount_GivenEmptyElevenTimes() { + for (int i = 0; i < 11; i++) { + problemTracker.updateEmptyDataCount(true); + } + + verify(auditor, times(1)).warning("Datafeed has been retrieving no data for a while"); + } + + public void testUpdateEmptyDataCount_GivenNonEmptyAfterNineEmpty() { + for (int i = 0; i < 9; i++) { + problemTracker.updateEmptyDataCount(true); + } + problemTracker.updateEmptyDataCount(false); + + Mockito.verifyNoMoreInteractions(auditor); + } + + public void testUpdateEmptyDataCount_GivenNonEmptyAfterTenEmpty() { + for (int i = 0; i < 10; i++) { + problemTracker.updateEmptyDataCount(true); + } + problemTracker.updateEmptyDataCount(false); + + verify(auditor).warning("Datafeed has been retrieving no data for a while"); + verify(auditor).info("Datafeed has started retrieving data again"); + } + + public void testFinishReport_GivenNoProblems() { + problemTracker.finishReport(); + + assertFalse(problemTracker.hasProblems()); + Mockito.verifyNoMoreInteractions(auditor); + } + + public void testFinishReport_GivenRecovery() { + problemTracker.reportExtractionProblem("bar"); + problemTracker.finishReport(); + problemTracker.finishReport(); + + verify(auditor).error("Datafeed is encountering errors extracting data: bar"); + verify(auditor).info("Datafeed has recovered data extraction and analysis"); + assertFalse(problemTracker.hasProblems()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java new file mode 100644 index 00000000000..64054b2c291 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java @@ -0,0 +1,204 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.aggregations.AggregationBuilders; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.datafeed.extractor.scroll.ScrollDataExtractorTests; +import org.junit.Before; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.Term; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createHistogramBucket; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createTerms; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.stringContainsInOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AggregationDataExtractorTests extends ESTestCase { + + private Client client; + private List capturedSearchRequests; + private String jobId; + private String timeField; + private List types; + private List indexes; + private QueryBuilder query; + private AggregatorFactories.Builder aggs; + + private class TestDataExtractor extends AggregationDataExtractor { + + private SearchResponse nextResponse; + + TestDataExtractor(long start, long end) { + super(client, createContext(start, end)); + } + + @Override + protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { + capturedSearchRequests.add(searchRequestBuilder); + return nextResponse; + } + + void setNextResponse(SearchResponse searchResponse) { + nextResponse = searchResponse; + } + } + + @Before + public void setUpTests() { + client = mock(Client.class); + capturedSearchRequests = new ArrayList<>(); + jobId = "test-job"; + timeField = "time"; + indexes = Arrays.asList("index-1", "index-2"); + types = Arrays.asList("type-1", "type-2"); + query = QueryBuilders.matchAllQuery(); + aggs = new AggregatorFactories.Builder() + .addAggregator(AggregationBuilders.histogram("time").field("time").subAggregation( + AggregationBuilders.terms("airline").field("airline").subAggregation( + AggregationBuilders.avg("responsetime").field("responsetime")))); + } + + public void testExtraction() throws IOException { + List histogramBuckets = Arrays.asList( + createHistogramBucket(1000L, 3, Arrays.asList( + createTerms("airline", new Term("a", 1, "responsetime", 11.0), new Term("b", 2, "responsetime", 12.0)))), + createHistogramBucket(2000L, 0, Arrays.asList()), + createHistogramBucket(3000L, 7, Arrays.asList( + createTerms("airline", new Term("c", 4, "responsetime", 31.0), new Term("b", 3, "responsetime", 32.0)))) + ); + + TestDataExtractor extractor = new TestDataExtractor(1000L, 4000L); + + SearchResponse response = createSearchResponse("time", histogramBuckets); + extractor.setNextResponse(response); + + assertThat(extractor.hasNext(), is(true)); + Optional stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + String expectedStream = "{\"time\":1000,\"airline\":\"a\",\"responsetime\":11.0,\"doc_count\":1} " + + "{\"time\":1000,\"airline\":\"b\",\"responsetime\":12.0,\"doc_count\":2} " + + "{\"time\":3000,\"airline\":\"c\",\"responsetime\":31.0,\"doc_count\":4} " + + "{\"time\":3000,\"airline\":\"b\",\"responsetime\":32.0,\"doc_count\":3}"; + assertThat(asString(stream.get()), equalTo(expectedStream)); + assertThat(extractor.hasNext(), is(false)); + assertThat(capturedSearchRequests.size(), equalTo(1)); + + String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"size\":0")); + assertThat(searchRequest, containsString("\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," + + "{\"range\":{\"time\":{\"from\":1000,\"to\":4000,\"include_lower\":true,\"include_upper\":false," + + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]")); + assertThat(searchRequest, + stringContainsInOrder(Arrays.asList("aggregations", "histogram", "time", "terms", "airline", "avg", "responsetime"))); + } + + public void testExtractionGivenCancelBeforeNext() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 4000L); + SearchResponse response = createSearchResponse("time", Collections.emptyList()); + extractor.setNextResponse(response); + + extractor.cancel(); + assertThat(extractor.hasNext(), is(false)); + } + + public void testExtractionGivenSearchResponseHasError() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + extractor.setNextResponse(createErrorResponse()); + + assertThat(extractor.hasNext(), is(true)); + expectThrows(IOException.class, () -> extractor.next()); + } + + public void testExtractionGivenSearchResponseHasShardFailures() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + extractor.setNextResponse(createResponseWithShardFailures()); + + assertThat(extractor.hasNext(), is(true)); + IOException e = expectThrows(IOException.class, () -> extractor.next()); + } + + public void testExtractionGivenInitSearchResponseEncounteredUnavailableShards() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + extractor.setNextResponse(createResponseWithUnavailableShards(2)); + + assertThat(extractor.hasNext(), is(true)); + IOException e = expectThrows(IOException.class, () -> extractor.next()); + assertThat(e.getMessage(), equalTo("[" + jobId + "] Search request encountered [2] unavailable shards")); + } + + private AggregationDataExtractorContext createContext(long start, long end) { + return new AggregationDataExtractorContext(jobId, timeField, indexes, types, query, aggs, start, end); + } + + private SearchResponse createSearchResponse(String histogramName, List histogramBuckets) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getScrollId()).thenReturn(randomAsciiOfLength(1000)); + + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn(histogramName); + when(histogram.getBuckets()).thenReturn(histogramBuckets); + + Aggregations searchAggs = mock(Aggregations.class); + when(searchAggs.asList()).thenReturn(Arrays.asList(histogram)); + when(searchResponse.getAggregations()).thenReturn(searchAggs); + return searchResponse; + } + + private SearchResponse createErrorResponse() { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.INTERNAL_SERVER_ERROR); + return searchResponse; + } + + private SearchResponse createResponseWithShardFailures() { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getShardFailures()).thenReturn( + new ShardSearchFailure[] { new ShardSearchFailure(new RuntimeException("shard failed"))}); + return searchResponse; + } + + private SearchResponse createResponseWithUnavailableShards(int unavailableShards) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getSuccessfulShards()).thenReturn(3); + when(searchResponse.getTotalShards()).thenReturn(3 + unavailableShards); + return searchResponse; + } + + private static String asString(InputStream inputStream) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java new file mode 100644 index 00000000000..a3848a18132 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationTestUtils.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; +import org.joda.time.DateTime; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public final class AggregationTestUtils { + + private AggregationTestUtils() {} + + static Histogram.Bucket createHistogramBucket(long timestamp, long docCount, List subAggregations) { + Histogram.Bucket bucket = createHistogramBucket(timestamp, docCount); + Aggregations aggs = createAggs(subAggregations); + when(bucket.getAggregations()).thenReturn(aggs); + return bucket; + } + + static Aggregations createAggs(List aggsList) { + Aggregations aggs = mock(Aggregations.class); + when(aggs.asList()).thenReturn(aggsList); + return aggs; + } + + static Histogram.Bucket createHistogramBucket(long timestamp, long docCount) { + Histogram.Bucket bucket = mock(Histogram.Bucket.class); + when(bucket.getKey()).thenReturn(timestamp); + when(bucket.getDocCount()).thenReturn(docCount); + return bucket; + } + + static Histogram.Bucket createDateHistogramBucket(DateTime timestamp, long docCount) { + Histogram.Bucket bucket = mock(Histogram.Bucket.class); + when(bucket.getKey()).thenReturn(timestamp); + when(bucket.getDocCount()).thenReturn(docCount); + return bucket; + } + + static NumericMetricsAggregation.SingleValue createSingleValue(String name, double value) { + NumericMetricsAggregation.SingleValue singleValue = mock(NumericMetricsAggregation.SingleValue.class); + when(singleValue.getName()).thenReturn(name); + when(singleValue.value()).thenReturn(value); + return singleValue; + } + + static Terms createTerms(String name, Term... terms) { + Terms termsAgg = mock(Terms.class); + when(termsAgg.getName()).thenReturn(name); + List buckets = new ArrayList<>(); + for (Term term: terms) { + StringTerms.Bucket bucket = mock(StringTerms.Bucket.class); + when(bucket.getKey()).thenReturn(term.key); + when(bucket.getDocCount()).thenReturn(term.count); + if (term.value != null) { + NumericMetricsAggregation.SingleValue termValue = createSingleValue(term.valueName, term.value); + Aggregations aggs = createAggs(Arrays.asList(termValue)); + when(bucket.getAggregations()).thenReturn(aggs); + } + buckets.add(bucket); + } + when(termsAgg.getBuckets()).thenReturn(buckets); + return termsAgg; + } + + static class Term { + String key; + long count; + String valueName; + Double value; + + Term(String key, long count) { + this(key, count, null, null); + } + + Term(String key, long count, String valueName, Double value) { + this.key = key; + this.count = count; + this.valueName = valueName; + this.value = value; + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java new file mode 100644 index 00000000000..cd35d32f690 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationToJsonProcessorTests.java @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; + +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; +import org.elasticsearch.search.aggregations.bucket.terms.Terms; +import org.elasticsearch.test.ESTestCase; +import org.joda.time.DateTime; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.Term; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createAggs; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createDateHistogramBucket; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createHistogramBucket; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createSingleValue; +import static org.elasticsearch.xpack.ml.datafeed.extractor.aggregation.AggregationTestUtils.createTerms; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AggregationToJsonProcessorTests extends ESTestCase { + + public void testProcessGivenHistogramOnly() throws IOException { + List histogramBuckets = Arrays.asList( + createHistogramBucket(1000L, 3), + createHistogramBucket(2000L, 5) + ); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("time"); + when(histogram.getBuckets()).thenReturn(histogramBuckets); + + String json = aggToString(histogram); + + assertThat(json, equalTo("{\"time\":1000,\"doc_count\":3} {\"time\":2000,\"doc_count\":5}")); + } + + public void testProcessGivenSingleMetricPerHistogram() throws IOException { + List histogramBuckets = Arrays.asList( + createHistogramBucket(1000L, 3, Arrays.asList(createSingleValue("my_value", 1.0))), + createHistogramBucket(2000L, 5, Arrays.asList(createSingleValue("my_value", 2.0))) + ); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("time"); + when(histogram.getBuckets()).thenReturn(histogramBuckets); + + String json = aggToString(histogram); + + assertThat(json, equalTo("{\"time\":1000,\"my_value\":1.0,\"doc_count\":3} {\"time\":2000,\"my_value\":2.0,\"doc_count\":5}")); + } + + public void testProcessGivenTermsPerHistogram() throws IOException { + List histogramBuckets = Arrays.asList( + createHistogramBucket(1000L, 4, Arrays.asList( + createTerms("my_field", new Term("a", 1), new Term("b", 2), new Term("c", 1)))), + createHistogramBucket(2000L, 5, Arrays.asList(createTerms("my_field", new Term("a", 5), new Term("b", 2)))), + createHistogramBucket(3000L, 0, Arrays.asList()), + createHistogramBucket(4000L, 7, Arrays.asList(createTerms("my_field", new Term("c", 4), new Term("b", 3)))) + ); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("time"); + when(histogram.getBuckets()).thenReturn(histogramBuckets); + + String json = aggToString(histogram); + + assertThat(json, equalTo("{\"time\":1000,\"my_field\":\"a\",\"doc_count\":1} " + + "{\"time\":1000,\"my_field\":\"b\",\"doc_count\":2} " + + "{\"time\":1000,\"my_field\":\"c\",\"doc_count\":1} " + + "{\"time\":2000,\"my_field\":\"a\",\"doc_count\":5} " + + "{\"time\":2000,\"my_field\":\"b\",\"doc_count\":2} " + + "{\"time\":4000,\"my_field\":\"c\",\"doc_count\":4} " + + "{\"time\":4000,\"my_field\":\"b\",\"doc_count\":3}")); + } + + public void testProcessGivenSingleMetricPerSingleTermsPerHistogram() throws IOException { + List histogramBuckets = Arrays.asList( + createHistogramBucket(1000L, 4, Arrays.asList(createTerms("my_field", + new Term("a", 1, "my_value", 11.0), new Term("b", 2, "my_value", 12.0), new Term("c", 1, "my_value", 13.0)))), + createHistogramBucket(2000L, 5, Arrays.asList(createTerms("my_field", + new Term("a", 5, "my_value", 21.0), new Term("b", 2, "my_value", 22.0)))), + createHistogramBucket(3000L, 0, Arrays.asList()), + createHistogramBucket(4000L, 7, Arrays.asList(createTerms("my_field", + new Term("c", 4, "my_value", 41.0), new Term("b", 3, "my_value", 42.0)))) + ); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("time"); + when(histogram.getBuckets()).thenReturn(histogramBuckets); + + String json = aggToString(histogram); + + assertThat(json, equalTo("{\"time\":1000,\"my_field\":\"a\",\"my_value\":11.0,\"doc_count\":1} " + + "{\"time\":1000,\"my_field\":\"b\",\"my_value\":12.0,\"doc_count\":2} " + + "{\"time\":1000,\"my_field\":\"c\",\"my_value\":13.0,\"doc_count\":1} " + + "{\"time\":2000,\"my_field\":\"a\",\"my_value\":21.0,\"doc_count\":5} " + + "{\"time\":2000,\"my_field\":\"b\",\"my_value\":22.0,\"doc_count\":2} " + + "{\"time\":4000,\"my_field\":\"c\",\"my_value\":41.0,\"doc_count\":4} " + + "{\"time\":4000,\"my_field\":\"b\",\"my_value\":42.0,\"doc_count\":3}")); + } + + public void testProcessGivenTopLevelAggIsNotHistogram() throws IOException { + Terms terms = mock(Terms.class); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> aggToString(terms)); + assertThat(e.getMessage(), containsString("Top level aggregation should be [histogram]")); + } + + public void testProcessGivenUnsupportedAggregationUnderHistogram() throws IOException { + Histogram.Bucket histogramBucket = createHistogramBucket(1000L, 2); + Histogram anotherHistogram = mock(Histogram.class); + when(anotherHistogram.getName()).thenReturn("nested-agg"); + Aggregations subAggs = createAggs(Arrays.asList(anotherHistogram)); + when(histogramBucket.getAggregations()).thenReturn(subAggs); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("buckets"); + when(histogram.getBuckets()).thenReturn(Arrays.asList(histogramBucket)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> aggToString(histogram)); + assertThat(e.getMessage(), containsString("Unsupported aggregation type [nested-agg]")); + } + + public void testProcessGivenMultipleNestedAggregations() throws IOException { + Histogram.Bucket histogramBucket = createHistogramBucket(1000L, 2); + Terms terms1 = mock(Terms.class); + Terms terms2 = mock(Terms.class); + Aggregations subAggs = createAggs(Arrays.asList(terms1, terms2)); + when(histogramBucket.getAggregations()).thenReturn(subAggs); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("buckets"); + when(histogram.getBuckets()).thenReturn(Arrays.asList(histogramBucket)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> aggToString(histogram)); + assertThat(e.getMessage(), containsString("Multiple nested aggregations are not supported")); + } + + public void testProcessGivenHistogramWithDateTimeKeys() throws IOException { + List histogramBuckets = Arrays.asList( + createDateHistogramBucket(new DateTime(1000L), 3), + createDateHistogramBucket(new DateTime(2000L), 5) + ); + Histogram histogram = mock(Histogram.class); + when(histogram.getName()).thenReturn("time"); + when(histogram.getBuckets()).thenReturn(histogramBuckets); + + String json = aggToString(histogram); + + assertThat(json, equalTo("{\"time\":1000,\"doc_count\":3} {\"time\":2000,\"doc_count\":5}")); + } + + private String aggToString(Aggregation aggregation) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (AggregationToJsonProcessor processor = new AggregationToJsonProcessor(outputStream)) { + processor.process(aggregation); + } + return outputStream.toString(StandardCharsets.UTF_8.name()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java new file mode 100644 index 00000000000..cc7b81bc1eb --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java @@ -0,0 +1,475 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.mock.orig.Mockito; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractor; +import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ChunkedDataExtractorTests extends ESTestCase { + + private Client client; + private List capturedSearchRequests; + private String jobId; + private String timeField; + private List types; + private List indexes; + private QueryBuilder query; + private int scrollSize; + private Long chunkSpan; + private DataExtractorFactory dataExtractorFactory; + + private class TestDataExtractor extends ChunkedDataExtractor { + + private SearchResponse nextResponse; + + TestDataExtractor(long start, long end) { + super(client, dataExtractorFactory, createContext(start, end)); + } + + @Override + protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { + capturedSearchRequests.add(searchRequestBuilder); + return nextResponse; + } + + void setNextResponse(SearchResponse searchResponse) { + nextResponse = searchResponse; + } + } + + @Before + public void setUpTests() { + client = mock(Client.class); + capturedSearchRequests = new ArrayList<>(); + jobId = "test-job"; + timeField = "time"; + indexes = Arrays.asList("index-1", "index-2"); + types = Arrays.asList("type-1", "type-2"); + query = QueryBuilders.matchAllQuery(); + scrollSize = 1000; + chunkSpan = null; + dataExtractorFactory = mock(DataExtractorFactory.class); + } + + public void testExtractionGivenNoData() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createSearchResponse(0L, 0L, 0L)); + + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + } + + public void testExtractionGivenSpecifiedChunk() throws IOException { + chunkSpan = 1000L; + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + + InputStream inputStream1 = mock(InputStream.class); + InputStream inputStream2 = mock(InputStream.class); + InputStream inputStream3 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1, inputStream2); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + + DataExtractor subExtactor2 = new StubSubExtractor(inputStream3); + when(dataExtractorFactory.newExtractor(2000L, 2300L)).thenReturn(subExtactor2); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream2, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream3, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + + verify(dataExtractorFactory).newExtractor(1000L, 2000L); + verify(dataExtractorFactory).newExtractor(2000L, 2300L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"size\":0")); + assertThat(searchRequest, containsString("\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," + + "{\"range\":{\"time\":{\"from\":1000,\"to\":2300,\"include_lower\":true,\"include_upper\":false," + + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]")); + assertThat(searchRequest, containsString("\"aggregations\":{\"earliest_time\":{\"min\":{\"field\":\"time\"}}," + + "\"latest_time\":{\"max\":{\"field\":\"time\"}}}}")); + assertThat(searchRequest, not(containsString("\"sort\""))); + } + + public void testExtractionGivenAutoChunkAndScrollSize1000() throws IOException { + chunkSpan = null; + scrollSize = 1000; + TestDataExtractor extractor = new TestDataExtractor(100000L, 450000L); + + // 300K millis * 1000 * 10 / 15K docs = 200000 + extractor.setNextResponse(createSearchResponse(15000L, 100000L, 400000L)); + + InputStream inputStream1 = mock(InputStream.class); + InputStream inputStream2 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(100000L, 300000L)).thenReturn(subExtactor1); + + DataExtractor subExtactor2 = new StubSubExtractor(inputStream2); + when(dataExtractorFactory.newExtractor(300000L, 450000L)).thenReturn(subExtactor2); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream2, extractor.next().get()); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(100000L, 300000L); + verify(dataExtractorFactory).newExtractor(300000L, 450000L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + } + + public void testExtractionGivenAutoChunkAndScrollSize500() throws IOException { + chunkSpan = null; + scrollSize = 500; + TestDataExtractor extractor = new TestDataExtractor(100000L, 450000L); + + // 300K millis * 500 * 10 / 15K docs = 100000 + extractor.setNextResponse(createSearchResponse(15000L, 100000L, 400000L)); + + InputStream inputStream1 = mock(InputStream.class); + InputStream inputStream2 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(100000L, 200000L)).thenReturn(subExtactor1); + + DataExtractor subExtactor2 = new StubSubExtractor(inputStream2); + when(dataExtractorFactory.newExtractor(200000L, 300000L)).thenReturn(subExtactor2); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream2, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + + verify(dataExtractorFactory).newExtractor(100000L, 200000L); + verify(dataExtractorFactory).newExtractor(200000L, 300000L); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + } + + public void testExtractionGivenAutoChunkIsLessThanMinChunk() throws IOException { + chunkSpan = null; + scrollSize = 1000; + TestDataExtractor extractor = new TestDataExtractor(100000L, 450000L); + + // 30K millis * 1000 * 10 / 150K docs = 2000 < min of 60K + extractor.setNextResponse(createSearchResponse(150000L, 100000L, 400000L)); + + InputStream inputStream1 = mock(InputStream.class); + InputStream inputStream2 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(100000L, 160000L)).thenReturn(subExtactor1); + + DataExtractor subExtactor2 = new StubSubExtractor(inputStream2); + when(dataExtractorFactory.newExtractor(160000L, 220000L)).thenReturn(subExtactor2); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream2, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + + verify(dataExtractorFactory).newExtractor(100000L, 160000L); + verify(dataExtractorFactory).newExtractor(160000L, 220000L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + } + + public void testExtractionGivenAutoChunkAndDataTimeSpreadIsZero() throws IOException { + chunkSpan = null; + scrollSize = 1000; + TestDataExtractor extractor = new TestDataExtractor(100L, 500L); + + extractor.setNextResponse(createSearchResponse(150000L, 300L, 300L)); + + InputStream inputStream1 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(300L, 500L)).thenReturn(subExtactor1); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(300L, 500L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + } + + public void testExtractionGivenAutoChunkAndTotalTimeRangeSmallerThanChunk() throws IOException { + chunkSpan = null; + scrollSize = 1000; + TestDataExtractor extractor = new TestDataExtractor(1L, 101L); + + // 100 millis * 1000 * 10 / 10 docs = 100000 + extractor.setNextResponse(createSearchResponse(10L, 1L, 101L)); + + InputStream inputStream1 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(1L, 101L)).thenReturn(subExtactor1); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(1L, 101L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + + assertThat(capturedSearchRequests.size(), equalTo(1)); + } + + public void testExtractionGivenAutoChunkAndIntermediateEmptySearchShouldReconfigure() throws IOException { + chunkSpan = null; + scrollSize = 500; + TestDataExtractor extractor = new TestDataExtractor(100000L, 400000L); + + // 300K millis * 500 * 10 / 15K docs = 100000 + extractor.setNextResponse(createSearchResponse(15000L, 100000L, 400000L)); + + InputStream inputStream1 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(100000L, 200000L)).thenReturn(subExtactor1); + + // This one is empty + DataExtractor subExtactor2 = new StubSubExtractor(); + when(dataExtractorFactory.newExtractor(200000, 300000L)).thenReturn(subExtactor2); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + + // Now we have: 200K millis * 500 * 10 / 5K docs = 200000 + extractor.setNextResponse(createSearchResponse(5000, 200000L, 400000L)); + + // This is the last one + InputStream inputStream2 = mock(InputStream.class); + DataExtractor subExtactor3 = new StubSubExtractor(inputStream2); + when(dataExtractorFactory.newExtractor(200000, 400000)).thenReturn(subExtactor3); + + assertEquals(inputStream2, extractor.next().get()); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(100000L, 200000L); + verify(dataExtractorFactory).newExtractor(200000L, 300000L); + verify(dataExtractorFactory).newExtractor(200000L, 400000L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + + assertThat(capturedSearchRequests.size(), equalTo(2)); + + String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"from\":100000,\"to\":400000")); + searchRequest = capturedSearchRequests.get(1).toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"from\":200000,\"to\":400000")); + } + + public void testCancelGivenNextWasNeverCalled() throws IOException { + chunkSpan = 1000L; + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + + InputStream inputStream1 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + + assertThat(extractor.hasNext(), is(true)); + + extractor.cancel(); + + assertThat(extractor.isCancelled(), is(true)); + assertThat(extractor.hasNext(), is(false)); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + } + + public void testCancelGivenCurrentSubExtractorHasMore() throws IOException { + chunkSpan = 1000L; + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + + InputStream inputStream1 = mock(InputStream.class); + InputStream inputStream2 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1, inputStream2); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + + extractor.cancel(); + + assertThat(extractor.isCancelled(), is(true)); + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream2, extractor.next().get()); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(1000L, 2000L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + } + + public void testCancelGivenCurrentSubExtractorIsDone() throws IOException { + chunkSpan = 1000L; + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createSearchResponse(10L, 1000L, 2200L)); + + InputStream inputStream1 = mock(InputStream.class); + + DataExtractor subExtactor1 = new StubSubExtractor(inputStream1); + when(dataExtractorFactory.newExtractor(1000L, 2000L)).thenReturn(subExtactor1); + + assertThat(extractor.hasNext(), is(true)); + assertEquals(inputStream1, extractor.next().get()); + + extractor.cancel(); + + assertThat(extractor.isCancelled(), is(true)); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + + verify(dataExtractorFactory).newExtractor(1000L, 2000L); + Mockito.verifyNoMoreInteractions(dataExtractorFactory); + } + + public void testDataSummaryRequestIsNotOk() { + chunkSpan = 2000L; + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createErrorResponse()); + + assertThat(extractor.hasNext(), is(true)); + expectThrows(IOException.class, () -> extractor.next()); + } + + public void testDataSummaryRequestHasShardFailures() { + chunkSpan = 2000L; + TestDataExtractor extractor = new TestDataExtractor(1000L, 2300L); + extractor.setNextResponse(createResponseWithShardFailures()); + + assertThat(extractor.hasNext(), is(true)); + expectThrows(IOException.class, () -> extractor.next()); + } + + private SearchResponse createSearchResponse(long totalHits, long earliestTime, long latestTime) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + SearchHit[] hits = new SearchHit[(int)totalHits]; + SearchHits searchHits = new SearchHits(hits, totalHits, 1); + when(searchResponse.getHits()).thenReturn(searchHits); + + Aggregations aggs = mock(Aggregations.class); + when(aggs.getProperty("earliest_time.value")).thenReturn((double) earliestTime); + when(aggs.getProperty("latest_time.value")).thenReturn((double) latestTime); + when(searchResponse.getAggregations()).thenReturn(aggs); + return searchResponse; + } + + private SearchResponse createErrorResponse() { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.INTERNAL_SERVER_ERROR); + return searchResponse; + } + + private SearchResponse createResponseWithShardFailures() { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getShardFailures()).thenReturn( + new ShardSearchFailure[] { new ShardSearchFailure(new RuntimeException("shard failed"))}); + return searchResponse; + } + + private ChunkedDataExtractorContext createContext(long start, long end) { + return new ChunkedDataExtractorContext(jobId, timeField, indexes, types, query, scrollSize, start, end, chunkSpan); + } + + private static class StubSubExtractor implements DataExtractor { + List streams = new ArrayList<>(); + boolean hasNext = true; + + StubSubExtractor() {} + + StubSubExtractor(InputStream... streams) { + for (InputStream stream : streams) { + this.streams.add(stream); + } + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public Optional next() throws IOException { + if (streams.isEmpty()) { + hasNext = false; + return Optional.empty(); + } + return Optional.of(streams.remove(0)); + } + + @Override + public boolean isCancelled() { + return false; + } + + @Override + public void cancel() { + // do nothing + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java new file mode 100644 index 00000000000..64e40f21c40 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.test.ESTestCase; +import org.joda.time.DateTime; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class ExtractedFieldTests extends ESTestCase { + + public void testValueGivenDocValue() { + SearchHit hit = new SearchHitBuilder(42).addField("single", "bar").addField("array", Arrays.asList("a", "b")).build(); + + ExtractedField single = ExtractedField.newField("single", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(single.value(hit), equalTo(new String[] { "bar" })); + + ExtractedField array = ExtractedField.newField("array", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(array.value(hit), equalTo(new String[] { "a", "b" })); + + ExtractedField missing = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(missing.value(hit), equalTo(new Object[0])); + } + + public void testValueGivenScriptField() { + SearchHit hit = new SearchHitBuilder(42).addField("single", "bar").addField("array", Arrays.asList("a", "b")).build(); + + ExtractedField single = ExtractedField.newField("single", ExtractedField.ExtractionMethod.SCRIPT_FIELD); + assertThat(single.value(hit), equalTo(new String[] { "bar" })); + + ExtractedField array = ExtractedField.newField("array", ExtractedField.ExtractionMethod.SCRIPT_FIELD); + assertThat(array.value(hit), equalTo(new String[] { "a", "b" })); + + ExtractedField missing = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.SCRIPT_FIELD); + assertThat(missing.value(hit), equalTo(new Object[0])); + } + + public void testValueGivenSource() { + SearchHit hit = new SearchHitBuilder(42).setSource("{\"single\":\"bar\",\"array\":[\"a\",\"b\"]}").build(); + + ExtractedField single = ExtractedField.newField("single", ExtractedField.ExtractionMethod.SOURCE); + assertThat(single.value(hit), equalTo(new String[] { "bar" })); + + ExtractedField array = ExtractedField.newField("array", ExtractedField.ExtractionMethod.SOURCE); + assertThat(array.value(hit), equalTo(new String[] { "a", "b" })); + + ExtractedField missing = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.SOURCE); + assertThat(missing.value(hit), equalTo(new Object[0])); + } + + public void testValueGivenNestedSource() { + SearchHit hit = new SearchHitBuilder(42).setSource("{\"level_1\":{\"level_2\":{\"foo\":\"bar\"}}}").build(); + + ExtractedField nested = ExtractedField.newField("level_1.level_2.foo", ExtractedField.ExtractionMethod.SOURCE); + assertThat(nested.value(hit), equalTo(new String[] { "bar" })); + } + + public void testValueGivenSourceAndHitWithNoSource() { + ExtractedField missing = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.SOURCE); + assertThat(missing.value(new SearchHitBuilder(3).build()), equalTo(new Object[0])); + } + + public void testValueGivenMismatchingMethod() { + SearchHit hit = new SearchHitBuilder(42).addField("a", 1).setSource("{\"b\":2}").build(); + + ExtractedField invalidA = ExtractedField.newField("a", ExtractedField.ExtractionMethod.SOURCE); + assertThat(invalidA.value(hit), equalTo(new Object[0])); + ExtractedField validA = ExtractedField.newField("a", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(validA.value(hit), equalTo(new Integer[] { 1 })); + + ExtractedField invalidB = ExtractedField.newField("b", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(invalidB.value(hit), equalTo(new Object[0])); + ExtractedField validB = ExtractedField.newField("b", ExtractedField.ExtractionMethod.SOURCE); + assertThat(validB.value(hit), equalTo(new Integer[] { 2 })); + } + + public void testValueGivenEmptyHit() { + SearchHit hit = new SearchHitBuilder(42).build(); + + ExtractedField docValue = ExtractedField.newField("a", ExtractedField.ExtractionMethod.SOURCE); + assertThat(docValue.value(hit), equalTo(new Object[0])); + + ExtractedField sourceField = ExtractedField.newField("b", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(sourceField.value(hit), equalTo(new Object[0])); + } + + public void testNewTimeFieldGivenSource() { + expectThrows(IllegalArgumentException.class, () -> ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.SOURCE)); + } + + public void testValueGivenTimeField() { + SearchHit hit = new SearchHitBuilder(42).addField("time", new DateTime(123456789L)).build(); + + ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + + assertThat(timeField.value(hit), equalTo(new Object[] { 123456789L })); + } + + static class SearchHitBuilder { + + private final SearchHit hit; + private final Map fields; + + SearchHitBuilder(int docId) { + hit = new SearchHit(docId); + fields = new HashMap<>(); + } + + SearchHitBuilder addField(String name, Object value) { + return addField(name, Arrays.asList(value)); + } + + SearchHitBuilder addField(String name, List values) { + fields.put(name, new SearchHitField(name, values)); + return this; + } + + SearchHitBuilder setSource(String sourceJson) { + hit.sourceRef(new BytesArray(sourceJson)); + return this; + } + + SearchHit build() { + if (!fields.isEmpty()) { + hit.fields(fields); + } + return hit; + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java new file mode 100644 index 00000000000..dbaaa648f78 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESTestCase; +import org.joda.time.DateTime; + +import java.util.Arrays; +import java.util.Collections; + +import static org.hamcrest.Matchers.equalTo; + +public class ExtractedFieldsTests extends ESTestCase { + + private ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + + public void testInvalidConstruction() { + expectThrows(IllegalArgumentException.class, () -> new ExtractedFields(timeField, Collections.emptyList())); + } + + public void testTimeFieldOnly() { + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField)); + + assertThat(extractedFields.getAllFields(), equalTo(Arrays.asList(timeField))); + assertThat(extractedFields.timeField(), equalTo("time")); + assertThat(extractedFields.getDocValueFields(), equalTo(new String[] { timeField.getName() })); + assertThat(extractedFields.getSourceFields().length, equalTo(0)); + } + + public void testAllTypesOfFields() { + ExtractedField docValue1 = ExtractedField.newField("doc1", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField docValue2 = ExtractedField.newField("doc2", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField scriptField1 = ExtractedField.newField("scripted1", ExtractedField.ExtractionMethod.SCRIPT_FIELD); + ExtractedField scriptField2 = ExtractedField.newField("scripted2", ExtractedField.ExtractionMethod.SCRIPT_FIELD); + ExtractedField sourceField1 = ExtractedField.newField("src1", ExtractedField.ExtractionMethod.SOURCE); + ExtractedField sourceField2 = ExtractedField.newField("src2", ExtractedField.ExtractionMethod.SOURCE); + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField, + docValue1, docValue2, scriptField1, scriptField2, sourceField1, sourceField2)); + + assertThat(extractedFields.getAllFields().size(), equalTo(7)); + assertThat(extractedFields.timeField(), equalTo("time")); + assertThat(extractedFields.getDocValueFields(), equalTo(new String[] {"time", "doc1", "doc2"})); + assertThat(extractedFields.getSourceFields(), equalTo(new String[] {"src1", "src2"})); + } + + public void testTimeFieldValue() { + SearchHit hit = new ExtractedFieldTests.SearchHitBuilder(1).addField("time", new DateTime(1000L)).build(); + + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField)); + + assertThat(extractedFields.timeFieldValue(hit), equalTo(1000L)); + } + + public void testTimeFieldValueGivenEmptyArray() { + SearchHit hit = new ExtractedFieldTests.SearchHitBuilder(1).addField("time", Collections.emptyList()).build(); + + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField)); + + expectThrows(RuntimeException.class, () -> extractedFields.timeFieldValue(hit)); + } + + public void testTimeFieldValueGivenValueHasTwoElements() { + SearchHit hit = new ExtractedFieldTests.SearchHitBuilder(1).addField("time", Arrays.asList(1L, 2L)).build(); + + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField)); + + expectThrows(RuntimeException.class, () -> extractedFields.timeFieldValue(hit)); + } + + public void testTimeFieldValueGivenValueIsString() { + SearchHit hit = new ExtractedFieldTests.SearchHitBuilder(1).addField("time", "a string").build(); + + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField)); + + expectThrows(RuntimeException.class, () -> extractedFields.timeFieldValue(hit)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java new file mode 100644 index 00000000000..eb1ebd1489d --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java @@ -0,0 +1,327 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class ScrollDataExtractorTests extends ESTestCase { + + private Client client; + private List capturedSearchRequests; + private List capturedContinueScrollIds; + private List capturedClearScrollIds; + private String jobId; + private ExtractedFields extractedFields; + private List types; + private List indexes; + private QueryBuilder query; + private List scriptFields; + private int scrollSize; + + private class TestDataExtractor extends ScrollDataExtractor { + + private SearchResponse nextResponse; + + TestDataExtractor(long start, long end) { + super(client, createContext(start, end)); + } + + @Override + protected SearchResponse executeSearchRequest(SearchRequestBuilder searchRequestBuilder) { + capturedSearchRequests.add(searchRequestBuilder); + return nextResponse; + } + + @Override + protected SearchResponse executeSearchScrollRequest(String scrollId) { + capturedContinueScrollIds.add(scrollId); + return nextResponse; + } + + @Override + void clearScroll(String scrollId) { + capturedClearScrollIds.add(scrollId); + } + + void setNextResponse(SearchResponse searchResponse) { + nextResponse = searchResponse; + } + } + + @Before + public void setUpTests() { + client = mock(Client.class); + capturedSearchRequests = new ArrayList<>(); + capturedContinueScrollIds = new ArrayList<>(); + capturedClearScrollIds = new ArrayList<>(); + jobId = "test-job"; + ExtractedField timeField = ExtractedField.newField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + extractedFields = new ExtractedFields(timeField, + Arrays.asList(timeField, ExtractedField.newField("field_1", ExtractedField.ExtractionMethod.DOC_VALUE))); + indexes = Arrays.asList("index-1", "index-2"); + types = Arrays.asList("type-1", "type-2"); + query = QueryBuilders.matchAllQuery(); + scriptFields = Collections.emptyList(); + scrollSize = 1000; + } + + public void testSinglePageExtraction() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + + SearchResponse response1 = createSearchResponse( + Arrays.asList(1100L, 1200L), + Arrays.asList("a1", "a2"), + Arrays.asList("b1", "b2") + ); + extractor.setNextResponse(response1); + + assertThat(extractor.hasNext(), is(true)); + Optional stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + String expectedStream = "{\"time\":1100,\"field_1\":\"a1\"} {\"time\":1200,\"field_1\":\"a2\"}"; + assertThat(asString(stream.get()), equalTo(expectedStream)); + + SearchResponse response2 = createEmptySearchResponse(); + extractor.setNextResponse(response2); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + assertThat(capturedSearchRequests.size(), equalTo(1)); + + String searchRequest = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + assertThat(searchRequest, containsString("\"size\":1000")); + assertThat(searchRequest, containsString("\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," + + "{\"range\":{\"time\":{\"from\":1000,\"to\":2000,\"include_lower\":true,\"include_upper\":false," + + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]")); + assertThat(searchRequest, containsString("\"sort\":[{\"time\":{\"order\":\"asc\"}}]")); + assertThat(searchRequest, containsString("\"stored_fields\":\"_none_\"")); + + assertThat(capturedContinueScrollIds.size(), equalTo(1)); + assertThat(capturedContinueScrollIds.get(0), equalTo(response1.getScrollId())); + + assertThat(capturedClearScrollIds.size(), equalTo(1)); + assertThat(capturedClearScrollIds.get(0), equalTo(response2.getScrollId())); + } + + public void testMultiplePageExtraction() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 10000L); + + SearchResponse response1 = createSearchResponse( + Arrays.asList(1000L, 2000L), + Arrays.asList("a1", "a2"), + Arrays.asList("b1", "b2") + ); + extractor.setNextResponse(response1); + + assertThat(extractor.hasNext(), is(true)); + Optional stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + String expectedStream = "{\"time\":1000,\"field_1\":\"a1\"} {\"time\":2000,\"field_1\":\"a2\"}"; + assertThat(asString(stream.get()), equalTo(expectedStream)); + + SearchResponse response2 = createSearchResponse( + Arrays.asList(3000L, 4000L), + Arrays.asList("a3", "a4"), + Arrays.asList("b3", "b4") + ); + extractor.setNextResponse(response2); + + assertThat(extractor.hasNext(), is(true)); + stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + expectedStream = "{\"time\":3000,\"field_1\":\"a3\"} {\"time\":4000,\"field_1\":\"a4\"}"; + assertThat(asString(stream.get()), equalTo(expectedStream)); + + SearchResponse response3 = createEmptySearchResponse(); + extractor.setNextResponse(response3); + assertThat(extractor.hasNext(), is(true)); + assertThat(extractor.next().isPresent(), is(false)); + assertThat(extractor.hasNext(), is(false)); + assertThat(capturedSearchRequests.size(), equalTo(1)); + + String searchRequest1 = capturedSearchRequests.get(0).toString().replaceAll("\\s", ""); + assertThat(searchRequest1, containsString("\"size\":1000")); + assertThat(searchRequest1, containsString("\"query\":{\"bool\":{\"filter\":[{\"match_all\":{\"boost\":1.0}}," + + "{\"range\":{\"time\":{\"from\":1000,\"to\":10000,\"include_lower\":true,\"include_upper\":false," + + "\"format\":\"epoch_millis\",\"boost\":1.0}}}]")); + assertThat(searchRequest1, containsString("\"sort\":[{\"time\":{\"order\":\"asc\"}}]")); + + assertThat(capturedContinueScrollIds.size(), equalTo(2)); + assertThat(capturedContinueScrollIds.get(0), equalTo(response1.getScrollId())); + assertThat(capturedContinueScrollIds.get(1), equalTo(response2.getScrollId())); + + assertThat(capturedClearScrollIds.size(), equalTo(1)); + assertThat(capturedClearScrollIds.get(0), equalTo(response3.getScrollId())); + } + + public void testMultiplePageExtractionGivenCancel() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 10000L); + + SearchResponse response1 = createSearchResponse( + Arrays.asList(1000L, 2000L), + Arrays.asList("a1", "a2"), + Arrays.asList("b1", "b2") + ); + extractor.setNextResponse(response1); + + assertThat(extractor.hasNext(), is(true)); + Optional stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + String expectedStream = "{\"time\":1000,\"field_1\":\"a1\"} {\"time\":2000,\"field_1\":\"a2\"}"; + assertThat(asString(stream.get()), equalTo(expectedStream)); + + extractor.cancel(); + + SearchResponse response2 = createSearchResponse( + Arrays.asList(2000L, 3000L), + Arrays.asList("a3", "a4"), + Arrays.asList("b3", "b4") + ); + extractor.setNextResponse(response2); + + assertThat(extractor.isCancelled(), is(true)); + assertThat(extractor.hasNext(), is(true)); + stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + expectedStream = "{\"time\":2000,\"field_1\":\"a3\"}"; + assertThat(asString(stream.get()), equalTo(expectedStream)); + assertThat(extractor.hasNext(), is(false)); + + assertThat(capturedClearScrollIds.size(), equalTo(1)); + assertThat(capturedClearScrollIds.get(0), equalTo(response2.getScrollId())); + } + + public void testExtractionGivenInitSearchResponseHasError() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + extractor.setNextResponse(createErrorResponse()); + + assertThat(extractor.hasNext(), is(true)); + expectThrows(IOException.class, () -> extractor.next()); + } + + public void testExtractionGivenContinueScrollResponseHasError() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 10000L); + + SearchResponse response1 = createSearchResponse( + Arrays.asList(1000L, 2000L), + Arrays.asList("a1", "a2"), + Arrays.asList("b1", "b2") + ); + extractor.setNextResponse(response1); + + assertThat(extractor.hasNext(), is(true)); + Optional stream = extractor.next(); + assertThat(stream.isPresent(), is(true)); + + extractor.setNextResponse(createErrorResponse()); + assertThat(extractor.hasNext(), is(true)); + expectThrows(IOException.class, () -> extractor.next()); + } + + public void testExtractionGivenInitSearchResponseHasShardFailures() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + extractor.setNextResponse(createResponseWithShardFailures()); + + assertThat(extractor.hasNext(), is(true)); + expectThrows(IOException.class, () -> extractor.next()); + } + + public void testExtractionGivenInitSearchResponseEncounteredUnavailableShards() throws IOException { + TestDataExtractor extractor = new TestDataExtractor(1000L, 2000L); + extractor.setNextResponse(createResponseWithUnavailableShards(1)); + + assertThat(extractor.hasNext(), is(true)); + IOException e = expectThrows(IOException.class, () -> extractor.next()); + assertThat(e.getMessage(), equalTo("[" + jobId + "] Search request encountered [1] unavailable shards")); + } + + private ScrollDataExtractorContext createContext(long start, long end) { + return new ScrollDataExtractorContext(jobId, extractedFields, indexes, types, query, scriptFields, scrollSize, start, end); + } + + private SearchResponse createEmptySearchResponse() { + return createSearchResponse(Collections.emptyList(), Collections.emptyList(), Collections.emptyList()); + } + + private SearchResponse createSearchResponse(List timestamps, List field1Values, List field2Values) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getScrollId()).thenReturn(randomAsciiOfLength(1000)); + List hits = new ArrayList<>(); + for (int i = 0; i < timestamps.size(); i++) { + SearchHit hit = new SearchHit(randomInt()); + Map fields = new HashMap<>(); + fields.put(extractedFields.timeField(), new SearchHitField("time", Arrays.asList(timestamps.get(i)))); + fields.put("field_1", new SearchHitField("field_1", Arrays.asList(field1Values.get(i)))); + fields.put("field_2", new SearchHitField("field_2", Arrays.asList(field2Values.get(i)))); + hit.fields(fields); + hits.add(hit); + } + SearchHits searchHits = new SearchHits(hits.toArray(new SearchHit[0]), hits.size(), 1); + when(searchResponse.getHits()).thenReturn(searchHits); + return searchResponse; + } + + private SearchResponse createErrorResponse() { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.INTERNAL_SERVER_ERROR); + return searchResponse; + } + + private SearchResponse createResponseWithShardFailures() { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getShardFailures()).thenReturn( + new ShardSearchFailure[] { new ShardSearchFailure(new RuntimeException("shard failed"))}); + return searchResponse; + } + + private SearchResponse createResponseWithUnavailableShards(int unavailableShards) { + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.status()).thenReturn(RestStatus.OK); + when(searchResponse.getSuccessfulShards()).thenReturn(2); + when(searchResponse.getTotalShards()).thenReturn(2 + unavailableShards); + return searchResponse; + } + + private static String asString(InputStream inputStream) throws IOException { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java new file mode 100644 index 00000000000..7167fc127dd --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/SearchHitToJsonProcessorTests.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; + +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESTestCase; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.hamcrest.Matchers.equalTo; + +public class SearchHitToJsonProcessorTests extends ESTestCase { + + public void testProcessGivenSingleHit() throws IOException { + ExtractedField timeField = ExtractedField.newField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField missingField = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField singleField = ExtractedField.newField("single", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField arrayField = ExtractedField.newField("array", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField, missingField, singleField, arrayField)); + + SearchHit hit = new ExtractedFieldTests.SearchHitBuilder(8) + .addField("time", 1000L) + .addField("single", "a") + .addField("array", Arrays.asList("b", "c")) + .build(); + + String json = searchHitToString(extractedFields, hit); + + assertThat(json, equalTo("{\"time\":1000,\"single\":\"a\",\"array\":[\"b\",\"c\"]}")); + } + + public void testProcessGivenMultipleHits() throws IOException { + ExtractedField timeField = ExtractedField.newField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField missingField = ExtractedField.newField("missing", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField singleField = ExtractedField.newField("single", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedField arrayField = ExtractedField.newField("array", ExtractedField.ExtractionMethod.DOC_VALUE); + ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField, missingField, singleField, arrayField)); + + SearchHit hit1 = new ExtractedFieldTests.SearchHitBuilder(8) + .addField("time", 1000L) + .addField("single", "a1") + .addField("array", Arrays.asList("b1", "c1")) + .build(); + + SearchHit hit2 = new ExtractedFieldTests.SearchHitBuilder(8) + .addField("time", 2000L) + .addField("single", "a2") + .addField("array", Arrays.asList("b2", "c2")) + .build(); + + String json = searchHitToString(extractedFields, hit1, hit2); + + assertThat(json, equalTo("{\"time\":1000,\"single\":\"a1\",\"array\":[\"b1\",\"c1\"]} " + + "{\"time\":2000,\"single\":\"a2\",\"array\":[\"b2\",\"c2\"]}")); + } + + private String searchHitToString(ExtractedFields fields, SearchHit... searchHits) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try (SearchHitToJsonProcessor hitProcessor = new SearchHitToJsonProcessor(fields, outputStream)) { + for (int i = 0; i < searchHits.length; i++) { + hitProcessor.process(searchHits[i]); + } + } + return outputStream.toString(StandardCharsets.UTF_8.name()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java new file mode 100644 index 00000000000..3807389bb8a --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -0,0 +1,496 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.persistence.BucketsQueryBuilder; +import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.elasticsearch.xpack.ml.job.persistence.RecordsQueryBuilder; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcess; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.AutoDetectResultProcessor; +import org.elasticsearch.xpack.ml.job.process.autodetect.output.FlushAcknowledgement; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.process.normalizer.Renormalizer; +import org.elasticsearch.xpack.ml.job.process.normalizer.noop.NoOpRenormalizer; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.AutodetectResult; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketTests; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinitionTests; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.ModelDebugOutput; +import org.elasticsearch.xpack.ml.job.results.ModelDebugOutputTests; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class AutodetectResultProcessorIT extends ESSingleNodeTestCase { + private static final String JOB_ID = "foo"; + + private Renormalizer renormalizer; + private JobResultsPersister jobResultsPersister; + private JobProvider jobProvider; + + @Before + public void createComponents() { + renormalizer = new NoOpRenormalizer(); + jobResultsPersister = new JobResultsPersister(nodeSettings(), client()); + jobProvider = new JobProvider(client(), 1); + } + + public void testProcessResults() throws Exception { + createJob(); + AutoDetectResultProcessor resultProcessor = new AutoDetectResultProcessor(JOB_ID, renormalizer, jobResultsPersister); + + ResultsBuilder builder = new ResultsBuilder(); + Bucket bucket = createBucket(false); + builder.addBucket(bucket); + List records = createRecords(false); + builder.addRecords(records); + List influencers = createInfluencers(false); + builder.addInfluencers(influencers); + CategoryDefinition categoryDefinition = createCategoryDefinition(); + builder.addCategoryDefinition(categoryDefinition); + ModelDebugOutput modelDebugOutput = createModelDebugOutput(); + builder.addModelDebugOutput(modelDebugOutput); + ModelSizeStats modelSizeStats = createModelSizeStats(); + builder.addModelSizeStats(modelSizeStats); + ModelSnapshot modelSnapshot = createModelSnapshot(); + builder.addModelSnapshot(modelSnapshot); + Quantiles quantiles = createQuantiles(); + builder.addQuantiles(quantiles); + + resultProcessor.process(builder.buildTestProcess(), false); + jobResultsPersister.commitResultWrites(JOB_ID); + + BucketsQueryBuilder.BucketsQuery bucketsQuery = new BucketsQueryBuilder().includeInterim(true).build(); + QueryPage persistedBucket = getBucketQueryPage(bucketsQuery); + assertEquals(1, persistedBucket.count()); + // Records are not persisted to Elasticsearch as an array within the bucket + // documents, so remove them from the expected bucket before comparing + bucket.setRecords(Collections.emptyList()); + assertEquals(bucket, persistedBucket.results().get(0)); + + QueryPage persistedRecords = getRecords(new RecordsQueryBuilder().build()); + assertResultsAreSame(records, persistedRecords); + + QueryPage persistedInfluencers = getInfluencers(); + assertResultsAreSame(influencers, persistedInfluencers); + + QueryPage persistedDefinition = + getCategoryDefinition(Long.toString(categoryDefinition.getCategoryId())); + assertEquals(1, persistedDefinition.count()); + assertEquals(categoryDefinition, persistedDefinition.results().get(0)); + + QueryPage persistedModelDebugOutput = jobProvider.modelDebugOutput(JOB_ID, 0, 100); + assertEquals(1, persistedModelDebugOutput.count()); + assertEquals(modelDebugOutput, persistedModelDebugOutput.results().get(0)); + + ModelSizeStats persistedModelSizeStats = getModelSizeStats(); + assertEquals(modelSizeStats, persistedModelSizeStats); + + QueryPage persistedModelSnapshot = getModelSnapshots(); + assertEquals(1, persistedModelSnapshot.count()); + assertEquals(modelSnapshot, persistedModelSnapshot.results().get(0)); + + Optional persistedQuantiles = getQuantiles(); + assertTrue(persistedQuantiles.isPresent()); + assertEquals(quantiles, persistedQuantiles.get()); + } + + public void testDeleteInterimResults() throws Exception { + createJob(); + AutoDetectResultProcessor resultProcessor = new AutoDetectResultProcessor(JOB_ID, renormalizer, jobResultsPersister); + Bucket nonInterimBucket = createBucket(false); + Bucket interimBucket = createBucket(true); + + ResultsBuilder resultBuilder = new ResultsBuilder() + .addRecords(createRecords(true)) + .addInfluencers(createInfluencers(true)) + .addBucket(interimBucket) // this will persist the interim results + .addFlushAcknowledgement(createFlushAcknowledgement()) + .addBucket(nonInterimBucket); // and this will delete the interim results + + resultProcessor.process(resultBuilder.buildTestProcess(), false); + jobResultsPersister.commitResultWrites(JOB_ID); + + QueryPage persistedBucket = getBucketQueryPage(new BucketsQueryBuilder().includeInterim(true).build()); + assertEquals(1, persistedBucket.count()); + // Records are not persisted to Elasticsearch as an array within the bucket + // documents, so remove them from the expected bucket before comparing + nonInterimBucket.setRecords(Collections.emptyList()); + assertEquals(nonInterimBucket, persistedBucket.results().get(0)); + + QueryPage persistedInfluencers = getInfluencers(); + assertEquals(0, persistedInfluencers.count()); + + QueryPage persistedRecords = getRecords(new RecordsQueryBuilder().includeInterim(true).build()); + assertEquals(0, persistedRecords.count()); + } + + public void testMultipleFlushesBetweenPersisting() throws Exception { + createJob(); + AutoDetectResultProcessor resultProcessor = new AutoDetectResultProcessor(JOB_ID, renormalizer, jobResultsPersister); + Bucket finalBucket = createBucket(true); + List finalAnomalyRecords = createRecords(true); + + ResultsBuilder resultBuilder = new ResultsBuilder() + .addRecords(createRecords(true)) + .addInfluencers(createInfluencers(true)) + .addBucket(createBucket(true)) // this will persist the interim results + .addFlushAcknowledgement(createFlushAcknowledgement()) + .addRecords(createRecords(true)) + .addBucket(createBucket(true)) // and this will delete the interim results and persist the new interim bucket & records + .addFlushAcknowledgement(createFlushAcknowledgement()) + .addRecords(finalAnomalyRecords) + .addBucket(finalBucket); // this deletes the previous interim and persists final bucket & records + + resultProcessor.process(resultBuilder.buildTestProcess(), false); + jobResultsPersister.commitResultWrites(JOB_ID); + + QueryPage persistedBucket = getBucketQueryPage(new BucketsQueryBuilder().includeInterim(true).build()); + assertEquals(1, persistedBucket.count()); + // Records are not persisted to Elasticsearch as an array within the bucket + // documents, so remove them from the expected bucket before comparing + finalBucket.setRecords(Collections.emptyList()); + assertEquals(finalBucket, persistedBucket.results().get(0)); + + QueryPage persistedRecords = getRecords(new RecordsQueryBuilder().includeInterim(true).build()); + assertResultsAreSame(finalAnomalyRecords, persistedRecords); + } + + public void testEndOfStreamTriggersPersisting() throws Exception { + createJob(); + AutoDetectResultProcessor resultProcessor = new AutoDetectResultProcessor(JOB_ID, renormalizer, jobResultsPersister); + Bucket bucket = createBucket(false); + List firstSetOfRecords = createRecords(false); + List secondSetOfRecords = createRecords(false); + + ResultsBuilder resultBuilder = new ResultsBuilder() + .addRecords(firstSetOfRecords) + .addBucket(bucket) // bucket triggers persistence + .addRecords(secondSetOfRecords); + + resultProcessor.process(resultBuilder.buildTestProcess(), false); + jobResultsPersister.commitResultWrites(JOB_ID); + + QueryPage persistedBucket = getBucketQueryPage(new BucketsQueryBuilder().includeInterim(true).build()); + assertEquals(1, persistedBucket.count()); + + QueryPage persistedRecords = getRecords(new RecordsQueryBuilder().size(200).includeInterim(true).build()); + List allRecords = new ArrayList<>(firstSetOfRecords); + allRecords.addAll(secondSetOfRecords); + assertResultsAreSame(allRecords, persistedRecords); + } + + private void createJob() { + Detector.Builder detectorBuilder = new Detector.Builder("avg", "metric_field"); + detectorBuilder.setByFieldName("by_instance"); + Job.Builder jobBuilder = new Job.Builder(JOB_ID); + AnalysisConfig.Builder analysisConfBuilder = new AnalysisConfig.Builder(Collections.singletonList(detectorBuilder.build())); + analysisConfBuilder.setInfluencers(Collections.singletonList("influence_field")); + jobBuilder.setAnalysisConfig(analysisConfBuilder); + + jobProvider.createJobResultIndex(jobBuilder.build(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + } + + @Override + public void onFailure(Exception e) { + } + }); + } + + private Bucket createBucket(boolean isInterim) { + Bucket bucket = new BucketTests().createTestInstance(JOB_ID); + bucket.setInterim(isInterim); + return bucket; + } + + private List createRecords(boolean isInterim) { + List records = new ArrayList<>(); + + int count = randomIntBetween(0, 100); + Date now = new Date(randomNonNegativeLong()); + for (int i=0; i createInfluencers(boolean isInterim) { + List influencers = new ArrayList<>(); + + int count = randomIntBetween(0, 100); + Date now = new Date(); + for (int i=0; i results = new ArrayList<>(); + FlushAcknowledgement flushAcknowledgement; + + ResultsBuilder addBucket(Bucket bucket) { + results.add(new AutodetectResult(Objects.requireNonNull(bucket), null, null, null, null, null, null, null, null)); + return this; + } + + ResultsBuilder addRecords(List records) { + results.add(new AutodetectResult(null, records, null, null, null, null, null, null, null)); + return this; + } + + ResultsBuilder addInfluencers(List influencers) { + results.add(new AutodetectResult(null, null, influencers, null, null, null, null, null, null)); + return this; + } + + ResultsBuilder addCategoryDefinition(CategoryDefinition categoryDefinition) { + results.add(new AutodetectResult(null, null, null, null, null, null, null, categoryDefinition, null)); + return this; + } + + ResultsBuilder addModelDebugOutput(ModelDebugOutput modelDebugOutput) { + results.add(new AutodetectResult(null, null, null, null, null, null, modelDebugOutput, null, null)); + return this; + } + + ResultsBuilder addModelSizeStats(ModelSizeStats modelSizeStats) { + results.add(new AutodetectResult(null, null, null, null, null, modelSizeStats, null, null, null)); + return this; + } + + ResultsBuilder addModelSnapshot(ModelSnapshot modelSnapshot) { + results.add(new AutodetectResult(null, null, null, null, modelSnapshot, null, null, null, null)); + return this; + } + + ResultsBuilder addQuantiles(Quantiles quantiles) { + results.add(new AutodetectResult(null, null, null, quantiles, null, null, null, null, null)); + return this; + } + + ResultsBuilder addFlushAcknowledgement(FlushAcknowledgement flushAcknowledgement) { + results.add(new AutodetectResult(null, null, null, null, null, null, null, null, flushAcknowledgement)); + return this; + } + + + AutodetectProcess buildTestProcess() { + AutodetectResult[] results = this.results.toArray(new AutodetectResult[0]); + AutodetectProcess process = mock(AutodetectProcess.class); + when(process.readAutodetectResults()).thenReturn(Arrays.asList(results).iterator()); + return process; + } + } + + + private void assertResultsAreSame(List expected, QueryPage actual) { + assertEquals(expected.size(), actual.count()); + assertEquals(actual.results().size(), actual.count()); + Set expectedSet = new HashSet<>(expected); + expectedSet.removeAll(actual.results()); + assertEquals(0, expectedSet.size()); + } + + private QueryPage getBucketQueryPage(BucketsQueryBuilder.BucketsQuery bucketsQuery) throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference> resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.buckets(JOB_ID, bucketsQuery, r -> { + resultHolder.set(r); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } + + private QueryPage getCategoryDefinition(String categoryId) throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference> resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.categoryDefinitions(JOB_ID, categoryId, null, null, r -> { + resultHolder.set(r); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } + + private ModelSizeStats getModelSizeStats() throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.modelSizeStats(JOB_ID, modelSizeStats -> { + resultHolder.set(modelSizeStats); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } + + private QueryPage getInfluencers() throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference> resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.influencers(JOB_ID, new InfluencersQueryBuilder().build(), page -> { + resultHolder.set(page); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } + + private QueryPage getRecords(RecordsQueryBuilder.RecordsQuery recordsQuery) throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference> resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.records(JOB_ID, recordsQuery, page -> { + resultHolder.set(page); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } + + private QueryPage getModelSnapshots() throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference> resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.modelSnapshots(JOB_ID, 0, 100, page -> { + resultHolder.set(page); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } + private Optional getQuantiles() throws Exception { + AtomicReference errorHolder = new AtomicReference<>(); + AtomicReference> resultHolder = new AtomicReference<>(); + CountDownLatch latch = new CountDownLatch(1); + jobProvider.getQuantiles(JOB_ID, q -> { + resultHolder.set(Optional.ofNullable(q)); + latch.countDown(); + }, e -> { + errorHolder.set(e); + latch.countDown(); + }); + latch.await(); + if (errorHolder.get() != null) { + throw errorHolder.get(); + } + return resultHolder.get(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java new file mode 100644 index 00000000000..eab0a30c2ab --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobIT.java @@ -0,0 +1,450 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.junit.After; +import org.junit.Before; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class DatafeedJobIT extends ESRestTestCase { + + @Before + public void setUpData() throws Exception { + // Create empty index + String mappings = "{" + + " \"mappings\": {" + + " \"response\": {" + + " \"properties\": {" + + " \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + " }" + + "}"; + client().performRequest("put", "airline-data-empty", Collections.emptyMap(), new StringEntity(mappings)); + + // Create index with source = enabled, doc_values = enabled, stored = false + mappings = "{" + + " \"mappings\": {" + + " \"response\": {" + + " \"properties\": {" + + " \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + " }" + + "}"; + client().performRequest("put", "airline-data", Collections.emptyMap(), new StringEntity(mappings)); + + client().performRequest("put", "airline-data/response/1", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}")); + client().performRequest("put", "airline-data/response/2", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}")); + + // Create index with source = enabled, doc_values = disabled (except time), stored = false + mappings = "{" + + " \"mappings\": {" + + " \"response\": {" + + " \"properties\": {" + + " \"time stamp\": { \"type\":\"date\"}," + + " \"airline\": { \"type\":\"keyword\", \"doc_values\":false}," + + " \"responsetime\": { \"type\":\"float\", \"doc_values\":false}" + + " }" + + " }" + + " }" + + "}"; + client().performRequest("put", "airline-data-disabled-doc-values", Collections.emptyMap(), new StringEntity(mappings)); + + client().performRequest("put", "airline-data-disabled-doc-values/response/1", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}")); + client().performRequest("put", "airline-data-disabled-doc-values/response/2", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}")); + + // Create index with source = disabled, doc_values = enabled (except time), stored = true + mappings = "{" + + " \"mappings\": {" + + " \"response\": {" + + " \"_source\":{\"enabled\":false}," + + " \"properties\": {" + + " \"time stamp\": { \"type\":\"date\", \"store\":true}," + + " \"airline\": { \"type\":\"keyword\", \"store\":true}," + + " \"responsetime\": { \"type\":\"float\", \"store\":true}" + + " }" + + " }" + + " }" + + "}"; + client().performRequest("put", "airline-data-disabled-source", Collections.emptyMap(), new StringEntity(mappings)); + + client().performRequest("put", "airline-data-disabled-source/response/1", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":135.22}")); + client().performRequest("put", "airline-data-disabled-source/response/2", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:59:00Z\",\"airline\":\"AAA\",\"responsetime\":541.76}")); + + // Create index with nested documents + mappings = "{" + + " \"mappings\": {" + + " \"response\": {" + + " \"properties\": {" + + " \"time\": { \"type\":\"date\"}" + + " }" + + " }" + + " }" + + "}"; + client().performRequest("put", "nested-data", Collections.emptyMap(), new StringEntity(mappings)); + + client().performRequest("put", "nested-data/response/1", Collections.emptyMap(), + new StringEntity("{\"time\":\"2016-06-01T00:00:00Z\", \"responsetime\":{\"millis\":135.22}}")); + client().performRequest("put", "nested-data/response/2", Collections.emptyMap(), + new StringEntity("{\"time\":\"2016-06-01T01:59:00Z\",\"responsetime\":{\"millis\":222.0}}")); + + // Create index with multiple docs per time interval for aggregation testing + mappings = "{" + + " \"mappings\": {" + + " \"response\": {" + + " \"properties\": {" + + " \"time stamp\": { \"type\":\"date\"}," // space in 'time stamp' is intentional + + " \"airline\": { \"type\":\"keyword\"}," + + " \"responsetime\": { \"type\":\"float\"}" + + " }" + + " }" + + " }" + + "}"; + client().performRequest("put", "airline-data-aggs", Collections.emptyMap(), new StringEntity(mappings)); + + client().performRequest("put", "airline-data-aggs/response/1", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"AAA\",\"responsetime\":100.0}")); + client().performRequest("put", "airline-data-aggs/response/2", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"AAA\",\"responsetime\":200.0}")); + client().performRequest("put", "airline-data-aggs/response/3", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:00:00Z\",\"airline\":\"BBB\",\"responsetime\":1000.0}")); + client().performRequest("put", "airline-data-aggs/response/4", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T00:01:00Z\",\"airline\":\"BBB\",\"responsetime\":2000.0}")); + client().performRequest("put", "airline-data-aggs/response/5", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"AAA\",\"responsetime\":300.0}")); + client().performRequest("put", "airline-data-aggs/response/6", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"AAA\",\"responsetime\":400.0}")); + client().performRequest("put", "airline-data-aggs/response/7", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:00:00Z\",\"airline\":\"BBB\",\"responsetime\":3000.0}")); + client().performRequest("put", "airline-data-aggs/response/8", Collections.emptyMap(), + new StringEntity("{\"time stamp\":\"2016-06-01T01:01:00Z\",\"airline\":\"BBB\",\"responsetime\":4000.0}")); + + // Ensure all data is searchable + client().performRequest("post", "_refresh"); + } + + public void testLookbackOnly() throws Exception { + new LookbackOnlyTestHelper("lookback-1", "airline-data").setShouldSucceedProcessing(true).execute(); + } + + public void testLookbackOnlyWithDatafeedSourceEnabled() throws Exception { + new LookbackOnlyTestHelper("lookback-2", "airline-data").setEnableDatafeedSource(true).execute(); + } + + public void testLookbackOnlyWithDocValuesDisabledAndDatafeedSourceDisabled() throws Exception { + new LookbackOnlyTestHelper("lookback-3", "airline-data-disabled-doc-values").setShouldSucceedInput(false) + .setShouldSucceedProcessing(false).execute(); + } + + public void testLookbackOnlyWithDocValuesDisabledAndDatafeedSourceEnabled() throws Exception { + new LookbackOnlyTestHelper("lookback-4", "airline-data-disabled-doc-values").setEnableDatafeedSource(true).execute(); + } + + public void testLookbackOnlyWithSourceDisabled() throws Exception { + new LookbackOnlyTestHelper("lookback-5", "airline-data-disabled-source").execute(); + } + + @AwaitsFix(bugUrl = "This test uses painless which is not available in the integTest phase") + public void testLookbackOnlyWithScriptFields() throws Exception { + new LookbackOnlyTestHelper("lookback-6", "airline-data-disabled-source").setAddScriptedFields(true).execute(); + } + + public void testLookbackOnlyWithNestedFieldsAndDatafeedSourceDisabled() throws Exception { + executeTestLookbackOnlyWithNestedFields("lookback-7", false); + } + + public void testLookbackOnlyWithNestedFieldsAndDatafeedSourceEnabled() throws Exception { + executeTestLookbackOnlyWithNestedFields("lookback-8", true); + } + + public void testLookbackOnlyGivenEmptyIndex() throws Exception { + new LookbackOnlyTestHelper("lookback-9", "airline-data-empty").setShouldSucceedInput(false).setShouldSucceedProcessing(false) + .execute(); + } + + public void testLookbackOnlyGivenAggregationsWithHistogram() throws Exception { + String jobId = "aggs-histogram-job"; + String job = "{\"description\":\"Aggs job\",\"analysis_config\" :{\"bucket_span\":3600,\"summary_count_field_name\":\"doc_count\"," + + "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]}," + + "\"data_description\" : {\"time_field\":\"time stamp\"}" + + "}"; + client().performRequest("put", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), new StringEntity(job)); + + String datafeedId = "datafeed-" + jobId; + String aggregations = "{\"time stamp\":{\"histogram\":{\"field\":\"time stamp\",\"interval\":3600000}," + + "\"aggregations\":{\"airline\":{\"terms\":{\"field\":\"airline\",\"size\":10}," + + "\"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}"; + new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").setAggregations(aggregations).build(); + openJob(client(), jobId); + + startDatafeedAndWaitUntilStopped(datafeedId); + Response jobStatsResponse = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); + String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); + assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4")); + assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4")); + assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); + } + + public void testLookbackOnlyGivenAggregationsWithDateHistogram() throws Exception { + String jobId = "aggs-date-histogram-job"; + String job = "{\"description\":\"Aggs job\",\"analysis_config\" :{\"bucket_span\":3600,\"summary_count_field_name\":\"doc_count\"," + + "\"detectors\":[{\"function\":\"mean\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]}," + + "\"data_description\" : {\"time_field\":\"time stamp\"}" + + "}"; + client().performRequest("put", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), new StringEntity(job)); + + String datafeedId = "datafeed-" + jobId; + String aggregations = "{\"time stamp\":{\"date_histogram\":{\"field\":\"time stamp\",\"interval\":\"1h\"}," + + "\"aggregations\":{\"airline\":{\"terms\":{\"field\":\"airline\",\"size\":10}," + + "\"aggregations\":{\"responsetime\":{\"avg\":{\"field\":\"responsetime\"}}}}}}}"; + new DatafeedBuilder(datafeedId, jobId, "airline-data-aggs", "response").setAggregations(aggregations).build(); + openJob(client(), jobId); + + startDatafeedAndWaitUntilStopped(datafeedId); + Response jobStatsResponse = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); + String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); + assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":4")); + assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":4")); + assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); + } + + public void testRealtime() throws Exception { + String jobId = "job-realtime-1"; + createJob(jobId); + String datafeedId = jobId + "-datafeed"; + new DatafeedBuilder(datafeedId, jobId, "airline-data", "response").build(); + openJob(client(), jobId); + + Response response = client().performRequest("post", + MlPlugin.BASE_PATH + "datafeeds/" + datafeedId + "/_start?start=2016-06-01T00:00:00Z"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"started\":true}")); + assertBusy(() -> { + try { + Response getJobResponse = client().performRequest("get", + MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); + String responseAsString = responseEntityToString(getJobResponse); + assertThat(responseAsString, containsString("\"processed_record_count\":2")); + } catch (Exception e1) { + throw new RuntimeException(e1); + } + }); + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("delete", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId)); + response = e.getResponse(); + assertThat(response.getStatusLine().getStatusCode(), equalTo(409)); + assertThat(responseEntityToString(response), containsString("Cannot delete job [" + jobId + "] while datafeed [" + datafeedId + + "] refers to it")); + + response = client().performRequest("post", MlPlugin.BASE_PATH + "datafeeds/" + datafeedId + "/_stop"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); + + client().performRequest("POST", "/_xpack/ml/anomaly_detectors/" + jobId + "/_close"); + + response = client().performRequest("delete", MlPlugin.BASE_PATH + "datafeeds/" + datafeedId); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); + + response = client().performRequest("delete", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(response), equalTo("{\"acknowledged\":true}")); + } + + private class LookbackOnlyTestHelper { + private String jobId; + private String dataIndex; + private boolean addScriptedFields; + private boolean enableDatafeedSource; + private boolean shouldSucceedInput; + private boolean shouldSucceedProcessing; + + LookbackOnlyTestHelper(String jobId, String dataIndex) { + this.jobId = jobId; + this.dataIndex = dataIndex; + this.shouldSucceedInput = true; + this.shouldSucceedProcessing = true; + } + + public LookbackOnlyTestHelper setAddScriptedFields(boolean value) { + addScriptedFields = value; + return this; + } + + public LookbackOnlyTestHelper setEnableDatafeedSource(boolean value) { + enableDatafeedSource = value; + return this; + } + + public LookbackOnlyTestHelper setShouldSucceedInput(boolean value) { + shouldSucceedInput = value; + return this; + } + + public LookbackOnlyTestHelper setShouldSucceedProcessing(boolean value) { + shouldSucceedProcessing = value; + return this; + } + + public void execute() throws Exception { + createJob(jobId); + String datafeedId = "datafeed-" + jobId; + new DatafeedBuilder(datafeedId, jobId, dataIndex, "response") + .setSource(enableDatafeedSource) + .setScriptedFields(addScriptedFields ? + "{\"airline\":{\"script\":{\"lang\":\"painless\",\"inline\":\"doc['airline'].value\"}}}" : null) + .build(); + openJob(client(), jobId); + + startDatafeedAndWaitUntilStopped(datafeedId); + Response jobStatsResponse = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); + String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); + if (shouldSucceedInput) { + assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2")); + } else { + assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":0")); + } + if (shouldSucceedProcessing) { + assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2")); + } else { + assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":0")); + } + assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); + } + } + + private void startDatafeedAndWaitUntilStopped(String datafeedId) throws Exception { + Response startDatafeedRequest = client().performRequest("post", + MlPlugin.BASE_PATH + "datafeeds/" + datafeedId + "/_start?start=2016-06-01T00:00:00Z&end=2016-06-02T00:00:00Z"); + assertThat(startDatafeedRequest.getStatusLine().getStatusCode(), equalTo(200)); + assertThat(responseEntityToString(startDatafeedRequest), equalTo("{\"started\":true}")); + assertBusy(() -> { + try { + Response datafeedStatsResponse = client().performRequest("get", + MlPlugin.BASE_PATH + "datafeeds/" + datafeedId + "/_stats"); + assertThat(responseEntityToString(datafeedStatsResponse), containsString("\"state\":\"stopped\"")); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + private Response createJob(String id) throws Exception { + String job = "{\n" + " \"description\":\"Analysis of response time by airline\",\n" + + " \"analysis_config\" : {\n" + " \"bucket_span\":3600,\n" + + " \"detectors\" :[{\"function\":\"mean\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]\n" + + " },\n" + " \"data_description\" : {\n" + " \"format\":\"JSON\",\n" + + " \"time_field\":\"time stamp\",\n" + " \"time_format\":\"yyyy-MM-dd'T'HH:mm:ssX\"\n" + " }\n" + + "}"; + + return client().performRequest("put", MlPlugin.BASE_PATH + "anomaly_detectors/" + id, + Collections.emptyMap(), new StringEntity(job)); + } + + private static String responseEntityToString(Response response) throws Exception { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + + public static void openJob(RestClient client, String jobId) throws IOException { + Response response = client.performRequest("post", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_open"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + } + + private void executeTestLookbackOnlyWithNestedFields(String jobId, boolean source) throws Exception { + String job = "{\"description\":\"Nested job\", \"analysis_config\" : {\"bucket_span\":3600,\"detectors\" :" + + "[{\"function\":\"mean\",\"field_name\":\"responsetime.millis\"}]}, \"data_description\" : {\"time_field\":\"time\"}" + + "}"; + client().performRequest("put", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), new StringEntity(job)); + + String datafeedId = jobId + "-datafeed"; + new DatafeedBuilder(datafeedId, jobId, "nested-data", "response").setSource(source).build(); + openJob(client(), jobId); + + startDatafeedAndWaitUntilStopped(datafeedId); + Response jobStatsResponse = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats"); + String jobStatsResponseAsString = responseEntityToString(jobStatsResponse); + assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":2")); + assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":2")); + assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); + } + + @After + public void clearMlState() throws Exception { + new MlRestTestStateCleaner(client(), this).clearMlMetadata(); + } + + private static class DatafeedBuilder { + String datafeedId; + String jobId; + String index; + String type; + boolean source; + String scriptedFields; + String aggregations; + + DatafeedBuilder(String datafeedId, String jobId, String index, String type) { + this.datafeedId = datafeedId; + this.jobId = jobId; + this.index = index; + this.type = type; + } + + DatafeedBuilder setSource(boolean enableSource) { + this.source = enableSource; + return this; + } + + DatafeedBuilder setScriptedFields(String scriptedFields) { + this.scriptedFields = scriptedFields; + return this; + } + + DatafeedBuilder setAggregations(String aggregations) { + this.aggregations = aggregations; + return this; + } + + Response build() throws IOException { + String datafeedConfig = "{" + + "\"job_id\": \"" + jobId + "\",\"indexes\":[\"" + index + "\"],\"types\":[\"" + type + "\"]" + + (source ? ",\"_source\":true" : "") + + (scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields) + + (aggregations == null ? "" : ",\"aggs\":" + aggregations) + + "}"; + return client().performRequest("put", MlPlugin.BASE_PATH + "datafeeds/" + datafeedId, Collections.emptyMap(), + new StringEntity(datafeedConfig)); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java new file mode 100644 index 00000000000..7d8bf831a07 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -0,0 +1,355 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.apache.http.entity.StringEntity; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.junit.After; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.isEmptyString; +import static org.hamcrest.Matchers.not; + +public class MlJobIT extends ESRestTestCase { + + private static final String RESULT_MAPPING = "{ \"mappings\": {\"result\": { \"properties\": { " + + "\"result_type\": { \"type\" : \"keyword\" }," + + "\"timestamp\": { \"type\" : \"date\" }, " + + "\"anomaly_score\": { \"type\" : \"double\" }, " + + "\"normalized_probability\": { \"type\" : \"double\" }, " + + "\"over_field_value\": { \"type\" : \"keyword\" }, " + + "\"partition_field_value\": { \"type\" : \"keyword\" }, " + + "\"by_field_value\": { \"type\" : \"keyword\" }, " + + "\"field_name\": { \"type\" : \"keyword\" }, " + + "\"function\": { \"type\" : \"keyword\" } " + + "} } } }"; + + public void testPutJob_GivenFarequoteConfig() throws Exception { + Response response = createFarequoteJob(); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); + } + + public void testGetJob_GivenNoSuchJob() throws Exception { + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/non-existing-job/_stats")); + + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("No known job with id 'non-existing-job'")); + } + + public void testGetJob_GivenJobExists() throws Exception { + createFarequoteJob(); + + Response response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/farequote/_stats"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":1")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); + } + + public void testGetJobs_GivenSingleJob() throws Exception { + createFarequoteJob(); + + // Explicit _all + Response response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/_all"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":1")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); + + // Implicit _all + response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":1")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote\"")); + } + + public void testGetJobs_GivenMultipleJobs() throws Exception { + createFarequoteJob("farequote_1"); + createFarequoteJob("farequote_2"); + createFarequoteJob("farequote_3"); + + // Explicit _all + Response response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/_all"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":3")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote_1\"")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote_2\"")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote_3\"")); + + // Implicit _all + response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors"); + + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":3")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote_1\"")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote_2\"")); + assertThat(responseAsString, containsString("\"job_id\":\"farequote_3\"")); + } + + private Response createFarequoteJob() throws Exception { + return createFarequoteJob("farequote"); + } + + private Response createFarequoteJob(String jobId) throws Exception { + String job = "{\n" + " \"description\":\"Analysis of response time by airline\",\n" + + " \"analysis_config\" : {\n" + " \"bucket_span\":3600,\n" + + " \"detectors\" :[{\"function\":\"metric\",\"field_name\":\"responsetime\",\"by_field_name\":\"airline\"}]\n" + + " },\n" + " \"data_description\" : {\n" + " \"field_delimiter\":\",\",\n" + " " + + "\"time_field\":\"time\",\n" + + " \"time_format\":\"yyyy-MM-dd HH:mm:ssX\"\n" + " }\n" + "}"; + + return client().performRequest("put", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId, + Collections.emptyMap(), new StringEntity(job)); + } + + public void testGetBucketResults() throws Exception { + Map params = new HashMap<>(); + params.put("start", "1200"); // inclusive + params.put("end", "1400"); // exclusive + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/buckets", params)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("No known job with id '1'")); + + addBucketResult("1", "1234", 1); + addBucketResult("1", "1235", 1); + addBucketResult("1", "1236", 1); + Response response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/buckets", params); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":3")); + + params.put("end", "1235"); + response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/buckets", params); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":1")); + + e = expectThrows(ResponseException.class, () -> client().performRequest("get", MlPlugin.BASE_PATH + + "anomaly_detectors/2/results/buckets/1234")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("No known job with id '2'")); + + e = expectThrows(ResponseException.class, () -> client().performRequest("get", + MlPlugin.BASE_PATH + "anomaly_detectors/1/results/buckets/1")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + + response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/buckets/1234"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, not(isEmptyString())); + } + + public void testGetRecordResults() throws Exception { + Map params = new HashMap<>(); + params.put("start", "1200"); // inclusive + params.put("end", "1400"); // exclusive + + ResponseException e = expectThrows(ResponseException.class, + () -> client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/records", params)); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404)); + assertThat(e.getMessage(), containsString("No known job with id '1'")); + + addRecordResult("1", "1234", 1, 1); + addRecordResult("1", "1235", 1, 2); + addRecordResult("1", "1236", 1, 3); + Response response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/records", params); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":3")); + + params.put("end", "1235"); + response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/1/results/records", params); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":1")); + } + + public void testCreateJobWithIndexNameOption() throws Exception { + String jobTemplate = "{\n" + + " \"analysis_config\" : {\n" + + " \"detectors\" :[{\"function\":\"metric\",\"field_name\":\"responsetime\"}]\n" + + " },\n" + + " \"index_name\" : \"%s\"}"; + + String jobId = "aliased-job"; + String indexName = "non-default-index"; + String jobConfig = String.format(Locale.ROOT, jobTemplate, indexName); + + Response response = client().performRequest("put", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId, Collections.emptyMap(), + new StringEntity(jobConfig)); + assertEquals(200, response.getStatusLine().getStatusCode()); + + response = client().performRequest("get", "_aliases"); + assertEquals(200, response.getStatusLine().getStatusCode()); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"" + AnomalyDetectorsIndex.jobResultsIndexName(indexName) + + "\":{\"aliases\":{\"" + AnomalyDetectorsIndex.jobResultsIndexName(jobId) + "\"")); + + response = client().performRequest("get", "_cat/indices"); + assertEquals(200, response.getStatusLine().getStatusCode()); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString(indexName)); + + addBucketResult(indexName, "1234", 1); + addBucketResult(indexName, "1236", 1); + response = client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/results/buckets"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"count\":2")); + + response = client().performRequest("get", AnomalyDetectorsIndex.jobResultsIndexName(indexName) + "/result/_search"); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString("\"total\":2")); + + // test that we can't create another job with the same index_name + String jobConfigSameIndexName = String.format(Locale.ROOT, jobTemplate, "new-job-id", indexName); + expectThrows(ResponseException.class, () -> client().performRequest("put", + MlPlugin.BASE_PATH + "anomaly_detectors", Collections.emptyMap(), new StringEntity(jobConfigSameIndexName))); + + response = client().performRequest("delete", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + + // check index and alias were deleted + response = client().performRequest("get", "_aliases"); + assertEquals(200, response.getStatusLine().getStatusCode()); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, not(containsString(AnomalyDetectorsIndex.jobResultsIndexName(jobId)))); + + response = client().performRequest("get", "_cat/indices"); + assertEquals(200, response.getStatusLine().getStatusCode()); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, not(containsString(indexName))); + } + + public void testDeleteJob() throws Exception { + String jobId = "foo"; + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + createFarequoteJob(jobId); + + Response response = client().performRequest("get", "_cat/indices"); + assertEquals(200, response.getStatusLine().getStatusCode()); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString(indexName)); + + response = client().performRequest("delete", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + + // check index was deleted + response = client().performRequest("get", "_cat/indices"); + assertEquals(200, response.getStatusLine().getStatusCode()); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, not(containsString(indexName))); + + // check that the job itself is gone + expectThrows(ResponseException.class, () -> + client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")); + } + + public void testDeleteJobAfterMissingIndex() throws Exception { + String jobId = "foo"; + String indexName = AnomalyDetectorsIndex.jobResultsIndexName(jobId); + createFarequoteJob(jobId); + + Response response = client().performRequest("get", "_cat/indices"); + assertEquals(200, response.getStatusLine().getStatusCode()); + String responseAsString = responseEntityToString(response); + assertThat(responseAsString, containsString(indexName)); + + // Manually delete the index so that we can test that deletion proceeds + // normally anyway + response = client().performRequest("delete", indexName); + assertEquals(200, response.getStatusLine().getStatusCode()); + + response = client().performRequest("delete", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId); + assertThat(response.getStatusLine().getStatusCode(), equalTo(200)); + + // check index was deleted + response = client().performRequest("get", "_cat/indices"); + assertEquals(200, response.getStatusLine().getStatusCode()); + responseAsString = responseEntityToString(response); + assertThat(responseAsString, not(containsString(indexName))); + + expectThrows(ResponseException.class, () -> + client().performRequest("get", MlPlugin.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")); + } + + private Response addBucketResult(String jobId, String timestamp, long bucketSpan) throws Exception { + try { + client().performRequest("put", AnomalyDetectorsIndex.jobResultsIndexName(jobId), + Collections.emptyMap(), new StringEntity(RESULT_MAPPING)); + } catch (ResponseException e) { + // it is ok: the index already exists + assertThat(e.getMessage(), containsString("resource_already_exists_exception")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + } + + String bucketResult = String.format(Locale.ROOT, + "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"result_type\":\"bucket\", \"bucket_span\": \"%s\"}", + jobId, timestamp, bucketSpan); + String id = String.format(Locale.ROOT, + "%s_%s_%s", jobId, timestamp, bucketSpan); + return client().performRequest("put", AnomalyDetectorsIndex.jobResultsIndexName(jobId) + "/result/" + id, + Collections.singletonMap("refresh", "true"), new StringEntity(bucketResult)); + } + + private Response addRecordResult(String jobId, String timestamp, long bucketSpan, int sequenceNum) throws Exception { + try { + client().performRequest("put", AnomalyDetectorsIndex.jobResultsIndexName(jobId), Collections.emptyMap(), + new StringEntity(RESULT_MAPPING)); + } catch (ResponseException e) { + // it is ok: the index already exists + assertThat(e.getMessage(), containsString("resource_already_exists_exception")); + assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(400)); + } + + String recordResult = + String.format(Locale.ROOT, + "{\"job_id\":\"%s\", \"timestamp\": \"%s\", \"bucket_span\":%d, \"sequence_num\": %d, \"result_type\":\"record\"}", + jobId, timestamp, bucketSpan, sequenceNum); + return client().performRequest("put", AnomalyDetectorsIndex.jobResultsIndexName(jobId) + "/result/" + timestamp, + Collections.singletonMap("refresh", "true"), new StringEntity(recordResult)); + } + + private static String responseEntityToString(Response response) throws Exception { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(response.getEntity().getContent(), StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + + @After + public void clearMlState() throws IOException { + new MlRestTestStateCleaner(client(), this).clearMlMetadata(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlRestTestStateCleaner.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlRestTestStateCleaner.java new file mode 100644 index 00000000000..797bfddf7be --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/MlRestTestStateCleaner.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MlRestTestStateCleaner { + + private final RestClient client; + private final ESRestTestCase testCase; + + public MlRestTestStateCleaner(RestClient client, ESRestTestCase testCase) { + this.client = client; + this.testCase = testCase; + } + + public void clearMlMetadata() throws IOException { + deleteAllDatafeeds(); + deleteAllJobs(); + } + + @SuppressWarnings("unchecked") + private void deleteAllDatafeeds() throws IOException { + Map clusterStateAsMap = testCase.entityAsMap(client.performRequest("GET", "/_cluster/state", + Collections.singletonMap("filter_path", "metadata.ml.datafeeds"))); + List> datafeeds = + (List>) XContentMapValues.extractValue("metadata.ml.datafeeds", clusterStateAsMap); + if (datafeeds == null) { + return; + } + + for (Map datafeed : datafeeds) { + String datafeedId = (String) datafeed.get("datafeed_id"); + try { + client.performRequest("POST", "/_xpack/ml/datafeeds/" + datafeedId + "/_stop"); + } catch (Exception e) { + // ignore + } + client.performRequest("DELETE", "/_xpack/ml/datafeeds/" + datafeedId); + } + } + + private void deleteAllJobs() throws IOException { + Map clusterStateAsMap = testCase.entityAsMap(client.performRequest("GET", "/_cluster/state", + Collections.singletonMap("filter_path", "metadata.ml.jobs"))); + @SuppressWarnings("unchecked") + List> jobConfigs = + (List>) XContentMapValues.extractValue("metadata.ml.jobs", clusterStateAsMap); + if (jobConfigs == null) { + return; + } + + for (Map jobConfig : jobConfigs) { + String jobId = (String) jobConfig.get("job_id"); + try { + client.performRequest("POST", "/_xpack/ml/anomaly_detectors/" + jobId + "/_close"); + } catch (Exception e) { + // ignore + } + client.performRequest("DELETE", "/_xpack/ml/anomaly_detectors/" + jobId); + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java new file mode 100644 index 00000000000..862200a1a79 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.integration; + +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.ml.MlPlugin; +import org.elasticsearch.xpack.ml.action.DatafeedJobsIT; +import org.elasticsearch.xpack.ml.action.GetJobsStatsAction; +import org.elasticsearch.xpack.ml.action.OpenJobAction; +import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.DataDescription; +import org.elasticsearch.xpack.ml.job.config.Detector; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager; +import org.elasticsearch.xpack.persistent.PersistentActionRequest; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; +import org.junit.After; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.XContentTestUtils.convertToMap; +import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; + +@ESIntegTestCase.ClusterScope(numDataNodes = 1) +public class TooManyJobsIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Collections.singleton(MlPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return nodePlugins(); + } + + @After + public void clearMlMetadata() throws Exception { + DatafeedJobsIT.clearMlMetadata(client()); + } + + public void testCannotStartTooManyAnalyticalProcesses() throws Exception { + int maxRunningJobsPerNode = AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getDefault(Settings.EMPTY); + logger.info("[{}] is [{}]", AutodetectProcessManager.MAX_RUNNING_JOBS_PER_NODE.getKey(), maxRunningJobsPerNode); + for (int i = 1; i <= (maxRunningJobsPerNode + 1); i++) { + Job.Builder job = createJob(Integer.toString(i)); + PutJobAction.Request putJobRequest = new PutJobAction.Request(job.build(true, job.getId())); + PutJobAction.Response putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); + assertTrue(putJobResponse.isAcknowledged()); + + try { + OpenJobAction.Request openJobRequest = new OpenJobAction.Request(job.getId()); + OpenJobAction.Response openJobResponse = client().execute(OpenJobAction.INSTANCE, openJobRequest).get(); + assertTrue(openJobResponse.isOpened()); + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); + assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED); + }); + logger.info("Opened {}th job", i); + } catch (Exception e) { + Throwable cause = e.getCause().getCause(); + if (IllegalArgumentException.class.equals(cause.getClass()) == false) { + logger.warn("Unexpected cause", e); + } + assertEquals(IllegalArgumentException.class, cause.getClass()); + assertEquals("Timeout expired while waiting for job state to change to [opened]", cause.getMessage()); + logger.info("good news everybody --> reached maximum number of allowed opened jobs, after trying to open the {}th job", i); + + // now manually clean things up and see if we can succeed to run one new job + clearMlMetadata(); + putJobResponse = client().execute(PutJobAction.INSTANCE, putJobRequest).get(); + assertTrue(putJobResponse.isAcknowledged()); + OpenJobAction.Response openJobResponse = client().execute(OpenJobAction.INSTANCE, new OpenJobAction.Request(job.getId())) + .get(); + assertTrue(openJobResponse.isOpened()); + assertBusy(() -> { + GetJobsStatsAction.Response statsResponse = + client().execute(GetJobsStatsAction.INSTANCE, new GetJobsStatsAction.Request(job.getId())).actionGet(); + assertEquals(statsResponse.getResponse().results().get(0).getState(), JobState.OPENED); + }); + return; + } + } + + fail("shouldn't be able to add more than [" + maxRunningJobsPerNode + "] jobs"); + } + + private Job.Builder createJob(String id) { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(DataDescription.DataFormat.JSON); + dataDescription.setTimeFormat(DataDescription.EPOCH_MS); + + Detector.Builder d = new Detector.Builder("count", null); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(d.build())); + + Job.Builder builder = new Job.Builder(); + builder.setId(id); + + builder.setAnalysisConfig(analysisConfig); + builder.setDataDescription(dataDescription); + return builder; + } + + @Override + protected void ensureClusterStateConsistency() throws IOException { + ensureClusterStateConsistencyWorkAround(); + } + + // TODO: Fix in ES. In ESIntegTestCase we should get all NamedWriteableRegistry.Entry entries from ESIntegTestCase#nodePlugins() + public static void ensureClusterStateConsistencyWorkAround() throws IOException { + if (cluster() != null && cluster().size() > 0) { + List namedWritables = new ArrayList<>(ClusterModule.getNamedWriteables()); + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + namedWritables.addAll(searchModule.getNamedWriteables()); + namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new)); + namedWritables.add(new NamedWriteableRegistry.Entry(ClusterState.Custom.class, PersistentTasksInProgress.TYPE, + PersistentTasksInProgress::new)); + namedWritables.add(new NamedWriteableRegistry.Entry(PersistentActionRequest.class, StartDatafeedAction.NAME, + StartDatafeedAction.Request::new)); + final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(namedWritables); + ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState(); + byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); + // remove local node reference + masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); + Map masterStateMap = convertToMap(masterClusterState); + int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; + String masterId = masterClusterState.nodes().getMasterNodeId(); + for (Client client : cluster().getClients()) { + ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState(); + byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState); + // remove local node reference + localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry); + final Map localStateMap = convertToMap(localClusterState); + final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; + // Check that the non-master node has the same version of the cluster state as the master and + // that the master node matches the master (otherwise there is no requirement for the cluster state to match) + if (masterClusterState.version() == localClusterState.version() && + masterId.equals(localClusterState.nodes().getMasterNodeId())) { + try { + assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), + localClusterState.stateUUID()); + // We cannot compare serialization bytes since serialization order of maps is not guaranteed + // but we can compare serialization sizes - they should be the same + assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize); + // Compare JSON serialization + assertNull("clusterstate JSON serialization does not match", + differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)); + } catch (AssertionError error) { + fail("Cluster state from master:\n" + masterClusterState.toString() + "\nLocal cluster state:\n" + + localClusterState.toString()); + throw error; + } + } + } + } + + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java new file mode 100644 index 00000000000..3c8cfa06cde --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job; + +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.AckedClusterStateUpdateTask; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.AliasMetaData; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.Index; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.metadata.MlMetadata; +import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; +import org.junit.Before; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class JobManagerTests extends ESTestCase { + + private ClusterService clusterService; + private JobProvider jobProvider; + + @Before + public void setupMocks() { + clusterService = mock(ClusterService.class); + jobProvider = mock(JobProvider.class); + Auditor auditor = mock(Auditor.class); + when(jobProvider.audit(anyString())).thenReturn(auditor); + } + + public void testGetJob() { + JobManager jobManager = createJobManager(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(buildJobBuilder("foo").build(), false); + ClusterState clusterState = ClusterState.builder(new ClusterName("name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, builder.build())).build(); + QueryPage doc = jobManager.getJob("foo", clusterState); + assertTrue(doc.count() > 0); + assertThat(doc.results().get(0).getId(), equalTo("foo")); + } + + public void testFilter() { + Set running = new HashSet<>(Arrays.asList("henry", "dim", "dave")); + Set diff = new HashSet<>(Arrays.asList("dave", "tom")).stream().filter((s) -> !running.contains(s)) + .collect(Collectors.toCollection(HashSet::new)); + + assertTrue(diff.size() == 1); + assertTrue(diff.contains("tom")); + } + + public void testGetJobOrThrowIfUnknown_GivenUnknownJob() { + JobManager jobManager = createJobManager(); + ClusterState cs = createClusterState(); + ESTestCase.expectThrows(ResourceNotFoundException.class, () -> jobManager.getJobOrThrowIfUnknown(cs, "foo")); + } + + public void testGetJobOrThrowIfUnknown_GivenKnownJob() { + JobManager jobManager = createJobManager(); + Job job = buildJobBuilder("foo").build(); + MlMetadata mlMetadata = new MlMetadata.Builder().putJob(job, false).build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)).build(); + + assertEquals(job, jobManager.getJobOrThrowIfUnknown(cs, "foo")); + } + + public void tesGetJobAllocation() { + JobManager jobManager = createJobManager(); + Job job = buildJobBuilder("foo").build(); + MlMetadata mlMetadata = new MlMetadata.Builder() + .putJob(job, false) + .assignToNode("foo", "nodeId") + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata)).build(); + when(clusterService.state()).thenReturn(cs); + + assertEquals("nodeId", jobManager.getJobAllocation("foo").getNodeId()); + expectThrows(ResourceNotFoundException.class, () -> jobManager.getJobAllocation("bar")); + } + + public void testGetJob_GivenJobIdIsAll() { + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + for (int i = 0; i < 3; i++) { + mlMetadata.putJob(buildJobBuilder(Integer.toString(i)).build(), false); + } + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata.build())).build(); + + JobManager jobManager = createJobManager(); + QueryPage result = jobManager.getJob("_all", clusterState); + assertThat(result.count(), equalTo(3L)); + assertThat(result.results().get(0).getId(), equalTo("0")); + assertThat(result.results().get(1).getId(), equalTo("1")); + assertThat(result.results().get(2).getId(), equalTo("2")); + } + + public void testGetJobs() { + MlMetadata.Builder mlMetadata = new MlMetadata.Builder(); + for (int i = 0; i < 10; i++) { + mlMetadata.putJob(buildJobBuilder(Integer.toString(i)).build(), false); + } + ClusterState clusterState = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, mlMetadata.build())).build(); + + JobManager jobManager = createJobManager(); + QueryPage result = jobManager.getJobs(clusterState); + assertThat(result.count(), equalTo(10L)); + assertThat(result.results().get(0).getId(), equalTo("0")); + assertThat(result.results().get(1).getId(), equalTo("1")); + assertThat(result.results().get(2).getId(), equalTo("2")); + assertThat(result.results().get(3).getId(), equalTo("3")); + assertThat(result.results().get(4).getId(), equalTo("4")); + assertThat(result.results().get(5).getId(), equalTo("5")); + assertThat(result.results().get(6).getId(), equalTo("6")); + assertThat(result.results().get(7).getId(), equalTo("7")); + assertThat(result.results().get(8).getId(), equalTo("8")); + assertThat(result.results().get(9).getId(), equalTo("9")); + } + + @SuppressWarnings("unchecked") + public void testPutJobFailsIfIndexExists() { + JobManager jobManager = createJobManager(); + Job.Builder jobBuilder = buildJobBuilder("foo"); + jobBuilder.setIndexName("my-special-place"); + PutJobAction.Request request = new PutJobAction.Request(jobBuilder.build()); + + Index index = mock(Index.class); + when(index.getName()).thenReturn(AnomalyDetectorsIndex.jobResultsIndexName("my-special-place")); + IndexMetaData indexMetaData = mock(IndexMetaData.class); + when(indexMetaData.getIndex()).thenReturn(index); + ImmutableOpenMap aliases = ImmutableOpenMap.of(); + when(indexMetaData.getAliases()).thenReturn(aliases); + + ImmutableOpenMap indexMap = ImmutableOpenMap.builder() + .fPut(AnomalyDetectorsIndex.jobResultsIndexName("my-special-place"), indexMetaData).build(); + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metaData(MetaData.builder().putCustom(MlMetadata.TYPE, MlMetadata.EMPTY_METADATA).indices(indexMap)).build(); + + doAnswer(invocationOnMock -> { + AckedClusterStateUpdateTask task = (AckedClusterStateUpdateTask) invocationOnMock.getArguments()[1]; + task.execute(cs); + return null; + }).when(clusterService).submitStateUpdateTask(eq("put-job-foo"), any(AckedClusterStateUpdateTask.class)); + + ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, () -> jobManager.putJob(request, + new ActionListener() { + @Override + public void onResponse(PutJobAction.Response response) { + } + + @Override + public void onFailure(Exception e) { + fail(e.toString()); + } + })); + + assertEquals("Cannot create index '.ml-anomalies-my-special-place' as it already exists", e.getMessage()); + } + + + private JobManager createJobManager() { + Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build(); + JobResultsPersister jobResultsPersister = mock(JobResultsPersister.class); + return new JobManager(settings, jobProvider, jobResultsPersister, clusterService); + } + + private ClusterState createClusterState() { + ClusterState.Builder builder = ClusterState.builder(new ClusterName("_name")); + builder.metaData(MetaData.builder().putCustom(MlMetadata.TYPE, MlMetadata.EMPTY_METADATA)); + return builder.build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java new file mode 100644 index 00000000000..6e09ea2995e --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java @@ -0,0 +1,802 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + + +public class AnalysisConfigTests extends AbstractSerializingTestCase { + + @Override + protected AnalysisConfig createTestInstance() { + List detectors = new ArrayList<>(); + int numDetectors = randomIntBetween(1, 10); + for (int i = 0; i < numDetectors; i++) { + detectors.add(new Detector.Builder("count", null).build()); + } + AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors); + + + if (randomBoolean()) { + builder.setBatchSpan(randomNonNegativeLong()); + } + long bucketSpan = AnalysisConfig.Builder.DEFAULT_BUCKET_SPAN; + if (randomBoolean()) { + bucketSpan = randomIntBetween(1, 1_000_000); + builder.setBucketSpan(bucketSpan); + } + if (randomBoolean()) { + builder.setCategorizationFieldName(randomAsciiOfLength(10)); + builder.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); + } + if (randomBoolean()) { + builder.setInfluencers(Arrays.asList(generateRandomStringArray(10, 10, false))); + } + if (randomBoolean()) { + builder.setLatency(randomNonNegativeLong()); + } + if (randomBoolean()) { + int numBucketSpans = randomIntBetween(0, 10); + List multipleBucketSpans = new ArrayList<>(); + for (int i = 2; i <= numBucketSpans; i++) { + multipleBucketSpans.add(bucketSpan * i); + } + builder.setMultipleBucketSpans(multipleBucketSpans); + } + if (randomBoolean()) { + builder.setMultivariateByFields(randomBoolean()); + } + if (randomBoolean()) { + builder.setOverlappingBuckets(randomBoolean()); + } + if (randomBoolean()) { + builder.setResultFinalizationWindow(randomNonNegativeLong()); + } + + builder.setUsePerPartitionNormalization(false); + return builder.build(); + } + + @Override + protected Writeable.Reader instanceReader() { + return AnalysisConfig::new; + } + + @Override + protected AnalysisConfig parseInstance(XContentParser parser) { + return AnalysisConfig.PARSER.apply(parser, null).build(); + } + + public void testFieldConfiguration_singleDetector_notPreSummarised() { + // Single detector, not pre-summarised + Detector.Builder det = new Detector.Builder("metric", "responsetime"); + det.setByFieldName("airline"); + det.setPartitionFieldName("sourcetype"); + AnalysisConfig ac = createConfigWithDetectors(Collections.singletonList(det.build())); + + Set termFields = new TreeSet<>(Arrays.asList(new String[]{ + "airline", "sourcetype"})); + Set analysisFields = new TreeSet<>(Arrays.asList(new String[]{ + "responsetime", "airline", "sourcetype"})); + + assertEquals(termFields.size(), ac.termFields().size()); + assertEquals(analysisFields.size(), ac.analysisFields().size()); + + for (String s : ac.termFields()) { + assertTrue(termFields.contains(s)); + } + + for (String s : termFields) { + assertTrue(ac.termFields().contains(s)); + } + + for (String s : ac.analysisFields()) { + assertTrue(analysisFields.contains(s)); + } + + for (String s : analysisFields) { + assertTrue(ac.analysisFields().contains(s)); + } + + assertEquals(1, ac.fields().size()); + assertTrue(ac.fields().contains("responsetime")); + + assertEquals(1, ac.byFields().size()); + assertTrue(ac.byFields().contains("airline")); + + assertEquals(1, ac.partitionFields().size()); + assertTrue(ac.partitionFields().contains("sourcetype")); + + assertNull(ac.getSummaryCountFieldName()); + + // Single detector, pre-summarised + analysisFields.add("summaryCount"); + AnalysisConfig.Builder builder = new AnalysisConfig.Builder(ac); + builder.setSummaryCountFieldName("summaryCount"); + ac = builder.build(); + + for (String s : ac.analysisFields()) { + assertTrue(analysisFields.contains(s)); + } + + for (String s : analysisFields) { + assertTrue(ac.analysisFields().contains(s)); + } + + assertEquals("summaryCount", ac.getSummaryCountFieldName()); + } + + public void testFieldConfiguration_multipleDetectors_NotPreSummarised() { + // Multiple detectors, not pre-summarised + List detectors = new ArrayList<>(); + + Detector.Builder det = new Detector.Builder("metric", "metric1"); + det.setByFieldName("by_one"); + det.setPartitionFieldName("partition_one"); + detectors.add(det.build()); + + det = new Detector.Builder("metric", "metric2"); + det.setByFieldName("by_two"); + det.setOverFieldName("over_field"); + detectors.add(det.build()); + + det = new Detector.Builder("metric", "metric2"); + det.setByFieldName("by_two"); + det.setPartitionFieldName("partition_two"); + detectors.add(det.build()); + + AnalysisConfig.Builder builder = new AnalysisConfig.Builder(detectors); + builder.setInfluencers(Collections.singletonList("Influencer_Field")); + AnalysisConfig ac = builder.build(); + + + Set termFields = new TreeSet<>(Arrays.asList(new String[]{ + "by_one", "by_two", "over_field", + "partition_one", "partition_two", "Influencer_Field"})); + Set analysisFields = new TreeSet<>(Arrays.asList(new String[]{ + "metric1", "metric2", "by_one", "by_two", "over_field", + "partition_one", "partition_two", "Influencer_Field"})); + + assertEquals(termFields.size(), ac.termFields().size()); + assertEquals(analysisFields.size(), ac.analysisFields().size()); + + for (String s : ac.termFields()) { + assertTrue(s, termFields.contains(s)); + } + + for (String s : termFields) { + assertTrue(s, ac.termFields().contains(s)); + } + + for (String s : ac.analysisFields()) { + assertTrue(analysisFields.contains(s)); + } + + for (String s : analysisFields) { + assertTrue(ac.analysisFields().contains(s)); + } + + assertEquals(2, ac.fields().size()); + assertTrue(ac.fields().contains("metric1")); + assertTrue(ac.fields().contains("metric2")); + + assertEquals(2, ac.byFields().size()); + assertTrue(ac.byFields().contains("by_one")); + assertTrue(ac.byFields().contains("by_two")); + + assertEquals(1, ac.overFields().size()); + assertTrue(ac.overFields().contains("over_field")); + + assertEquals(2, ac.partitionFields().size()); + assertTrue(ac.partitionFields().contains("partition_one")); + assertTrue(ac.partitionFields().contains("partition_two")); + + assertNull(ac.getSummaryCountFieldName()); + } + + public void testFieldConfiguration_multipleDetectors_PreSummarised() { + // Multiple detectors, pre-summarised + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setSummaryCountFieldName("summaryCount"); + AnalysisConfig ac = builder.build(); + + assertTrue(ac.analysisFields().contains("summaryCount")); + assertEquals("summaryCount", ac.getSummaryCountFieldName()); + + builder = createConfigBuilder(); + builder.setBucketSpan(1000L); + builder.setMultipleBucketSpans(Arrays.asList(5000L, 10000L, 24000L)); + ac = builder.build(); + assertTrue(ac.getMultipleBucketSpans().contains(5000L)); + assertTrue(ac.getMultipleBucketSpans().contains(10000L)); + assertTrue(ac.getMultipleBucketSpans().contains(24000L)); + } + + + public void testEquals_GivenSameReference() { + AnalysisConfig config = createFullyPopulatedConfig(); + assertTrue(config.equals(config)); + } + + public void testEquals_GivenDifferentClass() { + + assertFalse(createFullyPopulatedConfig().equals("a string")); + } + + + public void testEquals_GivenNull() { + assertFalse(createFullyPopulatedConfig().equals(null)); + } + + + public void testEquals_GivenEqualConfig() { + AnalysisConfig config1 = createFullyPopulatedConfig(); + AnalysisConfig config2 = createFullyPopulatedConfig(); + + assertTrue(config1.equals(config2)); + assertTrue(config2.equals(config1)); + assertEquals(config1.hashCode(), config2.hashCode()); + } + + + public void testEquals_GivenDifferentBatchSpan() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setBatchSpan(86400L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setBatchSpan(0L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentBucketSpan() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setBucketSpan(1800L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setBucketSpan(3600L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenCategorizationField() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setCategorizationFieldName("foo"); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setCategorizationFieldName("bar"); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentDetector() { + AnalysisConfig config1 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "low_count").build())); + + AnalysisConfig config2 = createConfigWithDetectors(Collections.singletonList(new Detector.Builder("min", "high_count").build())); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentInfluencers() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setInfluencers(Arrays.asList("foo")); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setInfluencers(Arrays.asList("bar")); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentLatency() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setLatency(1800L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setLatency(3600L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentPeriod() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setPeriod(1800L); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setPeriod(3600L); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenSummaryCountField() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setSummaryCountFieldName("foo"); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setSummaryCountFieldName("bar"); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenMultivariateByField() { + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setMultivariateByFields(true); + AnalysisConfig config1 = builder.build(); + + builder = createConfigBuilder(); + builder.setMultivariateByFields(false); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + + public void testEquals_GivenDifferentCategorizationFilters() { + AnalysisConfig config1 = createFullyPopulatedConfig(); + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setCategorizationFilters(Arrays.asList("foo", "bar")); + builder.setCategorizationFieldName("cat"); + AnalysisConfig config2 = builder.build(); + + assertFalse(config1.equals(config2)); + assertFalse(config2.equals(config1)); + } + + public void testBucketSpanOrDefault() { + AnalysisConfig config1 = new AnalysisConfig.Builder( + Collections.singletonList(new Detector.Builder("min", "count").build())).build(); + assertEquals(AnalysisConfig.Builder.DEFAULT_BUCKET_SPAN, config1.getBucketSpanOrDefault()); + AnalysisConfig.Builder builder = createConfigBuilder(); + builder.setBucketSpan(100L); + config1 = builder.build(); + assertEquals(100L, config1.getBucketSpanOrDefault()); + } + + public void testExtractReferencedLists() { + DetectionRule rule1 = + new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("foo", "filter1"))); + DetectionRule rule2 = + new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("foo", "filter2"))); + Detector.Builder detector1 = new Detector.Builder("count", null); + detector1.setByFieldName("foo"); + detector1.setDetectorRules(Arrays.asList(rule1)); + Detector.Builder detector2 = new Detector.Builder("count", null); + detector2.setDetectorRules(Arrays.asList(rule2)); + detector2.setByFieldName("foo"); + AnalysisConfig config = new AnalysisConfig.Builder( + Arrays.asList(detector1.build(), detector2.build(), new Detector.Builder("count", null).build())).build(); + + assertEquals(new HashSet<>(Arrays.asList("filter1", "filter2")), config.extractReferencedFilters()); + } + + private static AnalysisConfig createFullyPopulatedConfig() { + AnalysisConfig.Builder builder = new AnalysisConfig.Builder( + Collections.singletonList(new Detector.Builder("min", "count").build())); + builder.setBatchSpan(86400L); + builder.setBucketSpan(3600L); + builder.setCategorizationFieldName("cat"); + builder.setCategorizationFilters(Arrays.asList("foo")); + builder.setInfluencers(Arrays.asList("myInfluencer")); + builder.setLatency(3600L); + builder.setPeriod(100L); + builder.setSummaryCountFieldName("sumCount"); + return builder.build(); + } + + private static AnalysisConfig createConfigWithDetectors(List detectors) { + return new AnalysisConfig.Builder(detectors).build(); + } + + private static AnalysisConfig.Builder createConfigBuilder() { + return new AnalysisConfig.Builder(Collections.singletonList(new Detector.Builder("min", "count").build())); + } + + public void testVerify_throws() { + + // count works with no fields + Detector d = new Detector.Builder("count", null).build(); + new AnalysisConfig.Builder(Collections.singletonList(d)).build(); + + try { + d = new Detector.Builder("distinct_count", null).build(); + new AnalysisConfig.Builder(Collections.singletonList(d)).build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("Unless the function is 'count' one of field_name, by_field_name or over_field_name must be set", e.getMessage()); + } + + // should work now + Detector.Builder builder = new Detector.Builder("distinct_count", "somefield"); + builder.setOverFieldName("over"); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + + builder = new Detector.Builder("info_content", "somefield"); + builder.setOverFieldName("over"); + d = builder.build(); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + + builder.setByFieldName("by"); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + + try { + builder = new Detector.Builder("made_up_function", "somefield"); + builder.setOverFieldName("over"); + new AnalysisConfig.Builder(Collections.singletonList(builder.build())).build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("Unknown function 'made_up_function'", e.getMessage()); + } + + builder = new Detector.Builder("distinct_count", "somefield"); + AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(Collections.singletonList(builder.build())); + acBuilder.setBatchSpan(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("batch_span cannot be less than 0. Value = -1", e.getMessage()); + } + + acBuilder.setBatchSpan(10L); + acBuilder.setBucketSpan(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("bucket_span cannot be less than 0. Value = -1", e.getMessage()); + } + + acBuilder.setBucketSpan(3600L); + acBuilder.setPeriod(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("period cannot be less than 0. Value = -1", e.getMessage()); + } + + acBuilder.setPeriod(1L); + acBuilder.setLatency(-1L); + try { + acBuilder.build(); + assertTrue(false); // shouldn't get here + } catch (IllegalArgumentException e) { + assertEquals("latency cannot be less than 0. Value = -1", e.getMessage()); + } + } + + public void testVerify_GivenNegativeBucketSpan() { + AnalysisConfig.Builder config = createValidConfig(); + config.setBucketSpan(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "bucket_span", 0, -1), e.getMessage()); + } + + public void testVerify_GivenNegativeBatchSpan() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setBatchSpan(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "batch_span", 0, -1), e.getMessage()); + } + + + public void testVerify_GivenNegativeLatency() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setLatency(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "latency", 0, -1), e.getMessage()); + } + + + public void testVerify_GivenNegativePeriod() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setPeriod(-1L); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "period", 0, -1), e.getMessage()); + } + + + public void testVerify_GivenDefaultConfig_ShouldBeInvalidDueToNoDetectors() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setDetectors(null); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> analysisConfig.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_NO_DETECTORS), e.getMessage()); + } + + + public void testVerify_GivenValidConfig() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.build(); + } + + + public void testVerify_GivenValidConfigWithCategorizationFieldNameAndCategorizationFilters() { + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setCategorizationFieldName("myCategory"); + analysisConfig.setCategorizationFilters(Arrays.asList("foo", "bar")); + + analysisConfig.build(); + } + + + public void testVerify_OverlappingBuckets() { + List detectors; + Detector detector; + + boolean onByDefault = false; + + // Uncomment this when overlappingBuckets turned on by default + if (onByDefault) { + // Test overlappingBuckets unset + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("mean", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + AnalysisConfig ac = analysisConfig.build(); + assertTrue(ac.getOverlappingBuckets()); + + // Test overlappingBuckets unset + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("rare", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + ac = analysisConfig.build(); + assertFalse(ac.getOverlappingBuckets()); + + // Test overlappingBuckets unset + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("min", "value").build(); + detectors.add(detector); + detector = new Detector.Builder("max", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + ac = analysisConfig.build(); + assertFalse(ac.getOverlappingBuckets()); + } + + // Test overlappingBuckets set + AnalysisConfig.Builder analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + Detector.Builder builder = new Detector.Builder("rare", null); + builder.setByFieldName("value"); + detectors.add(builder.build()); + analysisConfig.setOverlappingBuckets(false); + analysisConfig.setDetectors(detectors); + assertFalse(analysisConfig.build().getOverlappingBuckets()); + + // Test overlappingBuckets set + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + analysisConfig.setOverlappingBuckets(true); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + builder = new Detector.Builder("rare", null); + builder.setByFieldName("value"); + detectors.add(builder.build()); + analysisConfig.setDetectors(detectors); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, analysisConfig::build); + assertEquals("Overlapping buckets cannot be used with function '[rare]'", e.getMessage()); + + // Test overlappingBuckets set + analysisConfig = createValidConfig(); + analysisConfig.setBucketSpan(5000L); + analysisConfig.setBatchSpan(0L); + analysisConfig.setOverlappingBuckets(false); + detectors = new ArrayList<>(); + detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + detector = new Detector.Builder("mean", "value").build(); + detectors.add(detector); + analysisConfig.setDetectors(detectors); + AnalysisConfig ac = analysisConfig.build(); + assertFalse(ac.getOverlappingBuckets()); + } + + + public void testMultipleBucketsConfig() { + AnalysisConfig.Builder ac = createValidConfig(); + ac.setMultipleBucketSpans(Arrays.asList(10L, 15L, 20L, 25L, 30L, 35L)); + List detectors = new ArrayList<>(); + Detector detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + ac.setDetectors(detectors); + + ac.setBucketSpan(4L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, ac::build); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, 10, 4), e.getMessage()); + + ac.setBucketSpan(5L); + ac.build(); + + AnalysisConfig.Builder ac2 = createValidConfig(); + ac2.setBucketSpan(5L); + ac2.setDetectors(detectors); + ac2.setMultipleBucketSpans(Arrays.asList(10L, 15L, 20L, 25L, 30L)); + assertFalse(ac.equals(ac2)); + ac2.setMultipleBucketSpans(Arrays.asList(10L, 15L, 20L, 25L, 30L, 35L)); + + ac.setBucketSpan(222L); + ac.setMultipleBucketSpans(Arrays.asList()); + ac.build(); + + ac.setMultipleBucketSpans(Arrays.asList(222L)); + e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> ac.build()); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, 222, 222), e.getMessage()); + + ac.setMultipleBucketSpans(Arrays.asList(-444L, -888L)); + e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> ac.build()); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MULTIPLE_BUCKETSPANS_MUST_BE_MULTIPLE, -444, 222), e.getMessage()); + } + + + public void testVerify_GivenCategorizationFiltersButNoCategorizationFieldName() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFilters(Arrays.asList("foo")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_REQUIRE_CATEGORIZATION_FIELD_NAME), e.getMessage()); + } + + + public void testVerify_GivenDuplicateCategorizationFilters() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFieldName("myCategory"); + config.setCategorizationFilters(Arrays.asList("foo", "bar", "foo")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES), e.getMessage()); + } + + + public void testVerify_GivenEmptyCategorizationFilter() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFieldName("myCategory"); + config.setCategorizationFilters(Arrays.asList("foo", "")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_EMPTY), e.getMessage()); + } + + + public void testCheckDetectorsHavePartitionFields() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setUsePerPartitionNormalization(true); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_REQUIRES_PARTITION_FIELD), e.getMessage()); + } + + + public void testCheckDetectorsHavePartitionFields_doesntThrowWhenValid() { + AnalysisConfig.Builder config = createValidConfig(); + Detector.Builder builder = new Detector.Builder(config.build().getDetectors().get(0)); + builder.setPartitionFieldName("pField"); + config.build().getDetectors().set(0, builder.build()); + config.setUsePerPartitionNormalization(true); + + config.build(); + } + + + public void testCheckNoInfluencersAreSet() { + + AnalysisConfig.Builder config = createValidConfig(); + Detector.Builder builder = new Detector.Builder(config.build().getDetectors().get(0)); + builder.setPartitionFieldName("pField"); + config.build().getDetectors().set(0, builder.build()); + config.setInfluencers(Arrays.asList("inf1", "inf2")); + config.setUsePerPartitionNormalization(true); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_PER_PARTITION_NORMALIZATION_CANNOT_USE_INFLUENCERS), e.getMessage()); + } + + + public void testVerify_GivenCategorizationFiltersContainInvalidRegex() { + + AnalysisConfig.Builder config = createValidConfig(); + config.setCategorizationFieldName("myCategory"); + config.setCategorizationFilters(Arrays.asList("foo", "(")); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> config.build()); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_INVALID_REGEX, "("), e.getMessage()); + } + + private static AnalysisConfig.Builder createValidConfig() { + List detectors = new ArrayList<>(); + Detector detector = new Detector.Builder("count", null).build(); + detectors.add(detector); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(detectors); + analysisConfig.setBucketSpan(3600L); + analysisConfig.setBatchSpan(0L); + analysisConfig.setLatency(0L); + analysisConfig.setPeriod(0L); + return analysisConfig; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java new file mode 100644 index 00000000000..7fe07f8c0bc --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisLimitsTests.java @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +public class AnalysisLimitsTests extends AbstractSerializingTestCase { + + @Override + protected AnalysisLimits createTestInstance() { + return new AnalysisLimits(randomBoolean() ? randomLong() : null, randomBoolean() ? randomNonNegativeLong() : null); + } + + @Override + protected Writeable.Reader instanceReader() { + return AnalysisLimits::new; + } + + @Override + protected AnalysisLimits parseInstance(XContentParser parser) { + return AnalysisLimits.PARSER.apply(parser, null); + } + + public void testEquals_GivenEqual() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(10L, 20L); + + assertTrue(analysisLimits1.equals(analysisLimits1)); + assertTrue(analysisLimits1.equals(analysisLimits2)); + assertTrue(analysisLimits2.equals(analysisLimits1)); + } + + + public void testEquals_GivenDifferentModelMemoryLimit() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(11L, 20L); + + assertFalse(analysisLimits1.equals(analysisLimits2)); + assertFalse(analysisLimits2.equals(analysisLimits1)); + } + + + public void testEquals_GivenDifferentCategorizationExamplesLimit() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(10L, 20L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(10L, 21L); + + assertFalse(analysisLimits1.equals(analysisLimits2)); + assertFalse(analysisLimits2.equals(analysisLimits1)); + } + + + public void testHashCode_GivenEqual() { + AnalysisLimits analysisLimits1 = new AnalysisLimits(5555L, 3L); + AnalysisLimits analysisLimits2 = new AnalysisLimits(5555L, 3L); + + assertEquals(analysisLimits1.hashCode(), analysisLimits2.hashCode()); + } + + public void testVerify_GivenNegativeCategorizationExamplesLimit() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new AnalysisLimits(1L, -1L)); + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, + AnalysisLimits.CATEGORIZATION_EXAMPLES_LIMIT, 0, -1L); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenValid() { + new AnalysisLimits(0L, 0L); + new AnalysisLimits(1L, null); + new AnalysisLimits(1L, 1L); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ConditionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ConditionTests.java new file mode 100644 index 00000000000..526869ed233 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ConditionTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import static org.hamcrest.Matchers.containsString; + +public class ConditionTests extends AbstractSerializingTestCase { + + public void testSetValues() { + Condition cond = new Condition(Operator.EQ, "5"); + assertEquals(Operator.EQ, cond.getOperator()); + assertEquals("5", cond.getValue()); + } + + public void testHashCodeAndEquals() { + Condition cond1 = new Condition(Operator.MATCH, "regex"); + Condition cond2 = new Condition(Operator.MATCH, "regex"); + + assertEquals(cond1, cond2); + assertEquals(cond1.hashCode(), cond2.hashCode()); + + Condition cond3 = new Condition(Operator.EQ, "5"); + assertFalse(cond1.equals(cond3)); + assertFalse(cond1.hashCode() == cond3.hashCode()); + } + + @Override + protected Condition createTestInstance() { + Operator op = randomFrom(Operator.values()); + Condition condition; + switch (op) { + case EQ: + case GT: + case GTE: + case LT: + case LTE: + condition = new Condition(op, Double.toString(randomDouble())); + break; + case MATCH: + condition = new Condition(op, randomAsciiOfLengthBetween(1, 20)); + break; + default: + throw new AssertionError("Unknown operator selected: " + op); + } + return condition; + } + + @Override + protected Reader instanceReader() { + return Condition::new; + } + + @Override + protected Condition parseInstance(XContentParser parser) { + return Condition.PARSER.apply(parser, null); + } + + public void testVerifyArgsNumericArgs() { + new Condition(Operator.LTE, "100"); + new Condition(Operator.GT, "10.0"); + } + + public void testVerify_GivenEmptyValue() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new Condition(Operator.LT, "")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, ""), e.getMessage()); + } + + public void testVerify_GivenInvalidRegex() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new Condition(Operator.MATCH, "[*")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX, "[*"), e.getMessage()); + } + + public void testVerify_GivenNullRegex() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, + () -> new Condition(Operator.MATCH, null)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NULL, "[*"), e.getMessage()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ConnectiveTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ConnectiveTests.java new file mode 100644 index 00000000000..0b34d489512 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ConnectiveTests.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class ConnectiveTests extends ESTestCase { + + public void testForString() { + assertEquals(Connective.OR, Connective.fromString("or")); + assertEquals(Connective.OR, Connective.fromString("OR")); + assertEquals(Connective.AND, Connective.fromString("and")); + assertEquals(Connective.AND, Connective.fromString("AND")); + } + + public void testToString() { + assertEquals("or", Connective.OR.toString()); + assertEquals("and", Connective.AND.toString()); + } + + public void testValidOrdinals() { + assertThat(Connective.OR.ordinal(), equalTo(0)); + assertThat(Connective.AND.ordinal(), equalTo(1)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Connective.OR.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Connective.AND.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Connective.readFromStream(in), equalTo(Connective.OR)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Connective.readFromStream(in), equalTo(Connective.AND)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + Connective.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown Connective ordinal [")); + } + } + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DataDescriptionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DataDescriptionTests.java new file mode 100644 index 00000000000..e47a86c7219 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DataDescriptionTests.java @@ -0,0 +1,250 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.ml.job.config.DataDescription.DataFormat; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; + +public class DataDescriptionTests extends AbstractSerializingTestCase { + + public void testVerify_GivenValidFormat() { + DataDescription.Builder description = new DataDescription.Builder(); + description.setTimeFormat("epoch"); + description.setTimeFormat("epoch_ms"); + description.setTimeFormat("yyyy-MM-dd HH"); + String goodFormat = "yyyy.MM.dd G 'at' HH:mm:ss z"; + description.setTimeFormat(goodFormat); + } + + public void testVerify_GivenInValidFormat() { + DataDescription.Builder description = new DataDescription.Builder(); + expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat(null)); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("invalid")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, "invalid"), e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, ""), e.getMessage()); + + e = expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("y-M-dd")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, "y-M-dd"), e.getMessage()); + expectThrows(IllegalArgumentException.class, () -> description.setTimeFormat("YYY-mm-UU hh:mm:ssY")); + } + + public void testTransform_GivenDelimitedAndEpoch() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setFormat(DataFormat.DELIMITED); + dd.setTimeFormat("epoch"); + assertFalse(dd.build().transform()); + } + + public void testTransform_GivenDelimitedAndEpochMs() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setFormat(DataFormat.DELIMITED); + dd.setTimeFormat("epoch_ms"); + assertTrue(dd.build().transform()); + } + + public void testIsTransformTime_GivenTimeFormatIsEpoch() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setTimeFormat("epoch"); + assertFalse(dd.build().isTransformTime()); + } + + public void testIsTransformTime_GivenTimeFormatIsEpochMs() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setTimeFormat("epoch_ms"); + assertTrue(dd.build().isTransformTime()); + } + + public void testIsTransformTime_GivenTimeFormatPattern() { + DataDescription.Builder dd = new DataDescription.Builder(); + dd.setTimeFormat("yyyy-MM-dd HH:mm:ss.SSSZ"); + assertTrue(dd.build().isTransformTime()); + } + + public void testEquals_GivenDifferentDateFormat() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.DELIMITED); + description2.setQuoteCharacter('"'); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentQuoteCharacter() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('\''); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentTimeField() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('"'); + description2.setTimeField("time"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentTimeFormat() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('"'); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch_ms"); + description2.setFieldDelimiter(','); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testEquals_GivenDifferentFieldDelimiter() { + DataDescription.Builder description1 = new DataDescription.Builder(); + description1.setFormat(DataFormat.JSON); + description1.setQuoteCharacter('"'); + description1.setTimeField("timestamp"); + description1.setTimeFormat("epoch"); + description1.setFieldDelimiter(','); + + DataDescription.Builder description2 = new DataDescription.Builder(); + description2.setFormat(DataFormat.JSON); + description2.setQuoteCharacter('"'); + description2.setTimeField("timestamp"); + description2.setTimeFormat("epoch"); + description2.setFieldDelimiter(';'); + + assertFalse(description1.build().equals(description2.build())); + assertFalse(description2.build().equals(description1.build())); + } + + public void testInvalidDataFormat() throws Exception { + BytesArray json = new BytesArray("{ \"format\":\"INEXISTENT_FORMAT\" }"); + XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, json); + ParsingException ex = expectThrows(ParsingException.class, + () -> DataDescription.PARSER.apply(parser, null)); + assertThat(ex.getMessage(), containsString("[data_description] failed to parse field [format]")); + Throwable cause = ex.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), + containsString("No enum constant org.elasticsearch.xpack.ml.job.config.DataDescription.DataFormat.INEXISTENT_FORMAT")); + } + + public void testInvalidFieldDelimiter() throws Exception { + BytesArray json = new BytesArray("{ \"field_delimiter\":\",,\" }"); + XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, json); + ParsingException ex = expectThrows(ParsingException.class, + () -> DataDescription.PARSER.apply(parser, null)); + assertThat(ex.getMessage(), containsString("[data_description] failed to parse field [field_delimiter]")); + Throwable cause = ex.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), + containsString("String must be a single character, found [,,]")); + } + + public void testInvalidQuoteCharacter() throws Exception { + BytesArray json = new BytesArray("{ \"quote_character\":\"''\" }"); + XContentParser parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, json); + ParsingException ex = expectThrows(ParsingException.class, + () -> DataDescription.PARSER.apply(parser, null)); + assertThat(ex.getMessage(), containsString("[data_description] failed to parse field [quote_character]")); + Throwable cause = ex.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(IllegalArgumentException.class)); + assertThat(cause.getMessage(), containsString("String must be a single character, found ['']")); + } + + @Override + protected DataDescription createTestInstance() { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + if (randomBoolean()) { + dataDescription.setFormat(randomFrom(DataFormat.values())); + } + if (randomBoolean()) { + dataDescription.setTimeField(randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + String format; + if (randomBoolean()) { + format = DataDescription.EPOCH; + } else if (randomBoolean()) { + format = DataDescription.EPOCH_MS; + } else { + format = "yyy.MM.dd G 'at' HH:mm:ss z"; + } + dataDescription.setTimeFormat(format); + } + if (randomBoolean()) { + dataDescription.setFieldDelimiter(randomAsciiOfLength(1).charAt(0)); + } + if (randomBoolean()) { + dataDescription.setQuoteCharacter(randomAsciiOfLength(1).charAt(0)); + } + return dataDescription.build(); + } + + @Override + protected Reader instanceReader() { + return DataDescription::new; + } + + @Override + protected DataDescription parseInstance(XContentParser parser) { + return DataDescription.PARSER.apply(parser, null).build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DataFormatTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DataFormatTests.java new file mode 100644 index 00000000000..5a53f177e9b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DataFormatTests.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.config.DataDescription.DataFormat; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class DataFormatTests extends ESTestCase { + + public void testFromString() { + assertEquals(DataFormat.DELIMITED, DataFormat.forString("delineated")); + assertEquals(DataFormat.DELIMITED, DataFormat.forString("DELINEATED")); + assertEquals(DataFormat.DELIMITED, DataFormat.forString("delimited")); + assertEquals(DataFormat.DELIMITED, DataFormat.forString("DELIMITED")); + + assertEquals(DataFormat.JSON, DataFormat.forString("json")); + assertEquals(DataFormat.JSON, DataFormat.forString("JSON")); + } + + public void testToString() { + assertEquals("delimited", DataFormat.DELIMITED.toString()); + assertEquals("json", DataFormat.JSON.toString()); + } + + public void testValidOrdinals() { + assertThat(DataFormat.JSON.ordinal(), equalTo(0)); + assertThat(DataFormat.DELIMITED.ordinal(), equalTo(1)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + DataFormat.JSON.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + DataFormat.DELIMITED.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(DataFormat.readFromStream(in), equalTo(DataFormat.JSON)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(DataFormat.readFromStream(in), equalTo(DataFormat.DELIMITED)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(4, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + DataFormat.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown DataFormat ordinal [")); + } + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DefaultDetectorDescriptionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DefaultDetectorDescriptionTests.java new file mode 100644 index 00000000000..6e60ab63c1c --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DefaultDetectorDescriptionTests.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.test.ESTestCase; + +public class DefaultDetectorDescriptionTests extends ESTestCase { + + + public void testOf_GivenOnlyFunctionAndFieldName() { + Detector detector = new Detector.Builder("min", "value").build(); + + assertEquals("min(value)", DefaultDetectorDescription.of(detector)); + } + + + public void testOf_GivenOnlyFunctionAndFieldNameWithNonWordChars() { + Detector detector = new Detector.Builder("min", "val-ue").build(); + + assertEquals("min(\"val-ue\")", DefaultDetectorDescription.of(detector)); + } + + + public void testOf_GivenFullyPopulatedDetector() { + Detector.Builder detector = new Detector.Builder("sum", "value"); + detector.setByFieldName("airline"); + detector.setOverFieldName("region"); + detector.setUseNull(true); + detector.setPartitionFieldName("planet"); + detector.setExcludeFrequent(Detector.ExcludeFrequent.ALL); + + assertEquals("sum(value) by airline over region usenull=true partitionfield=planet excludefrequent=all", + DefaultDetectorDescription.of(detector.build())); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DefaultFrequencyTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DefaultFrequencyTests.java new file mode 100644 index 00000000000..ba4237280bb --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DefaultFrequencyTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.test.ESTestCase; + +import java.time.Duration; + +public class DefaultFrequencyTests extends ESTestCase { + + public void testCalc_GivenNegative() { + ESTestCase.expectThrows(IllegalArgumentException.class, () -> DefaultFrequency.ofBucketSpan(-1)); + } + + + public void testCalc() { + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(1)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(30)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(60)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(90)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(120)); + assertEquals(Duration.ofMinutes(1), DefaultFrequency.ofBucketSpan(121)); + + assertEquals(Duration.ofSeconds(61), DefaultFrequency.ofBucketSpan(122)); + assertEquals(Duration.ofSeconds(75), DefaultFrequency.ofBucketSpan(150)); + assertEquals(Duration.ofSeconds(150), DefaultFrequency.ofBucketSpan(300)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1200)); + + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1201)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(1800)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(3600)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(7200)); + assertEquals(Duration.ofMinutes(10), DefaultFrequency.ofBucketSpan(12 * 3600)); + + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(12 * 3600 + 1)); + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(13 * 3600)); + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(24 * 3600)); + assertEquals(Duration.ofHours(1), DefaultFrequency.ofBucketSpan(48 * 3600)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectionRuleTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectionRuleTests.java new file mode 100644 index 00000000000..e210cd23620 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectionRuleTests.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +public class DetectionRuleTests extends AbstractSerializingTestCase { + + public void testExtractReferoencedLists() { + RuleCondition numericalCondition = + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "field", "value", new Condition(Operator.GT, "5"), null); + List conditions = Arrays.asList( + numericalCondition, + RuleCondition.createCategorical("foo", "filter1"), + RuleCondition.createCategorical("bar", "filter2")); + DetectionRule rule = new DetectionRule(null, null, Connective.OR, conditions); + + assertEquals(new HashSet<>(Arrays.asList("filter1", "filter2")), rule.extractReferencedFilters()); + } + + public void testEqualsGivenSameObject() { + DetectionRule rule = createFullyPopulated(); + assertTrue(rule.equals(rule)); + } + + public void testEqualsGivenString() { + assertFalse(createFullyPopulated().equals("a string")); + } + + public void testEqualsGivenDifferentTargetFieldName() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField2", "targetValue", Connective.AND, createRule("5")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenDifferentTargetFieldValue() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField", "targetValue2", Connective.AND, createRule("5")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenDifferentConjunction() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField", "targetValue", Connective.OR, createRule("5")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenRules() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = new DetectionRule("targetField", "targetValue", Connective.AND, createRule("10")); + assertFalse(rule1.equals(rule2)); + assertFalse(rule2.equals(rule1)); + } + + public void testEqualsGivenEqual() { + DetectionRule rule1 = createFullyPopulated(); + DetectionRule rule2 = createFullyPopulated(); + assertTrue(rule1.equals(rule2)); + assertTrue(rule2.equals(rule1)); + assertEquals(rule1.hashCode(), rule2.hashCode()); + } + + private static DetectionRule createFullyPopulated() { + return new DetectionRule("targetField", "targetValue", Connective.AND, createRule("5")); + } + + private static List createRule(String value) { + Condition condition = new Condition(Operator.GT, value); + return Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + } + + @Override + protected DetectionRule createTestInstance() { + String targetFieldName = null; + String targetFieldValue = null; + Connective connective = randomFrom(Connective.values()); + if (randomBoolean()) { + targetFieldName = randomAsciiOfLengthBetween(1, 20); + targetFieldValue = randomAsciiOfLengthBetween(1, 20); + } + int size = 1 + randomInt(20); + List ruleConditions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random condition (it is already tested) + ruleConditions.addAll(createRule(Double.toString(randomDouble()))); + } + return new DetectionRule(targetFieldName, targetFieldValue, connective, ruleConditions); + } + + @Override + protected Reader instanceReader() { + return DetectionRule::new; + } + + @Override + protected DetectionRule parseInstance(XContentParser parser) { + return DetectionRule.PARSER.apply(parser, null); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectorTests.java new file mode 100644 index 00000000000..070174c5fab --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectorTests.java @@ -0,0 +1,636 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; +import org.junit.Assert; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class DetectorTests extends AbstractSerializingTestCase { + + public void testEquals_GivenEqual() { + Detector.Builder builder = new Detector.Builder("mean", "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.setPartitionFieldName("partition"); + builder.setUseNull(false); + Detector detector1 = builder.build(); + + builder = new Detector.Builder("mean", "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.setPartitionFieldName("partition"); + builder.setUseNull(false); + Detector detector2 = builder.build(); + + assertTrue(detector1.equals(detector2)); + assertTrue(detector2.equals(detector1)); + assertEquals(detector1.hashCode(), detector2.hashCode()); + } + + public void testEquals_GivenDifferentDetectorDescription() { + Detector detector1 = createDetector().build(); + Detector.Builder builder = createDetector(); + builder.setDetectorDescription("bar"); + Detector detector2 = builder.build(); + + assertFalse(detector1.equals(detector2)); + } + + public void testEquals_GivenDifferentByFieldName() { + Detector detector1 = createDetector().build(); + Detector detector2 = createDetector().build(); + + assertEquals(detector1, detector2); + + Detector.Builder builder = new Detector.Builder(detector2); + builder.setByFieldName("by2"); + Condition condition = new Condition(Operator.GT, "5"); + DetectionRule rule = new DetectionRule("over", "targetValue", Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null))); + builder.setDetectorRules(Collections.singletonList(rule)); + detector2 = builder.build(); + assertFalse(detector1.equals(detector2)); + } + + public void testEquals_GivenDifferentRules() { + Detector detector1 = createDetector().build(); + Detector.Builder builder = new Detector.Builder(detector1); + DetectionRule rule = new DetectionRule(builder.getDetectorRules().get(0).getTargetFieldName(), + builder.getDetectorRules().get(0).getTargetFieldValue(), Connective.OR, + builder.getDetectorRules().get(0).getRuleConditions()); + builder.getDetectorRules().set(0, rule); + Detector detector2 = builder.build(); + + assertFalse(detector1.equals(detector2)); + assertFalse(detector2.equals(detector1)); + } + + public void testExtractAnalysisFields() { + Detector detector = createDetector().build(); + assertEquals(Arrays.asList("by", "over", "partition"), detector.extractAnalysisFields()); + Detector.Builder builder = new Detector.Builder(detector); + builder.setPartitionFieldName(null); + detector = builder.build(); + assertEquals(Arrays.asList("by", "over"), detector.extractAnalysisFields()); + builder = new Detector.Builder(detector); + Condition condition = new Condition(Operator.GT, "5"); + DetectionRule rule = new DetectionRule("over", "targetValue", Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))); + builder.setDetectorRules(Collections.singletonList(rule)); + builder.setByFieldName(null); + detector = builder.build(); + assertEquals(Arrays.asList("over"), detector.extractAnalysisFields()); + builder = new Detector.Builder(detector); + rule = new DetectionRule(null, null, Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))); + builder.setDetectorRules(Collections.singletonList(rule)); + builder.setOverFieldName(null); + detector = builder.build(); + assertTrue(detector.extractAnalysisFields().isEmpty()); + } + + public void testExtractReferencedLists() { + Detector.Builder builder = createDetector(); + builder.setDetectorRules(Arrays.asList( + new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("by", "list1"))), + new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("by", "list2"))))); + + Detector detector = builder.build(); + assertEquals(new HashSet<>(Arrays.asList("list1", "list2")), detector.extractReferencedFilters()); + } + + private Detector.Builder createDetector() { + Detector.Builder detector = new Detector.Builder("mean", "field"); + detector.setByFieldName("by"); + detector.setOverFieldName("over"); + detector.setPartitionFieldName("partition"); + detector.setUseNull(true); + Condition condition = new Condition(Operator.GT, "5"); + DetectionRule rule = new DetectionRule("over", "targetValue", Connective.AND, + Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by", "val", condition, null))); + detector.setDetectorRules(Arrays.asList(rule)); + return detector; + } + + @Override + protected Detector createTestInstance() { + String function; + Detector.Builder detector; + if (randomBoolean()) { + detector = new Detector.Builder(function = randomFrom(Detector.COUNT_WITHOUT_FIELD_FUNCTIONS), null); + } else { + Set functions = new HashSet<>(Detector.FIELD_NAME_FUNCTIONS); + functions.removeAll(Detector.Builder.FUNCTIONS_WITHOUT_RULE_SUPPORT); + detector = new Detector.Builder(function = randomFrom(functions), randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + detector.setDetectorDescription(randomAsciiOfLengthBetween(1, 20)); + } + String fieldName = null; + if (randomBoolean()) { + detector.setPartitionFieldName(fieldName = randomAsciiOfLengthBetween(1, 20)); + } else if (randomBoolean() && Detector.NO_OVER_FIELD_NAME_FUNCTIONS.contains(function) == false) { + detector.setOverFieldName(fieldName = randomAsciiOfLengthBetween(1, 20)); + } else if (randomBoolean() && Detector.NO_BY_FIELD_NAME_FUNCTIONS.contains(function) == false) { + detector.setByFieldName(fieldName = randomAsciiOfLengthBetween(1, 20)); + } + if (randomBoolean()) { + detector.setExcludeFrequent(randomFrom(Detector.ExcludeFrequent.values())); + } + if (randomBoolean()) { + int size = randomInt(10); + List detectorRules = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random DetectionRule (it is already tested) + Condition condition = new Condition(Operator.GT, "5"); + detectorRules.add(new DetectionRule(fieldName, null, Connective.OR, Collections.singletonList( + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)) + )); + } + detector.setDetectorRules(detectorRules); + } + if (randomBoolean()) { + detector.setUseNull(randomBoolean()); + } + return detector.build(); + } + + @Override + protected Reader instanceReader() { + return Detector::new; + } + + @Override + protected Detector parseInstance(XContentParser parser) { + return Detector.PARSER.apply(parser, null).build(); + } + + public void testVerifyFieldNames_givenInvalidChars() { + Collection testCaseArguments = getCharactersAndValidity(); + for (Object [] args : testCaseArguments) { + String character = (String) args[0]; + boolean valid = (boolean) args[1]; + Detector.Builder detector = createDetectorWithValidFieldNames(); + verifyFieldName(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyByFieldName(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyOverFieldName(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyPartitionFieldName(detector, character, valid); + } + } + + public void testVerifyFunctionForPreSummariedInput() { + Collection testCaseArguments = getCharactersAndValidity(); + for (Object [] args : testCaseArguments) { + String character = (String) args[0]; + boolean valid = (boolean) args[1]; + Detector.Builder detector = createDetectorWithValidFieldNames(); + verifyFieldNameGivenPresummarised(detector, character, valid); + detector = createDetectorWithValidFieldNames(); + verifyByFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + verifyOverFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + verifyByFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + verifyPartitionFieldNameGivenPresummarised(new Detector.Builder(detector.build()), character, valid); + } + } + + private static void verifyFieldName(Detector.Builder detector, String character, boolean valid) { + Detector.Builder updated = createDetectorWithSpecificFieldName(detector.build().getFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> updated.build()); + } + } + + private static void verifyByFieldName(Detector.Builder detector, String character, boolean valid) { + detector.setByFieldName(detector.build().getByFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> detector.build()); + } + } + + private static void verifyOverFieldName(Detector.Builder detector, String character, boolean valid) { + detector.setOverFieldName(detector.build().getOverFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> detector.build()); + } + } + + private static void verifyPartitionFieldName(Detector.Builder detector, String character, boolean valid) { + detector.setPartitionFieldName(detector.build().getPartitionFieldName() + character); + if (valid == false) { + expectThrows(IllegalArgumentException.class , () -> detector.build()); + } + } + + private static void verifyFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + Detector.Builder updated = createDetectorWithSpecificFieldName(detector.build().getFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> updated.build(true)); + } + + private static void verifyByFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + detector.setByFieldName(detector.build().getByFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> detector.build(true)); + } + + private static void verifyOverFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + detector.setOverFieldName(detector.build().getOverFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> detector.build(true)); + } + + private static void verifyPartitionFieldNameGivenPresummarised(Detector.Builder detector, String character, boolean valid) { + detector.setPartitionFieldName(detector.build().getPartitionFieldName() + character); + expectThrows(IllegalArgumentException.class , () -> detector.build(true)); + } + + private static Detector.Builder createDetectorWithValidFieldNames() { + Detector.Builder d = new Detector.Builder("metric", "field"); + d.setByFieldName("by"); + d.setOverFieldName("over"); + d.setPartitionFieldName("partition"); + return d; + } + + private static Detector.Builder createDetectorWithSpecificFieldName(String fieldName) { + Detector.Builder d = new Detector.Builder("metric", fieldName); + d.setByFieldName("by"); + d.setOverFieldName("over"); + d.setPartitionFieldName("partition"); + return d; + } + + private static Collection getCharactersAndValidity() { + return Arrays.asList(new Object[][]{ + // char, isValid? + {"a", true}, + {"[", true}, + {"]", true}, + {"(", true}, + {")", true}, + {"=", true}, + {"-", true}, + {" ", true}, + {"\"", false}, + {"\\", false}, + {"\t", false}, + {"\n", false}, + }); + } + + public void testVerify() throws Exception { + // if nothing else is set the count functions (excluding distinct count) + // are the only allowable functions + new Detector.Builder(Detector.COUNT, null).build(); + new Detector.Builder(Detector.COUNT, null).build(true); + + Set difference = new HashSet(Detector.ANALYSIS_FUNCTIONS); + difference.remove(Detector.COUNT); + difference.remove(Detector.HIGH_COUNT); + difference.remove(Detector.LOW_COUNT); + difference.remove(Detector.NON_ZERO_COUNT); + difference.remove(Detector.NZC); + difference.remove(Detector.LOW_NON_ZERO_COUNT); + difference.remove(Detector.LOW_NZC); + difference.remove(Detector.HIGH_NON_ZERO_COUNT); + difference.remove(Detector.HIGH_NZC); + difference.remove(Detector.TIME_OF_DAY); + difference.remove(Detector.TIME_OF_WEEK); + for (String f : difference) { + try { + new Detector.Builder(f, null).build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + new Detector.Builder(f, null).build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // certain fields aren't allowed with certain functions + // first do the over field + for (String f : new String[]{Detector.NON_ZERO_COUNT, Detector.NZC, + Detector.LOW_NON_ZERO_COUNT, Detector.LOW_NZC, Detector.HIGH_NON_ZERO_COUNT, + Detector.HIGH_NZC}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // these functions cannot have just an over field + difference = new HashSet<>(Detector.ANALYSIS_FUNCTIONS); + difference.remove(Detector.COUNT); + difference.remove(Detector.HIGH_COUNT); + difference.remove(Detector.LOW_COUNT); + difference.remove(Detector.TIME_OF_DAY); + difference.remove(Detector.TIME_OF_WEEK); + for (String f : difference) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // these functions can have just an over field + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.RARE, Detector.FREQ_RARE}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + builder.setByFieldName("by"); + builder.build(); + builder.build(true); + } + + + // some functions require a fieldname + for (String f : new String[]{Detector.DISTINCT_COUNT, Detector.DC, + Detector.HIGH_DISTINCT_COUNT, Detector.HIGH_DC, Detector.LOW_DISTINCT_COUNT, Detector.LOW_DC, + Detector.INFO_CONTENT, Detector.LOW_INFO_CONTENT, Detector.HIGH_INFO_CONTENT, + Detector.METRIC, Detector.MEAN, Detector.HIGH_MEAN, Detector.LOW_MEAN, Detector.AVG, + Detector.HIGH_AVG, Detector.LOW_AVG, Detector.MAX, Detector.MIN, Detector.SUM, + Detector.LOW_SUM, Detector.HIGH_SUM, Detector.NON_NULL_SUM, + Detector.LOW_NON_NULL_SUM, Detector.HIGH_NON_NULL_SUM, Detector.POPULATION_VARIANCE, + Detector.LOW_POPULATION_VARIANCE, Detector.HIGH_POPULATION_VARIANCE}) { + Detector.Builder builder = new Detector.Builder(f, "f"); + builder.setOverFieldName("over"); + builder.build(); + try { + builder.build(true); + Assert.assertFalse(Detector.METRIC.equals(f)); + } catch (IllegalArgumentException e) { + // "metric" is not allowed as the function for pre-summarised input + Assert.assertEquals(Detector.METRIC, f); + } + } + + // these functions cannot have a field name + difference = new HashSet<>(Detector.ANALYSIS_FUNCTIONS); + difference.remove(Detector.METRIC); + difference.remove(Detector.MEAN); + difference.remove(Detector.LOW_MEAN); + difference.remove(Detector.HIGH_MEAN); + difference.remove(Detector.AVG); + difference.remove(Detector.LOW_AVG); + difference.remove(Detector.HIGH_AVG); + difference.remove(Detector.MEDIAN); + difference.remove(Detector.MIN); + difference.remove(Detector.MAX); + difference.remove(Detector.SUM); + difference.remove(Detector.LOW_SUM); + difference.remove(Detector.HIGH_SUM); + difference.remove(Detector.NON_NULL_SUM); + difference.remove(Detector.LOW_NON_NULL_SUM); + difference.remove(Detector.HIGH_NON_NULL_SUM); + difference.remove(Detector.POPULATION_VARIANCE); + difference.remove(Detector.LOW_POPULATION_VARIANCE); + difference.remove(Detector.HIGH_POPULATION_VARIANCE); + difference.remove(Detector.DISTINCT_COUNT); + difference.remove(Detector.HIGH_DISTINCT_COUNT); + difference.remove(Detector.LOW_DISTINCT_COUNT); + difference.remove(Detector.DC); + difference.remove(Detector.LOW_DC); + difference.remove(Detector.HIGH_DC); + difference.remove(Detector.INFO_CONTENT); + difference.remove(Detector.LOW_INFO_CONTENT); + difference.remove(Detector.HIGH_INFO_CONTENT); + difference.remove(Detector.LAT_LONG); + for (String f : difference) { + Detector.Builder builder = new Detector.Builder(f, "f"); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + // these can have a by field + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.RARE, + Detector.NON_ZERO_COUNT, Detector.NZC}) { + Detector.Builder builder = new Detector.Builder(f, null); + builder.setByFieldName("b"); + builder.build(); + builder.build(true); + } + + Detector.Builder builder = new Detector.Builder(Detector.FREQ_RARE, null); + builder.setOverFieldName("over"); + builder.setByFieldName("b"); + builder.build(); + builder.build(true); + builder = new Detector.Builder(Detector.FREQ_RARE, null); + builder.setOverFieldName("over"); + builder.setByFieldName("b"); + builder.build(); + + // some functions require a fieldname + for (String f : new String[]{Detector.METRIC, Detector.MEAN, Detector.HIGH_MEAN, + Detector.LOW_MEAN, Detector.AVG, Detector.HIGH_AVG, Detector.LOW_AVG, + Detector.MEDIAN, Detector.MAX, Detector.MIN, Detector.SUM, Detector.LOW_SUM, + Detector.HIGH_SUM, Detector.NON_NULL_SUM, Detector.LOW_NON_NULL_SUM, + Detector.HIGH_NON_NULL_SUM, Detector.POPULATION_VARIANCE, + Detector.LOW_POPULATION_VARIANCE, Detector.HIGH_POPULATION_VARIANCE, + Detector.DISTINCT_COUNT, Detector.DC, + Detector.HIGH_DISTINCT_COUNT, Detector.HIGH_DC, Detector.LOW_DISTINCT_COUNT, + Detector.LOW_DC, Detector.INFO_CONTENT, Detector.LOW_INFO_CONTENT, + Detector.HIGH_INFO_CONTENT, Detector.LAT_LONG}) { + builder = new Detector.Builder(f, "f"); + builder.setByFieldName("b"); + builder.build(); + try { + builder.build(true); + Assert.assertFalse(Detector.METRIC.equals(f)); + } catch (IllegalArgumentException e) { + // "metric" is not allowed as the function for pre-summarised input + Assert.assertEquals(Detector.METRIC, f); + } + } + + + // these functions don't work with fieldname + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.NON_ZERO_COUNT, Detector.NZC, + Detector.RARE, Detector.FREQ_RARE, Detector.TIME_OF_DAY, + Detector.TIME_OF_WEEK}) { + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + + for (String f : new String[]{Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.NON_ZERO_COUNT, Detector.NZC, + Detector.RARE, Detector.FREQ_RARE, Detector.TIME_OF_DAY, + Detector.TIME_OF_WEEK}) { + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("b"); + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + + builder = new Detector.Builder(Detector.FREQ_RARE, "field"); + builder.setByFieldName("b"); + builder.setOverFieldName("over"); + try { + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + + + for (String f : new String[]{Detector.HIGH_COUNT, + Detector.LOW_COUNT, Detector.NON_ZERO_COUNT, Detector.NZC}) { + builder = new Detector.Builder(f, null); + builder.setByFieldName("by"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.COUNT, Detector.HIGH_COUNT, + Detector.LOW_COUNT}) { + builder = new Detector.Builder(f, null); + builder.setOverFieldName("over"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.HIGH_COUNT, + Detector.LOW_COUNT}) { + builder = new Detector.Builder(f, null); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.build(); + builder.build(true); + } + + for (String f : new String[]{Detector.NON_ZERO_COUNT, Detector.NZC}) { + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.build(); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + try { + builder = new Detector.Builder(f, "field"); + builder.setByFieldName("by"); + builder.setOverFieldName("over"); + builder.build(true); + Assert.fail("IllegalArgumentException not thrown when expected"); + } catch (IllegalArgumentException e) { + } + } + } + + public void testVerify_GivenInvalidDetectionRuleTargetFieldName() { + Detector.Builder detector = new Detector.Builder("mean", "metricVale"); + detector.setByFieldName("metricName"); + detector.setPartitionFieldName("instance"); + RuleCondition ruleCondition = + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null); + DetectionRule rule = new DetectionRule("instancE", null, Connective.OR, Arrays.asList(ruleCondition)); + detector.setDetectorRules(Arrays.asList(rule)); + + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, detector::build); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME, + "[metricName, instance]", "instancE"), + e.getMessage()); + } + + public void testVerify_GivenValidDetectionRule() { + Detector.Builder detector = new Detector.Builder("mean", "metricVale"); + detector.setByFieldName("metricName"); + detector.setPartitionFieldName("instance"); + RuleCondition ruleCondition = + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null); + DetectionRule rule = new DetectionRule("instance", null, Connective.OR, Arrays.asList(ruleCondition)); + detector.setDetectorRules(Arrays.asList(rule)); + detector.build(); + } + + public void testExcludeFrequentForString() { + assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("all")); + assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("ALL")); + assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("none")); + assertEquals(Detector.ExcludeFrequent.NONE, Detector.ExcludeFrequent.forString("NONE")); + assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("by")); + assertEquals(Detector.ExcludeFrequent.BY, Detector.ExcludeFrequent.forString("BY")); + assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("over")); + assertEquals(Detector.ExcludeFrequent.OVER, Detector.ExcludeFrequent.forString("OVER")); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/IgnoreDowntimeTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/IgnoreDowntimeTests.java new file mode 100644 index 00000000000..3e9fd3eac52 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/IgnoreDowntimeTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.test.ESTestCase; + +public class IgnoreDowntimeTests extends ESTestCase { + + public void testFromString() { + assertEquals(IgnoreDowntime.fromString("always"), IgnoreDowntime.ALWAYS); + assertEquals(IgnoreDowntime.fromString("never"), IgnoreDowntime.NEVER); + assertEquals(IgnoreDowntime.fromString("once"), IgnoreDowntime.ONCE); + } + + public void testToString() { + assertEquals("always", IgnoreDowntime.ALWAYS.toString()); + assertEquals("never", IgnoreDowntime.NEVER.toString()); + assertEquals("once", IgnoreDowntime.ONCE.toString()); + } + + public void testValidOrdinals() { + assertEquals(0, IgnoreDowntime.NEVER.ordinal()); + assertEquals(1, IgnoreDowntime.ONCE.ordinal()); + assertEquals(2, IgnoreDowntime.ALWAYS.ordinal()); + } + + public void testFromString_GivenLeadingWhitespace() { + assertEquals(IgnoreDowntime.ALWAYS, IgnoreDowntime.fromString(" \t ALWAYS")); + } + + + public void testFromString_GivenTrailingWhitespace() { + assertEquals(IgnoreDowntime.NEVER, IgnoreDowntime.fromString("NEVER \t ")); + } + + + public void testFromString_GivenExactMatches() { + assertEquals(IgnoreDowntime.NEVER, IgnoreDowntime.fromString("NEVER")); + assertEquals(IgnoreDowntime.ONCE, IgnoreDowntime.fromString("ONCE")); + assertEquals(IgnoreDowntime.ALWAYS, IgnoreDowntime.fromString("ALWAYS")); + } + + + public void testFromString_GivenMixedCaseCharacters() { + assertEquals(IgnoreDowntime.NEVER, IgnoreDowntime.fromString("nevEr")); + assertEquals(IgnoreDowntime.ONCE, IgnoreDowntime.fromString("oNce")); + assertEquals(IgnoreDowntime.ALWAYS, IgnoreDowntime.fromString("always")); + } + + public void testFromString_GivenNonMatchingString() { + ESTestCase.expectThrows(IllegalArgumentException.class, () -> IgnoreDowntime.fromString("nope")); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobStateTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobStateTests.java new file mode 100644 index 00000000000..e6d9b4b4975 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobStateTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.test.ESTestCase; + +public class JobStateTests extends ESTestCase { + + public void testFromString() { + assertEquals(JobState.fromString("closed"), JobState.CLOSED); + assertEquals(JobState.fromString("closing"), JobState.CLOSING); + assertEquals(JobState.fromString("failed"), JobState.FAILED); + assertEquals(JobState.fromString("opening"), JobState.OPENING); + assertEquals(JobState.fromString("opened"), JobState.OPENED); + assertEquals(JobState.fromString("CLOSED"), JobState.CLOSED); + assertEquals(JobState.fromString("CLOSING"), JobState.CLOSING); + assertEquals(JobState.fromString("FAILED"), JobState.FAILED); + assertEquals(JobState.fromString("OPENING"), JobState.OPENING); + assertEquals(JobState.fromString("OPENED"), JobState.OPENED); + } + + public void testToString() { + assertEquals("closed", JobState.CLOSED.toString()); + assertEquals("closing", JobState.CLOSING.toString()); + assertEquals("failed", JobState.FAILED.toString()); + assertEquals("opening", JobState.OPENING.toString()); + assertEquals("opened", JobState.OPENED.toString()); + } + + public void testValidOrdinals() { + assertEquals(0, JobState.CLOSING.ordinal()); + assertEquals(1, JobState.CLOSED.ordinal()); + assertEquals(2, JobState.OPENING.ordinal()); + assertEquals(3, JobState.OPENED.ordinal()); + assertEquals(4, JobState.FAILED.ordinal()); + } + + public void testIsAnyOf() { + assertFalse(JobState.OPENED.isAnyOf()); + assertFalse(JobState.OPENED.isAnyOf(JobState.CLOSED, JobState.CLOSING, JobState.FAILED, + JobState.OPENING)); + assertFalse(JobState.CLOSED.isAnyOf(JobState.CLOSING, JobState.FAILED, JobState.OPENING, JobState.OPENED)); + + assertTrue(JobState.OPENED.isAnyOf(JobState.OPENED)); + assertTrue(JobState.OPENED.isAnyOf(JobState.OPENED, JobState.CLOSED)); + assertTrue(JobState.CLOSING.isAnyOf(JobState.CLOSED, JobState.CLOSING)); + assertTrue(JobState.CLOSED.isAnyOf(JobState.CLOSED, JobState.CLOSING)); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java new file mode 100644 index 00000000000..794531cfc99 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobTests.java @@ -0,0 +1,429 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class JobTests extends AbstractSerializingTestCase { + + @Override + protected Job createTestInstance() { + return createRandomizedJob(); + } + + @Override + protected Writeable.Reader instanceReader() { + return Job::new; + } + + @Override + protected Job parseInstance(XContentParser parser) { + return Job.PARSER.apply(parser, null).build(); + } + + public void testConstructor_GivenEmptyJobConfiguration() { + Job job = buildJobBuilder("foo").build(true, "foo"); + + assertEquals("foo", job.getId()); + assertNotNull(job.getCreateTime()); + assertNotNull(job.getAnalysisConfig()); + assertNull(job.getAnalysisLimits()); + assertNull(job.getCustomSettings()); + assertNotNull(job.getDataDescription()); + assertNull(job.getDescription()); + assertNull(job.getFinishedTime()); + assertNull(job.getIgnoreDowntime()); + assertNull(job.getLastDataTime()); + assertNull(job.getModelDebugConfig()); + assertNull(job.getRenormalizationWindowDays()); + assertNull(job.getBackgroundPersistInterval()); + assertNull(job.getModelSnapshotRetentionDays()); + assertNull(job.getResultsRetentionDays()); + assertNotNull(job.allFields()); + assertFalse(job.allFields().isEmpty()); + } + + public void testConstructor_GivenJobConfigurationWithIgnoreDowntime() { + Job.Builder builder = new Job.Builder("foo"); + builder.setIgnoreDowntime(IgnoreDowntime.ONCE); + builder.setAnalysisConfig(createAnalysisConfig()); + Job job = builder.build(); + + assertEquals("foo", job.getId()); + assertEquals(IgnoreDowntime.ONCE, job.getIgnoreDowntime()); + } + + public void testNoId() { + expectThrows(IllegalArgumentException.class, () -> buildJobBuilder("").build(true, "")); + } + + public void testInconsistentId() { + Exception e = expectThrows(IllegalArgumentException.class, () -> buildJobBuilder("foo").build(true, "bar")); + assertEquals(Messages.getMessage(Messages.INCONSISTENT_ID, Job.ID.getPreferredName(), "foo", "bar"), e.getMessage()); + } + + public void testEquals_GivenDifferentClass() { + Job job = buildJobBuilder("foo").build(); + assertFalse(job.equals("a string")); + } + + public void testEquals_GivenDifferentIds() { + Date createTime = new Date(); + Job.Builder builder = buildJobBuilder("foo"); + builder.setCreateTime(createTime); + Job job1 = builder.build(); + builder.setId("bar"); + Job job2 = builder.build(); + assertFalse(job1.equals(job2)); + } + + public void testEquals_GivenDifferentRenormalizationWindowDays() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setRenormalizationWindowDays(3L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setRenormalizationWindowDays(4L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentBackgroundPersistInterval() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setBackgroundPersistInterval(10000L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setBackgroundPersistInterval(8000L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentModelSnapshotRetentionDays() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setModelSnapshotRetentionDays(10L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setModelSnapshotRetentionDays(8L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentResultsRetentionDays() { + Job.Builder jobDetails1 = new Job.Builder("foo"); + jobDetails1.setAnalysisConfig(createAnalysisConfig()); + jobDetails1.setResultsRetentionDays(30L); + Job.Builder jobDetails2 = new Job.Builder("foo"); + jobDetails2.setResultsRetentionDays(4L); + jobDetails2.setAnalysisConfig(createAnalysisConfig()); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentCustomSettings() { + Job.Builder jobDetails1 = buildJobBuilder("foo"); + Map customSettings1 = new HashMap<>(); + customSettings1.put("key1", "value1"); + jobDetails1.setCustomSettings(customSettings1); + Job.Builder jobDetails2 = buildJobBuilder("foo"); + Map customSettings2 = new HashMap<>(); + customSettings2.put("key2", "value2"); + jobDetails2.setCustomSettings(customSettings2); + assertFalse(jobDetails1.build().equals(jobDetails2.build())); + } + + public void testEquals_GivenDifferentIgnoreDowntime() { + Job.Builder job1 = new Job.Builder(); + job1.setIgnoreDowntime(IgnoreDowntime.NEVER); + Job.Builder job2 = new Job.Builder(); + job2.setIgnoreDowntime(IgnoreDowntime.ONCE); + + assertFalse(job1.equals(job2)); + assertFalse(job2.equals(job1)); + } + + public void testSetAnalysisLimits() { + Job.Builder builder = new Job.Builder(); + builder.setAnalysisLimits(new AnalysisLimits(42L, null)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> builder.setAnalysisLimits(new AnalysisLimits(41L, null))); + assertEquals("Invalid update value for analysis_limits: model_memory_limit cannot be decreased; existing is 42, update had 41", + e.getMessage()); + } + + // JobConfigurationTests: + + /** + * Test the {@link AnalysisConfig#analysisFields()} method which produces a + * list of analysis fields from the detectors + */ + public void testAnalysisConfigRequiredFields() { + Detector.Builder d1 = new Detector.Builder("max", "field"); + d1.setByFieldName("by"); + + Detector.Builder d2 = new Detector.Builder("metric", "field2"); + d2.setOverFieldName("over"); + + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build())); + ac.setSummaryCountFieldName("agg"); + + List analysisFields = ac.build().analysisFields(); + assertTrue(analysisFields.size() == 5); + + assertTrue(analysisFields.contains("agg")); + assertTrue(analysisFields.contains("field")); + assertTrue(analysisFields.contains("by")); + assertTrue(analysisFields.contains("field2")); + assertTrue(analysisFields.contains("over")); + + assertFalse(analysisFields.contains("max")); + assertFalse(analysisFields.contains("")); + assertFalse(analysisFields.contains(null)); + + Detector.Builder d3 = new Detector.Builder("count", null); + d3.setByFieldName("by2"); + d3.setPartitionFieldName("partition"); + + ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build(), d3.build())); + + analysisFields = ac.build().analysisFields(); + assertTrue(analysisFields.size() == 6); + + assertTrue(analysisFields.contains("partition")); + assertTrue(analysisFields.contains("field")); + assertTrue(analysisFields.contains("by")); + assertTrue(analysisFields.contains("by2")); + assertTrue(analysisFields.contains("field2")); + assertTrue(analysisFields.contains("over")); + + assertFalse(analysisFields.contains("count")); + assertFalse(analysisFields.contains("max")); + assertFalse(analysisFields.contains("")); + assertFalse(analysisFields.contains(null)); + } + + // JobConfigurationVerifierTests: + + public void testCheckValidId_IdTooLong() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setId("averyveryveryaveryveryveryaveryveryveryaveryveryveryaveryveryveryaveryveryverylongid"); + expectThrows(IllegalArgumentException.class, () -> builder.build()); + } + + public void testCheckValidId_GivenAllValidChars() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setId("abcdefghijklmnopqrstuvwxyz-._0123456789"); + builder.build(); + } + + public void testCheckValidId_ProhibitedChars() { + String invalidChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ!@#$%^&*()+?\"'~±/\\[]{},<>="; + Job.Builder builder = buildJobBuilder("foo"); + for (char c : invalidChars.toCharArray()) { + builder.setId(Character.toString(c)); + String errorMessage = Messages.getMessage(Messages.INVALID_ID, Job.ID.getPreferredName(), Character.toString(c)); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + } + + public void testCheckValidId_startsWithUnderscore() { + Job.Builder builder = buildJobBuilder("_foo"); + String errorMessage = Messages.getMessage(Messages.INVALID_ID, Job.ID.getPreferredName(), "_foo"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testCheckValidId_endsWithUnderscore() { + Job.Builder builder = buildJobBuilder("foo_"); + String errorMessage = Messages.getMessage(Messages.INVALID_ID, Job.ID.getPreferredName(), "foo_"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testCheckValidId_ControlChars() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setId("two\nlines"); + expectThrows(IllegalArgumentException.class, builder::build); + } + + public void jobConfigurationTest() { + Job.Builder builder = new Job.Builder(); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId("bad id with spaces"); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId("bad_id_with_UPPERCASE_chars"); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId("a very very very very very very very very very very very very very very very very very very very very long id"); + expectThrows(IllegalArgumentException.class, builder::build); + builder.setId(null); + expectThrows(IllegalArgumentException.class, builder::build); + + Detector.Builder d = new Detector.Builder("max", "a"); + d.setByFieldName("b"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Collections.singletonList(d.build())); + builder.setAnalysisConfig(ac); + builder.build(); + builder.setAnalysisLimits(new AnalysisLimits(-1L, null)); + expectThrows(IllegalArgumentException.class, builder::build); + AnalysisLimits limits = new AnalysisLimits(1000L, 4L); + builder.setAnalysisLimits(limits); + builder.build(); + DataDescription.Builder dc = new DataDescription.Builder(); + dc.setTimeFormat("YYY_KKKKajsatp*"); + builder.setDataDescription(dc); + expectThrows(IllegalArgumentException.class, builder::build); + dc = new DataDescription.Builder(); + builder.setDataDescription(dc); + expectThrows(IllegalArgumentException.class, builder::build); + builder.build(); + } + + public void testVerify_GivenNegativeRenormalizationWindowDays() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, + "renormalizationWindowDays", 0, -1); + Job.Builder builder = buildJobBuilder("foo"); + builder.setRenormalizationWindowDays(-1L); + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenNegativeModelSnapshotRetentionDays() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "modelSnapshotRetentionDays", 0, -1); + Job.Builder builder = buildJobBuilder("foo"); + builder.setModelSnapshotRetentionDays(-1L); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenLowBackgroundPersistInterval() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, "backgroundPersistInterval", 3600, 3599); + Job.Builder builder = buildJobBuilder("foo"); + builder.setBackgroundPersistInterval(3599L); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testVerify_GivenNegativeResultsRetentionDays() { + String errorMessage = Messages.getMessage(Messages.JOB_CONFIG_FIELD_VALUE_TOO_LOW, + "resultsRetentionDays", 0, -1); + Job.Builder builder = buildJobBuilder("foo"); + builder.setResultsRetentionDays(-1L); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, builder::build); + assertEquals(errorMessage, e.getMessage()); + } + + public void testBuilder_setsDefaultIndexName() { + Job.Builder builder = buildJobBuilder("foo"); + Job job = builder.build(); + assertEquals("foo", job.getIndexName()); + } + + public void testBuilder_setsIndexName() { + Job.Builder builder = buildJobBuilder("foo"); + builder.setIndexName("carol"); + Job job = builder.build(); + assertEquals("carol", job.getIndexName()); + } + + public void testBuilder_withInvalidIndexNameThrows () { + Job.Builder builder = buildJobBuilder("foo"); + builder.setIndexName("_bad^name"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> builder.build()); + assertEquals(Messages.getMessage(Messages.INVALID_ID, Job.INDEX_NAME.getPreferredName()), e.getMessage()); + } + + public static Job.Builder buildJobBuilder(String id) { + Job.Builder builder = new Job.Builder(id); + builder.setCreateTime(new Date()); + AnalysisConfig.Builder ac = createAnalysisConfig(); + DataDescription.Builder dc = new DataDescription.Builder(); + builder.setAnalysisConfig(ac); + builder.setDataDescription(dc); + return builder; + } + + public static String randomValidJobId() { + CodepointSetGenerator generator = new CodepointSetGenerator("abcdefghijklmnopqrstuvwxyz".toCharArray()); + return generator.ofCodePointsLength(random(), 10, 10); + } + + public static AnalysisConfig.Builder createAnalysisConfig() { + Detector.Builder d1 = new Detector.Builder("info_content", "domain"); + d1.setOverFieldName("client"); + Detector.Builder d2 = new Detector.Builder("min", "field"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build())); + return ac; + } + + public static Job createRandomizedJob() { + String jobId = randomValidJobId(); + Job.Builder builder = new Job.Builder(jobId); + if (randomBoolean()) { + builder.setDescription(randomAsciiOfLength(10)); + } + builder.setCreateTime(new Date(randomNonNegativeLong())); + if (randomBoolean()) { + builder.setFinishedTime(new Date(randomNonNegativeLong())); + } + if (randomBoolean()) { + builder.setLastDataTime(new Date(randomNonNegativeLong())); + } + AnalysisConfig.Builder analysisConfig = createAnalysisConfig(); + analysisConfig.setBucketSpan(100L); + builder.setAnalysisConfig(analysisConfig); + builder.setAnalysisLimits(new AnalysisLimits(randomNonNegativeLong(), randomNonNegativeLong())); + if (randomBoolean()) { + DataDescription.Builder dataDescription = new DataDescription.Builder(); + dataDescription.setFormat(randomFrom(DataDescription.DataFormat.values())); + builder.setDataDescription(dataDescription); + } + String[] outputs; + AnalysisConfig ac = analysisConfig.build(); + if (randomBoolean()) { + outputs = new String[] {ac.getDetectors().get(0).getFieldName(), ac.getDetectors().get(0).getOverFieldName()}; + } else { + outputs = new String[] {ac.getDetectors().get(0).getFieldName()}; + } + if (randomBoolean()) { + builder.setModelDebugConfig(new ModelDebugConfig(randomDouble(), randomAsciiOfLength(10))); + } + builder.setIgnoreDowntime(randomFrom(IgnoreDowntime.values())); + if (randomBoolean()) { + builder.setRenormalizationWindowDays(randomNonNegativeLong()); + } + if (randomBoolean()) { + builder.setBackgroundPersistInterval(randomNonNegativeLong()); + } + if (randomBoolean()) { + builder.setModelSnapshotRetentionDays(randomNonNegativeLong()); + } + if (randomBoolean()) { + builder.setResultsRetentionDays(randomNonNegativeLong()); + } + if (randomBoolean()) { + builder.setCustomSettings(Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10))); + } + if (randomBoolean()) { + builder.setModelSnapshotId(randomAsciiOfLength(10)); + } + if (randomBoolean()) { + builder.setIndexName(randomValidJobId()); + } + return builder.build(); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java new file mode 100644 index 00000000000..475abdf30a8 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mock; + +public class JobUpdateTests extends AbstractSerializingTestCase { + + @Override + protected JobUpdate createTestInstance() { + String description = null; + if (randomBoolean()) { + description = randomAsciiOfLength(20); + } + List detectorUpdates = null; + if (randomBoolean()) { + int size = randomInt(10); + detectorUpdates = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + String detectorDescription = null; + if (randomBoolean()) { + detectorDescription = randomAsciiOfLength(12); + } + List detectionRules = null; + if (randomBoolean()) { + detectionRules = new ArrayList<>(); + Condition condition = new Condition(Operator.GT, "5"); + detectionRules.add(new DetectionRule("foo", null, Connective.OR, Collections.singletonList( + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))); + } + detectorUpdates.add(new JobUpdate.DetectorUpdate(i, detectorDescription, detectionRules)); + } + } + ModelDebugConfig modelDebugConfig = null; + if (randomBoolean()) { + modelDebugConfig = new ModelDebugConfig(randomDouble(), randomAsciiOfLength(10)); + } + AnalysisLimits analysisLimits = null; + if (randomBoolean()) { + analysisLimits = new AnalysisLimits(randomNonNegativeLong(), randomNonNegativeLong()); + } + Long renormalizationWindowDays = null; + if (randomBoolean()) { + renormalizationWindowDays = randomNonNegativeLong(); + } + Long backgroundPersistInterval = null; + if (randomBoolean()) { + backgroundPersistInterval = randomNonNegativeLong(); + } + Long modelSnapshotRetentionDays = null; + if (randomBoolean()) { + modelSnapshotRetentionDays = randomNonNegativeLong(); + } + Long resultsRetentionDays = null; + if (randomBoolean()) { + resultsRetentionDays = randomNonNegativeLong(); + } + List categorizationFilters = null; + if (randomBoolean()) { + categorizationFilters = Arrays.asList(generateRandomStringArray(10, 10, false)); + } + Map customSettings = null; + if (randomBoolean()) { + customSettings = Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10)); + } + + return new JobUpdate(description, detectorUpdates, modelDebugConfig, analysisLimits, backgroundPersistInterval, + renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings); + } + + @Override + protected Writeable.Reader instanceReader() { + return JobUpdate::new; + } + + @Override + protected JobUpdate parseInstance(XContentParser parser) { + return JobUpdate.PARSER.apply(parser, null); + } + + public void testMergeWithJob() { + List detectorUpdates = new ArrayList<>(); + List detectionRules1 = Collections.singletonList(new DetectionRule("client", null, Connective.OR, + Collections.singletonList( + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5"), null)))); + detectorUpdates.add(new JobUpdate.DetectorUpdate(0, "description-1", detectionRules1)); + List detectionRules2 = Collections.singletonList(new DetectionRule("host", null, Connective.OR, + Collections.singletonList( + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5"), null)))); + detectorUpdates.add(new JobUpdate.DetectorUpdate(1, "description-2", detectionRules2)); + + ModelDebugConfig modelDebugConfig = new ModelDebugConfig(randomDouble(), randomAsciiOfLength(10)); + AnalysisLimits analysisLimits = new AnalysisLimits(randomNonNegativeLong(), randomNonNegativeLong()); + List categorizationFilters = Arrays.asList(generateRandomStringArray(10, 10, false)); + Map customSettings = Collections.singletonMap(randomAsciiOfLength(10), randomAsciiOfLength(10)); + + JobUpdate update = new JobUpdate("updated_description", detectorUpdates, modelDebugConfig, + analysisLimits, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + categorizationFilters, customSettings); + + Job.Builder jobBuilder = new Job.Builder("foo"); + Detector.Builder d1 = new Detector.Builder("info_content", "domain"); + d1.setOverFieldName("client"); + Detector.Builder d2 = new Detector.Builder("min", "field"); + d2.setOverFieldName("host"); + AnalysisConfig.Builder ac = new AnalysisConfig.Builder(Arrays.asList(d1.build(), d2.build())); + ac.setCategorizationFieldName("cat_field"); + jobBuilder.setAnalysisConfig(ac); + + Job updatedJob = update.mergeWithJob(jobBuilder.build()); + + assertEquals(update.getDescription(), updatedJob.getDescription()); + assertEquals(update.getModelDebugConfig(), updatedJob.getModelDebugConfig()); + assertEquals(update.getAnalysisLimits(), updatedJob.getAnalysisLimits()); + assertEquals(update.getRenormalizationWindowDays(), updatedJob.getRenormalizationWindowDays()); + assertEquals(update.getBackgroundPersistInterval(), updatedJob.getBackgroundPersistInterval()); + assertEquals(update.getModelSnapshotRetentionDays(), updatedJob.getModelSnapshotRetentionDays()); + assertEquals(update.getResultsRetentionDays(), updatedJob.getResultsRetentionDays()); + assertEquals(update.getCategorizationFilters(), updatedJob.getAnalysisConfig().getCategorizationFilters()); + assertEquals(update.getCustomSettings(), updatedJob.getCustomSettings()); + for (JobUpdate.DetectorUpdate detectorUpdate : update.getDetectorUpdates()) { + assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getIndex()).getDetectorDescription()); + assertEquals(detectorUpdate.getDescription(), + updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getIndex()).getDetectorDescription()); + assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getIndex()).getDetectorDescription()); + assertEquals(detectorUpdate.getRules(), + updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getIndex()).getDetectorRules()); + } + } + + public void testIsAutodetectProcessUpdate() { + JobUpdate update = new JobUpdate(null, null, null, null, null, null, null, null, null, null); + assertFalse(update.isAutodetectProcessUpdate()); + update = new JobUpdate(null, null, new ModelDebugConfig(1.0, "ff"), null, null, null, null, null, null, null); + assertTrue(update.isAutodetectProcessUpdate()); + update = new JobUpdate(null, Arrays.asList(mock(JobUpdate.DetectorUpdate.class)), null, null, null, null, null, null, null, null); + assertTrue(update.isAutodetectProcessUpdate()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java new file mode 100644 index 00000000000..62b7f0ec080 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class MlFilterTests extends AbstractSerializingTestCase { + + @Override + protected MlFilter createTestInstance() { + int size = randomInt(10); + List items = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + items.add(randomAsciiOfLengthBetween(1, 20)); + } + return new MlFilter(randomAsciiOfLengthBetween(1, 20), items); + } + + @Override + protected Reader instanceReader() { + return MlFilter::new; + } + + @Override + protected MlFilter parseInstance(XContentParser parser) { + return MlFilter.PARSER.apply(parser, null); + } + + public void testNullId() { + NullPointerException ex = expectThrows(NullPointerException.class, () -> new MlFilter(null, Collections.emptyList())); + assertEquals(MlFilter.ID.getPreferredName() + " must not be null", ex.getMessage()); + } + + public void testNullItems() { + NullPointerException ex = + expectThrows(NullPointerException.class, () -> new MlFilter(randomAsciiOfLengthBetween(1, 20), null)); + assertEquals(MlFilter.ITEMS.getPreferredName() + " must not be null", ex.getMessage()); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ModelDebugConfigTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ModelDebugConfigTests.java new file mode 100644 index 00000000000..6b70e6d61cd --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/ModelDebugConfigTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +public class ModelDebugConfigTests extends AbstractSerializingTestCase { + + public void testVerify_GivenBoundPercentileLessThanZero() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new ModelDebugConfig(-1.0, "")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE, ""), e.getMessage()); + } + + public void testVerify_GivenBoundPercentileGreaterThan100() { + IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> new ModelDebugConfig(100.1, "")); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_MODEL_DEBUG_CONFIG_INVALID_BOUNDS_PERCENTILE, ""), e.getMessage()); + } + + public void testVerify_GivenValid() { + new ModelDebugConfig(93.0, ""); + new ModelDebugConfig(93.0, "foo,bar"); + } + + @Override + protected ModelDebugConfig createTestInstance() { + return new ModelDebugConfig(randomDouble(), randomAsciiOfLengthBetween(1, 30)); + } + + @Override + protected Reader instanceReader() { + return ModelDebugConfig::new; + } + + @Override + protected ModelDebugConfig parseInstance(XContentParser parser) { + return ModelDebugConfig.PARSER.apply(parser, null); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java new file mode 100644 index 00000000000..695d7eea087 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.regex.Pattern; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class OperatorTests extends ESTestCase { + + public void testFromString() { + assertEquals(Operator.fromString("eq"), Operator.EQ); + assertEquals(Operator.fromString("gt"), Operator.GT); + assertEquals(Operator.fromString("gte"), Operator.GTE); + assertEquals(Operator.fromString("lte"), Operator.LTE); + assertEquals(Operator.fromString("lt"), Operator.LT); + assertEquals(Operator.fromString("match"), Operator.MATCH); + assertEquals(Operator.fromString("Gt"), Operator.GT); + assertEquals(Operator.fromString("EQ"), Operator.EQ); + assertEquals(Operator.fromString("GTE"), Operator.GTE); + assertEquals(Operator.fromString("Match"), Operator.MATCH); + } + + public void testToString() { + assertEquals("eq", Operator.EQ.toString()); + assertEquals("gt", Operator.GT.toString()); + assertEquals("gte", Operator.GTE.toString()); + assertEquals("lte", Operator.LTE.toString()); + assertEquals("lt", Operator.LT.toString()); + assertEquals("match", Operator.MATCH.toString()); + } + + public void testTest() { + assertTrue(Operator.GT.expectsANumericArgument()); + assertTrue(Operator.GT.test(1.0, 0.0)); + assertFalse(Operator.GT.test(0.0, 1.0)); + + assertTrue(Operator.GTE.expectsANumericArgument()); + assertTrue(Operator.GTE.test(1.0, 0.0)); + assertTrue(Operator.GTE.test(1.0, 1.0)); + assertFalse(Operator.GTE.test(0.0, 1.0)); + + assertTrue(Operator.EQ.expectsANumericArgument()); + assertTrue(Operator.EQ.test(0.0, 0.0)); + assertFalse(Operator.EQ.test(1.0, 0.0)); + + assertTrue(Operator.LT.expectsANumericArgument()); + assertTrue(Operator.LT.test(0.0, 1.0)); + assertFalse(Operator.LT.test(0.0, 0.0)); + + assertTrue(Operator.LTE.expectsANumericArgument()); + assertTrue(Operator.LTE.test(0.0, 1.0)); + assertTrue(Operator.LTE.test(1.0, 1.0)); + assertFalse(Operator.LTE.test(1.0, 0.0)); + } + + public void testMatch() { + assertFalse(Operator.MATCH.expectsANumericArgument()); + assertFalse(Operator.MATCH.test(0.0, 1.0)); + + Pattern pattern = Pattern.compile("^aa.*"); + + assertTrue(Operator.MATCH.match(pattern, "aaaaa")); + assertFalse(Operator.MATCH.match(pattern, "bbaaa")); + } + + public void testValidOrdinals() { + assertThat(Operator.EQ.ordinal(), equalTo(0)); + assertThat(Operator.GT.ordinal(), equalTo(1)); + assertThat(Operator.GTE.ordinal(), equalTo(2)); + assertThat(Operator.LT.ordinal(), equalTo(3)); + assertThat(Operator.LTE.ordinal(), equalTo(4)); + assertThat(Operator.MATCH.ordinal(), equalTo(5)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.EQ.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.GT.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.GTE.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.LT.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(3)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.LTE.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(4)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + Operator.MATCH.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(5)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.EQ)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.GT)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.GTE)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(3); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.LT)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(4); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.LTE)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(5); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(Operator.readFromStream(in), equalTo(Operator.MATCH)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(7, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + Operator.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown Operator ordinal [")); + } + } + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java new file mode 100644 index 00000000000..28312c01135 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.test.ESTestCase; + +public class RuleActionTests extends ESTestCase { + + public void testForString() { + assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("filter_results")); + assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("FILTER_RESULTS")); + assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("fiLTer_Results")); + } + + public void testToString() { + assertEquals("filter_results", RuleAction.FILTER_RESULTS.toString()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java new file mode 100644 index 00000000000..029bf08e8c6 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java @@ -0,0 +1,220 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.messages.Messages; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +public class RuleConditionTests extends AbstractSerializingTestCase { + + @Override + protected RuleCondition createTestInstance() { + Condition condition = null; + String fieldName = null; + String valueFilter = null; + String fieldValue = null; + RuleConditionType r = randomFrom(RuleConditionType.values()); + switch (r) { + case CATEGORICAL: + valueFilter = randomAsciiOfLengthBetween(1, 20); + if (randomBoolean()) { + fieldName = randomAsciiOfLengthBetween(1, 20); + } + break; + default: + // no need to randomize, it is properly randomily tested in + // ConditionTest + condition = new Condition(Operator.LT, Double.toString(randomDouble())); + if (randomBoolean()) { + fieldName = randomAsciiOfLengthBetween(1, 20); + fieldValue = randomAsciiOfLengthBetween(1, 20); + } + break; + } + return new RuleCondition(r, fieldName, fieldValue, condition, valueFilter); + } + + @Override + protected Reader instanceReader() { + return RuleCondition::new; + } + + @Override + protected RuleCondition parseInstance(XContentParser parser) { + return RuleCondition.PARSER.apply(parser, null); + } + + public void testConstructor() { + RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter"); + assertEquals(RuleConditionType.CATEGORICAL, condition.getConditionType()); + assertNull(condition.getFieldName()); + assertNull(condition.getFieldValue()); + assertNull(condition.getCondition()); + } + + public void testEqualsGivenSameObject() { + RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter"); + assertTrue(condition.equals(condition)); + } + + public void testEqualsGivenString() { + assertFalse(new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "filter").equals("a string")); + } + + public void testEqualsGivenDifferentType() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter"); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentFieldName() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricNameaaa", "cpu", + new Condition(Operator.LT, "5"), null); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentFieldValue() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpuaaa", + new Condition(Operator.LT, "5"), null); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentCondition() { + RuleCondition condition1 = createFullyPopulated(); + RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpu", + new Condition(Operator.GT, "5"), null); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + public void testEqualsGivenDifferentValueFilter() { + RuleCondition condition1 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "myFilter"); + RuleCondition condition2 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "myFilteraaa"); + assertFalse(condition1.equals(condition2)); + assertFalse(condition2.equals(condition1)); + } + + private static RuleCondition createFullyPopulated() { + return new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpu", new Condition(Operator.LT, "5"), null); + } + + public void testVerify_GivenCategoricalWithCondition() { + Condition condition = new Condition(Operator.MATCH, "text"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, condition, null)); + assertEquals("Invalid detector rule: a categorical rule_condition does not support condition", e.getMessage()); + } + + public void testVerify_GivenCategoricalWithFieldValue() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.CATEGORICAL, "metric", "CPU", null, null)); + assertEquals("Invalid detector rule: a categorical rule_condition does not support field_value", e.getMessage()); + } + + public void testVerify_GivenCategoricalWithoutValueFilter() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, null)); + assertEquals("Invalid detector rule: a categorical rule_condition requires value_filter to be set", e.getMessage()); + } + + public void testVerify_GivenNumericalActualWithValueFilter() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); + assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + } + + public void testVerify_GivenNumericalActualWithoutCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, null)); + assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage()); + } + + public void testVerify_GivenNumericalActualWithFieldNameButNoFieldValue() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", null, new Condition(Operator.LT, "5"), null)); + assertEquals("Invalid detector rule: a numerical rule_condition with field_name requires that field_value is set", e.getMessage()); + } + + public void testVerify_GivenNumericalTypicalWithValueFilter() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); + assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + } + + public void testVerify_GivenNumericalTypicalWithoutCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, null)); + assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage()); + } + + public void testVerify_GivenNumericalDiffAbsWithValueFilter() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, "myFilter")); + assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + } + + public void testVerify_GivenNumericalDiffAbsWithoutCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, null)); + assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage()); + } + + public void testVerify_GivenFieldValueWithoutFieldName() { + Condition condition = new Condition(Operator.LTE, "5"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, "foo", condition, null)); + assertEquals("Invalid detector rule: missing field_name in rule_condition where field_value 'foo' is set", e.getMessage()); + } + + public void testVerify_GivenNumericalAndOperatorEquals() { + Condition condition = new Condition(Operator.EQ, "5"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + assertEquals("Invalid detector rule: operator 'eq' is not allowed", e.getMessage()); + } + + public void testVerify_GivenNumericalAndOperatorMatch() { + Condition condition = new Condition(Operator.MATCH, "aaa"); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + assertEquals("Invalid detector rule: operator 'match' is not allowed", e.getMessage()); + } + + public void testVerify_GivenDetectionRuleWithInvalidCondition() { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "invalid"), + null)); + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, "invalid"), e.getMessage()); + } + + public void testVerify_GivenValidCategorical() { + // no validation error: + new RuleCondition(RuleConditionType.CATEGORICAL, "metric", null, null, "myFilter"); + } + + public void testVerify_GivenValidNumericalActual() { + // no validation error: + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", "cpu", new Condition(Operator.GT, "5"), null); + } + + public void testVerify_GivenValidNumericalTypical() { + // no validation error: + new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", "cpu", new Condition(Operator.GTE, "5"), null); + } + + public void testVerify_GivenValidNumericalDiffAbs() { + // no validation error: + new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, "metric", "cpu", new Condition(Operator.LT, "5"), null); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTypeTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTypeTests.java new file mode 100644 index 00000000000..288fd85c8f9 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTypeTests.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.config; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class RuleConditionTypeTests extends ESTestCase { + + public void testFromString() { + assertEquals(RuleConditionType.CATEGORICAL, RuleConditionType.fromString("categorical")); + assertEquals(RuleConditionType.CATEGORICAL, RuleConditionType.fromString("CATEGORICAL")); + assertEquals(RuleConditionType.NUMERICAL_ACTUAL, RuleConditionType.fromString("numerical_actual")); + assertEquals(RuleConditionType.NUMERICAL_ACTUAL, RuleConditionType.fromString("NUMERICAL_ACTUAL")); + assertEquals(RuleConditionType.NUMERICAL_TYPICAL, RuleConditionType.fromString("numerical_typical")); + assertEquals(RuleConditionType.NUMERICAL_TYPICAL, RuleConditionType.fromString("NUMERICAL_TYPICAL")); + assertEquals(RuleConditionType.NUMERICAL_DIFF_ABS, RuleConditionType.fromString("numerical_diff_abs")); + assertEquals(RuleConditionType.NUMERICAL_DIFF_ABS, RuleConditionType.fromString("NUMERICAL_DIFF_ABS")); + } + + public void testToString() { + assertEquals("categorical", RuleConditionType.CATEGORICAL.toString()); + assertEquals("numerical_actual", RuleConditionType.NUMERICAL_ACTUAL.toString()); + assertEquals("numerical_typical", RuleConditionType.NUMERICAL_TYPICAL.toString()); + assertEquals("numerical_diff_abs", RuleConditionType.NUMERICAL_DIFF_ABS.toString()); + } + + public void testValidOrdinals() { + assertThat(RuleConditionType.CATEGORICAL.ordinal(), equalTo(0)); + assertThat(RuleConditionType.NUMERICAL_ACTUAL.ordinal(), equalTo(1)); + assertThat(RuleConditionType.NUMERICAL_TYPICAL.ordinal(), equalTo(2)); + assertThat(RuleConditionType.NUMERICAL_DIFF_ABS.ordinal(), equalTo(3)); + } + + public void testwriteTo() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.CATEGORICAL.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(0)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.NUMERICAL_ACTUAL.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(1)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.NUMERICAL_TYPICAL.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(2)); + } + } + + try (BytesStreamOutput out = new BytesStreamOutput()) { + RuleConditionType.NUMERICAL_DIFF_ABS.writeTo(out); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(in.readVInt(), equalTo(3)); + } + } + } + + public void testReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(0); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.CATEGORICAL)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(1); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_ACTUAL)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(2); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_TYPICAL)); + } + } + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(3); + try (StreamInput in = out.bytes().streamInput()) { + assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_DIFF_ABS)); + } + } + } + + public void testInvalidReadFrom() throws Exception { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.writeVInt(randomIntBetween(4, Integer.MAX_VALUE)); + try (StreamInput in = out.bytes().streamInput()) { + RuleConditionType.readFromStream(in); + fail("Expected IOException"); + } catch (IOException e) { + assertThat(e.getMessage(), containsString("Unknown RuleConditionType ordinal [")); + } + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/messages/MessagesTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/messages/MessagesTests.java new file mode 100644 index 00000000000..b85c0740d2b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/messages/MessagesTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.messages; + +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.test.ESTestCase; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.List; +import java.util.ResourceBundle; +import java.util.Set; + +@SuppressForbidden(reason = "Need to use reflection to make sure all constants are resolvable") +public class MessagesTests extends ESTestCase { + + public void testAllStringsAreInTheResourceBundle() throws IllegalArgumentException, IllegalAccessException { + ResourceBundle bundle = Messages.load(); + + // get all the public string constants + // excluding BUNDLE_NAME + List publicStrings = new ArrayList<>(); + Field[] allFields = Messages.class.getDeclaredFields(); + for (Field field : allFields) { + if (Modifier.isPublic(field.getModifiers())) { + if (field.getType() == String.class && field.getName() != "BUNDLE_NAME") { + publicStrings.add(field); + } + } + } + + assertTrue(bundle.keySet().size() > 0); + + // Make debugging easier- print any keys not defined in Messages + Set keys = bundle.keySet(); + for (Field field : publicStrings) { + String key = (String) field.get(new String()); + keys.remove(key); + } + + assertEquals(bundle.keySet().size(), publicStrings.size()); + + for (Field field : publicStrings) { + String key = (String) field.get(new String()); + + assertTrue("String constant " + field.getName() + " = " + key + " not in resource bundle", bundle.containsKey(key)); + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/AllocationTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/AllocationTests.java new file mode 100644 index 00000000000..1814e9d731f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/AllocationTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.metadata; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; + +public class AllocationTests extends AbstractSerializingTestCase { + + @Override + protected Allocation createTestInstance() { + String nodeId = randomBoolean() ? randomAsciiOfLength(10) : null; + String jobId = randomAsciiOfLength(10); + boolean ignoreDowntime = randomBoolean(); + JobState jobState = randomFrom(JobState.values()); + String stateReason = randomBoolean() ? randomAsciiOfLength(10) : null; + return new Allocation(nodeId, jobId, ignoreDowntime, jobState, stateReason); + } + + @Override + protected Writeable.Reader instanceReader() { + return Allocation::new; + } + + @Override + protected Allocation parseInstance(XContentParser parser) { + return Allocation.PARSER.apply(parser, null).build(); + } + + public void testUnsetIgnoreDownTime() { + Allocation allocation = new Allocation("_node_id", "_job_id", true, JobState.OPENING, null); + assertTrue(allocation.isIgnoreDowntime()); + Allocation.Builder builder = new Allocation.Builder(allocation); + builder.setState(JobState.OPENED); + allocation = builder.build(); + assertFalse(allocation.isIgnoreDowntime()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/MlInitializationServiceTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/MlInitializationServiceTests.java new file mode 100644 index 00000000000..624bdd65c0f --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/MlInitializationServiceTests.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.metadata; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.notifications.Auditor; + +import java.net.InetAddress; +import java.util.concurrent.ExecutorService; + +import static org.elasticsearch.mock.orig.Mockito.doAnswer; +import static org.elasticsearch.mock.orig.Mockito.times; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class MlInitializationServiceTests extends ESTestCase { + + public void testInitialize() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executorService); + + ClusterService clusterService = mock(ClusterService.class); + JobProvider jobProvider = mock(JobProvider.class); + MlInitializationService initializationService = + new MlInitializationService(Settings.EMPTY, threadPool, clusterService, jobProvider); + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) + .localNodeId("_node_id") + .masterNodeId("_node_id")) + .metaData(MetaData.builder()) + .build(); + initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + + verify(clusterService, times(1)).submitStateUpdateTask(eq("install-ml-metadata"), any()); + verify(jobProvider, times(1)).createNotificationMessageIndex(any()); + verify(jobProvider, times(1)).createMetaIndex(any()); + verify(jobProvider, times(1)).createJobStateIndex(any()); + } + + public void testInitialize_noMasterNode() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executorService); + + ClusterService clusterService = mock(ClusterService.class); + JobProvider jobProvider = mock(JobProvider.class); + MlInitializationService initializationService = + new MlInitializationService(Settings.EMPTY, threadPool, clusterService, jobProvider); + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT))) + .metaData(MetaData.builder()) + .build(); + initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + + verify(clusterService, times(0)).submitStateUpdateTask(eq("install-ml-metadata"), any()); + verify(jobProvider, times(0)).createNotificationMessageIndex(any()); + verify(jobProvider, times(0)).createMetaIndex(any()); + verify(jobProvider, times(0)).createJobStateIndex(any()); + } + + public void testInitialize_alreadyInitialized() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executorService); + + ClusterService clusterService = mock(ClusterService.class); + JobProvider jobProvider = mock(JobProvider.class); + MlInitializationService initializationService = + new MlInitializationService(Settings.EMPTY, threadPool, clusterService, jobProvider); + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) + .localNodeId("_node_id") + .masterNodeId("_node_id")) + .metaData(MetaData.builder() + .put(IndexMetaData.builder(Auditor.NOTIFICATIONS_INDEX).settings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + )) + .put(IndexMetaData.builder(JobProvider.ML_META_INDEX).settings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + )) + .put(IndexMetaData.builder(AnomalyDetectorsIndex.jobStateIndexName()).settings(Settings.builder() + .put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + )) + .putCustom(MlMetadata.TYPE, new MlMetadata.Builder().build())) + .build(); + initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + + verify(clusterService, times(0)).submitStateUpdateTask(eq("install-ml-metadata"), any()); + verify(jobProvider, times(0)).createNotificationMessageIndex(any()); + verify(jobProvider, times(0)).createMetaIndex(any()); + verify(jobProvider, times(0)).createJobStateIndex(any()); + } + + public void testInitialize_onlyOnce() throws Exception { + ThreadPool threadPool = mock(ThreadPool.class); + ExecutorService executorService = mock(ExecutorService.class); + doAnswer(invocation -> { + ((Runnable) invocation.getArguments()[0]).run(); + return null; + }).when(executorService).execute(any(Runnable.class)); + when(threadPool.executor(ThreadPool.Names.GENERIC)).thenReturn(executorService); + + ClusterService clusterService = mock(ClusterService.class); + JobProvider jobProvider = mock(JobProvider.class); + MlInitializationService initializationService = + new MlInitializationService(Settings.EMPTY, threadPool, clusterService, jobProvider); + + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .nodes(DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_id", new TransportAddress(InetAddress.getLoopbackAddress(), 9200), Version.CURRENT)) + .localNodeId("_node_id") + .masterNodeId("_node_id")) + .metaData(MetaData.builder()) + .build(); + initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + initializationService.clusterChanged(new ClusterChangedEvent("_source", cs, cs)); + + verify(clusterService, times(1)).submitStateUpdateTask(eq("install-ml-metadata"), any()); + verify(jobProvider, times(1)).createNotificationMessageIndex(any()); + verify(jobProvider, times(1)).createMetaIndex(any()); + verify(jobProvider, times(1)).createJobStateIndex(any()); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/MlMetadataTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/MlMetadataTests.java new file mode 100644 index 00000000000..27c4e3ec4ba --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/metadata/MlMetadataTests.java @@ -0,0 +1,297 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.metadata; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.ml.action.StartDatafeedAction; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig; +import org.elasticsearch.xpack.ml.datafeed.DatafeedConfigTests; +import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.JobState; +import org.elasticsearch.xpack.ml.job.config.JobTests; +import org.elasticsearch.xpack.ml.support.AbstractSerializingTestCase; +import org.elasticsearch.xpack.persistent.PersistentTasksInProgress; + +import java.io.IOException; +import java.util.Collections; + +import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedConfig; +import static org.elasticsearch.xpack.ml.datafeed.DatafeedJobRunnerTests.createDatafeedJob; +import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.sameInstance; + +public class MlMetadataTests extends AbstractSerializingTestCase { + + @Override + protected MlMetadata createTestInstance() { + MlMetadata.Builder builder = new MlMetadata.Builder(); + int numJobs = randomIntBetween(0, 10); + for (int i = 0; i < numJobs; i++) { + Job job = JobTests.createRandomizedJob(); + if (randomBoolean()) { + DatafeedConfig datafeedConfig = DatafeedConfigTests.createRandomizedDatafeedConfig(job.getId()); + if (datafeedConfig.getAggregations() != null) { + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(job.getAnalysisConfig().getDetectors()); + analysisConfig.setSummaryCountFieldName("doc_count"); + Job.Builder jobBuilder = new Job.Builder(job); + jobBuilder.setAnalysisConfig(analysisConfig); + job = jobBuilder.build(); + } + builder.putJob(job, false); + builder.putDatafeed(datafeedConfig); + } else { + builder.putJob(job, false); + } + if (randomBoolean()) { + builder.updateState(job.getId(), JobState.OPENING, randomBoolean() ? "first reason" : null); + if (randomBoolean()) { + builder.updateState(job.getId(), JobState.OPENED, randomBoolean() ? "second reason" : null); + if (randomBoolean()) { + builder.updateState(job.getId(), JobState.CLOSING, randomBoolean() ? "third reason" : null); + } + } + } + } + return builder.build(); + } + + @Override + protected Writeable.Reader instanceReader() { + return MlMetadata::new; + } + + @Override + protected MlMetadata parseInstance(XContentParser parser) { + return MlMetadata.ML_METADATA_PARSER.apply(parser, null).build(); + } + + @Override + protected XContentBuilder toXContent(MlMetadata instance, XContentType contentType) throws IOException { + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (randomBoolean()) { + builder.prettyPrint(); + } + // In Metadata.Builder#toXContent(...) custom metadata always gets wrapped in an start and end object, + // so we simulate that here. The MlMetadata depends on that as it direct starts to write a start array. + builder.startObject(); + instance.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + return builder; + } + + public void testPutJob() { + Job job1 = buildJobBuilder("1").build(); + Job job2 = buildJobBuilder("2").build(); + + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + builder.putJob(job2, false); + + MlMetadata result = builder.build(); + assertThat(result.getJobs().get("1"), sameInstance(job1)); + assertThat(result.getAllocations().get("1").getState(), equalTo(JobState.CLOSED)); + assertThat(result.getDatafeeds().get("1"), nullValue()); + assertThat(result.getJobs().get("2"), sameInstance(job2)); + assertThat(result.getAllocations().get("2").getState(), equalTo(JobState.CLOSED)); + assertThat(result.getDatafeeds().get("2"), nullValue()); + + builder = new MlMetadata.Builder(result); + + MlMetadata.Builder builderReference = builder; + ResourceAlreadyExistsException e = expectThrows(ResourceAlreadyExistsException.class, () -> builderReference.putJob(job2, false)); + assertEquals("The job cannot be created with the Id '2'. The Id is already used.", e.getMessage()); + Job job2Attempt2 = buildJobBuilder("2").build(); + builder.putJob(job2Attempt2, true); + + result = builder.build(); + assertThat(result.getJobs().size(), equalTo(2)); + assertThat(result.getJobs().get("1"), sameInstance(job1)); + assertThat(result.getJobs().get("2"), sameInstance(job2Attempt2)); + } + + public void testRemoveJob() { + Job job1 = buildJobBuilder("1").build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + + MlMetadata result = builder.build(); + assertThat(result.getJobs().get("1"), sameInstance(job1)); + assertThat(result.getAllocations().get("1").getState(), equalTo(JobState.CLOSED)); + assertThat(result.getDatafeeds().get("1"), nullValue()); + + builder = new MlMetadata.Builder(result); + builder.updateState("1", JobState.DELETING, null); + assertThat(result.getJobs().get("1"), sameInstance(job1)); + assertThat(result.getAllocations().get("1").getState(), equalTo(JobState.CLOSED)); + assertThat(result.getDatafeeds().get("1"), nullValue()); + + builder.deleteJob("1"); + result = builder.build(); + assertThat(result.getJobs().get("1"), nullValue()); + assertThat(result.getAllocations().get("1"), nullValue()); + assertThat(result.getDatafeeds().get("1"), nullValue()); + } + + public void testRemoveJob_failBecauseJobIsOpen() { + Job job1 = buildJobBuilder("1").build(); + MlMetadata.Builder builder1 = new MlMetadata.Builder(); + builder1.putJob(job1, false); + builder1.updateState("1", JobState.OPENING, null); + builder1.updateState("1", JobState.OPENED, null); + + MlMetadata result = builder1.build(); + assertThat(result.getJobs().get("1"), sameInstance(job1)); + assertThat(result.getAllocations().get("1").getState(), equalTo(JobState.OPENED)); + assertThat(result.getDatafeeds().get("1"), nullValue()); + + MlMetadata.Builder builder2 = new MlMetadata.Builder(result); + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> builder2.deleteJob("1")); + assertThat(e.status(), equalTo(RestStatus.CONFLICT)); + } + + public void testRemoveJob_failDatafeedRefersToJob() { + Job job1 = createDatafeedJob().build(); + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", job1.getId()).build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + builder.putDatafeed(datafeedConfig1); + + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> builder.deleteJob(job1.getId())); + assertThat(e.status(), equalTo(RestStatus.CONFLICT)); + String expectedMsg = "Cannot delete job [" + job1.getId() + "] while datafeed [" + datafeedConfig1.getId() + "] refers to it"; + assertThat(e.getMessage(), equalTo(expectedMsg)); + } + + public void testRemoveJob_failBecauseJobDoesNotExist() { + MlMetadata.Builder builder1 = new MlMetadata.Builder(); + expectThrows(ResourceNotFoundException.class, () -> builder1.deleteJob("1")); + } + + public void testCrudDatafeed() { + Job job1 = createDatafeedJob().build(); + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", job1.getId()).build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + builder.putDatafeed(datafeedConfig1); + + MlMetadata result = builder.build(); + assertThat(result.getJobs().get("foo"), sameInstance(job1)); + assertThat(result.getAllocations().get("foo").getState(), equalTo(JobState.CLOSED)); + assertThat(result.getDatafeeds().get("datafeed1"), sameInstance(datafeedConfig1)); + + builder = new MlMetadata.Builder(result); + builder.removeDatafeed("datafeed1", new PersistentTasksInProgress(0, Collections.emptyMap())); + result = builder.build(); + assertThat(result.getJobs().get("foo"), sameInstance(job1)); + assertThat(result.getAllocations().get("foo").getState(), equalTo(JobState.CLOSED)); + assertThat(result.getDatafeeds().get("datafeed1"), nullValue()); + } + + public void testPutDatafeed_failBecauseJobDoesNotExist() { + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", "missing-job").build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + + expectThrows(ResourceNotFoundException.class, () -> builder.putDatafeed(datafeedConfig1)); + } + + public void testPutDatafeed_failBecauseDatafeedIdIsAlreadyTaken() { + Job job1 = createDatafeedJob().build(); + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", job1.getId()).build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + builder.putDatafeed(datafeedConfig1); + + expectThrows(ResourceAlreadyExistsException.class, () -> builder.putDatafeed(datafeedConfig1)); + } + + public void testPutDatafeed_failBecauseJobAlreadyHasDatafeed() { + Job job1 = createDatafeedJob().build(); + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", job1.getId()).build(); + DatafeedConfig datafeedConfig2 = createDatafeedConfig("datafeed2", job1.getId()).build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + builder.putDatafeed(datafeedConfig1); + + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, + () -> builder.putDatafeed(datafeedConfig2)); + assertThat(e.status(), equalTo(RestStatus.CONFLICT)); + } + + public void testPutDatafeed_failBecauseJobIsNotCompatibleForDatafeed() { + Job.Builder job1 = createDatafeedJob(); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(job1.build().getAnalysisConfig()); + analysisConfig.setLatency(3600L); + job1.setAnalysisConfig(analysisConfig); + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", job1.getId()).build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1.build(), false); + + expectThrows(IllegalArgumentException.class, () -> builder.putDatafeed(datafeedConfig1)); + } + + public void testRemoveDatafeed_failBecauseDatafeedStarted() { + Job job1 = createDatafeedJob().build(); + DatafeedConfig datafeedConfig1 = createDatafeedConfig("datafeed1", job1.getId()).build(); + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(job1, false); + builder.putDatafeed(datafeedConfig1); + builder.updateState("foo", JobState.OPENING, null); + builder.updateState("foo", JobState.OPENED, null); + + MlMetadata result = builder.build(); + assertThat(result.getJobs().get("foo"), sameInstance(job1)); + assertThat(result.getAllocations().get("foo").getState(), equalTo(JobState.OPENED)); + assertThat(result.getDatafeeds().get("datafeed1"), sameInstance(datafeedConfig1)); + + StartDatafeedAction.Request request = new StartDatafeedAction.Request("datafeed1", 0L); + PersistentTasksInProgress.PersistentTaskInProgress taskInProgress = + new PersistentTasksInProgress.PersistentTaskInProgress<>(0, StartDatafeedAction.NAME, request, null); + PersistentTasksInProgress tasksInProgress = + new PersistentTasksInProgress(1, Collections.singletonMap(taskInProgress.getId(), taskInProgress)); + + MlMetadata.Builder builder2 = new MlMetadata.Builder(result); + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, + () -> builder2.removeDatafeed("datafeed1", tasksInProgress)); + assertThat(e.status(), equalTo(RestStatus.CONFLICT)); + } + + public void testUpdateAllocation_setFinishedTime() { + MlMetadata.Builder builder = new MlMetadata.Builder(); + builder.putJob(buildJobBuilder("my_job_id").build(), false); + builder.updateState("my_job_id", JobState.OPENING, null); + + builder.updateState("my_job_id", JobState.OPENED, null); + MlMetadata mlMetadata = builder.build(); + assertThat(mlMetadata.getJobs().get("my_job_id").getFinishedTime(), nullValue()); + + builder.updateState("my_job_id", JobState.CLOSED, null); + mlMetadata = builder.build(); + assertThat(mlMetadata.getJobs().get("my_job_id").getFinishedTime(), notNullValue()); + } + + public void testUpdateState_failBecauseJobDoesNotExist() { + MlMetadata.Builder builder = new MlMetadata.Builder(); + expectThrows(ResourceNotFoundException.class, () -> builder.updateState("missing-job", JobState.CLOSED, "for testting")); + } + + public void testSetIgnoreDowntime_failBecauseJobDoesNotExist() { + MlMetadata.Builder builder = new MlMetadata.Builder(); + expectThrows(ResourceNotFoundException.class, () -> builder.setIgnoreDowntime("missing-job")); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java new file mode 100644 index 00000000000..98ad716743b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BatchedDocumentsIteratorTests.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.action.search.ClearScrollRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Deque; +import java.util.List; +import java.util.NoSuchElementException; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/prelert-legacy/issues/127") +public class BatchedDocumentsIteratorTests extends ESTestCase { + private static final String INDEX_NAME = ".ml-anomalies-foo"; + private static final String SCROLL_ID = "someScrollId"; + + private Client client; + private boolean wasScrollCleared; + + private TestIterator testIterator; + + @Before + public void setUpMocks() { + client = Mockito.mock(Client.class); + wasScrollCleared = false; + testIterator = new TestIterator(client, INDEX_NAME); + givenClearScrollRequest(); + } + + public void testQueryReturnsNoResults() { + new ScrollResponsesMocker().finishMock(); + + assertTrue(testIterator.hasNext()); + assertTrue(testIterator.next().isEmpty()); + assertFalse(testIterator.hasNext()); + assertTrue(wasScrollCleared); + } + + public void testCallingNextWhenHasNextIsFalseThrows() { + new ScrollResponsesMocker().addBatch("a", "b", "c").finishMock(); + testIterator.next(); + assertFalse(testIterator.hasNext()); + + ESTestCase.expectThrows(NoSuchElementException.class, () -> testIterator.next()); + } + + public void testQueryReturnsSingleBatch() { + new ScrollResponsesMocker().addBatch("a", "b", "c").finishMock(); + + assertTrue(testIterator.hasNext()); + Deque batch = testIterator.next(); + assertEquals(3, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("a", "b", "c"))); + assertFalse(testIterator.hasNext()); + assertTrue(wasScrollCleared); + } + + public void testQueryReturnsThreeBatches() { + new ScrollResponsesMocker() + .addBatch("a", "b", "c") + .addBatch("d", "e") + .addBatch("f") + .finishMock(); + + assertTrue(testIterator.hasNext()); + + Deque batch = testIterator.next(); + assertEquals(3, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("a", "b", "c"))); + + batch = testIterator.next(); + assertEquals(2, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("d", "e"))); + + batch = testIterator.next(); + assertEquals(1, batch.size()); + assertTrue(batch.containsAll(Arrays.asList("f"))); + + assertFalse(testIterator.hasNext()); + assertTrue(wasScrollCleared); + } + + private void givenClearScrollRequest() { + ClearScrollRequestBuilder requestBuilder = mock(ClearScrollRequestBuilder.class); + when(client.prepareClearScroll()).thenReturn(requestBuilder); + when(requestBuilder.setScrollIds(Arrays.asList(SCROLL_ID))).thenReturn(requestBuilder); + when(requestBuilder.get()).thenAnswer((invocation) -> { + wasScrollCleared = true; + return null; + }); + } + + private class ScrollResponsesMocker { + private List batches = new ArrayList<>(); + private long totalHits = 0; + private List nextRequestBuilders = new ArrayList<>(); + + ScrollResponsesMocker addBatch(String... hits) { + totalHits += hits.length; + batches.add(hits); + return this; + } + + void finishMock() { + if (batches.isEmpty()) { + givenInitialResponse(); + return; + } + givenInitialResponse(batches.get(0)); + for (int i = 1; i < batches.size(); ++i) { + givenNextResponse(batches.get(i)); + } + if (nextRequestBuilders.size() > 0) { + SearchScrollRequestBuilder first = nextRequestBuilders.get(0); + if (nextRequestBuilders.size() > 1) { + SearchScrollRequestBuilder[] rest = new SearchScrollRequestBuilder[batches.size() - 1]; + for (int i = 1; i < nextRequestBuilders.size(); ++i) { + rest[i - 1] = nextRequestBuilders.get(i); + } + when(client.prepareSearchScroll(SCROLL_ID)).thenReturn(first, rest); + } else { + when(client.prepareSearchScroll(SCROLL_ID)).thenReturn(first); + } + } + } + + private void givenInitialResponse(String... hits) { + SearchResponse searchResponse = createSearchResponseWithHits(hits); + SearchRequestBuilder requestBuilder = mock(SearchRequestBuilder.class); + when(client.prepareSearch(INDEX_NAME)).thenReturn(requestBuilder); + when(requestBuilder.setScroll("5m")).thenReturn(requestBuilder); + when(requestBuilder.setSize(10000)).thenReturn(requestBuilder); + when(requestBuilder.setTypes("String")).thenReturn(requestBuilder); + when(requestBuilder.setQuery(any(QueryBuilder.class))).thenReturn(requestBuilder); + when(requestBuilder.addSort(any(SortBuilder.class))).thenReturn(requestBuilder); + when(requestBuilder.get()).thenReturn(searchResponse); + } + + private void givenNextResponse(String... hits) { + SearchResponse searchResponse = createSearchResponseWithHits(hits); + SearchScrollRequestBuilder requestBuilder = mock(SearchScrollRequestBuilder.class); + when(requestBuilder.setScrollId(SCROLL_ID)).thenReturn(requestBuilder); + when(requestBuilder.setScroll("5m")).thenReturn(requestBuilder); + when(requestBuilder.get()).thenReturn(searchResponse); + nextRequestBuilders.add(requestBuilder); + } + + private SearchResponse createSearchResponseWithHits(String... hits) { + SearchHits searchHits = createHits(hits); + SearchResponse searchResponse = mock(SearchResponse.class); + when(searchResponse.getScrollId()).thenReturn(SCROLL_ID); + when(searchResponse.getHits()).thenReturn(searchHits); + return searchResponse; + } + + private SearchHits createHits(String... values) { + SearchHits searchHits = mock(SearchHits.class); + List hits = new ArrayList<>(); + for (String value : values) { + SearchHit hit = mock(SearchHit.class); + when(hit.getSourceAsString()).thenReturn(value); + hits.add(hit); + } + when(searchHits.getTotalHits()).thenReturn(totalHits); + when(searchHits.getHits()).thenReturn(hits.toArray(new SearchHit[hits.size()])); + return searchHits; + } + } + + private static class TestIterator extends BatchedDocumentsIterator { + TestIterator(Client client, String jobId) { + super(client, jobId); + } + + @Override + protected String getType() { + return "String"; + } + + @Override + protected String map(SearchHit hit) { + return hit.getSourceAsString(); + } + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java new file mode 100644 index 00000000000..a5a0c4edb1b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.test.ESTestCase; + +public class BucketsQueryBuilderTests extends ESTestCase { + + public void testDefaultBuild() throws Exception { + BucketsQueryBuilder.BucketsQuery query = new BucketsQueryBuilder().build(); + + assertEquals(0, query.getFrom()); + assertEquals(BucketsQueryBuilder.DEFAULT_SIZE, query.getSize()); + assertEquals(false, query.isIncludeInterim()); + assertEquals(false, query.isExpand()); + assertEquals(0.0, query.getAnomalyScoreFilter(), 0.0001); + assertEquals(0.0, query.getNormalizedProbability(), 0.0001); + assertNull(query.getStart()); + assertNull(query.getEnd()); + assertEquals("timestamp", query.getSortField()); + assertFalse(query.isSortDescending()); + } + + public void testAll() { + BucketsQueryBuilder.BucketsQuery query = new BucketsQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .start("1000") + .end("2000") + .partitionValue("foo") + .sortField("anomaly_score") + .sortDescending(true) + .build(); + + assertEquals(20, query.getFrom()); + assertEquals(40, query.getSize()); + assertEquals(true, query.isIncludeInterim()); + assertEquals(true, query.isExpand()); + assertEquals(50.0d, query.getAnomalyScoreFilter(), 0.00001); + assertEquals(70.0d, query.getNormalizedProbability(), 0.00001); + assertEquals("1000", query.getStart()); + assertEquals("2000", query.getEnd()); + assertEquals("foo", query.getPartitionValue()); + assertEquals("anomaly_score", query.getSortField()); + assertTrue(query.isSortDescending()); + } + + public void testEqualsHash() { + BucketsQueryBuilder query = new BucketsQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .start("1000") + .end("2000") + .partitionValue("foo"); + + BucketsQueryBuilder query2 = new BucketsQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .start("1000") + .end("2000") + .partitionValue("foo"); + + assertEquals(query.build(), query2.build()); + assertEquals(query.build().hashCode(), query2.build().hashCode()); + query2.clear(); + assertFalse(query.build().equals(query2.build())); + + query2.from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.0d) + .normalizedProbabilityThreshold(70.0d) + .start("1000") + .end("2000") + .partitionValue("foo"); + assertEquals(query.build(), query2.build()); + + query2.clear(); + query2.from(20) + .size(40) + .includeInterim(true) + .expand(true) + .anomalyScoreThreshold(50.1d) + .normalizedProbabilityThreshold(70.0d) + .start("1000") + .end("2000") + .partitionValue("foo"); + assertFalse(query.build().equals(query2.build())); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java new file mode 100644 index 00000000000..61e975aaa9b --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.ReservedFieldNames; +import org.elasticsearch.xpack.ml.job.results.Result; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + + +public class ElasticsearchMappingsTests extends ESTestCase { + private void parseJson(JsonParser parser, Set expected) throws IOException { + try { + JsonToken token = parser.nextToken(); + while (token != null && token != JsonToken.END_OBJECT) { + switch (token) { + case START_OBJECT: + parseJson(parser, expected); + break; + case FIELD_NAME: + String fieldName = parser.getCurrentName(); + expected.add(fieldName); + break; + default: + break; + } + token = parser.nextToken(); + } + } catch (JsonParseException e) { + fail("Cannot parse JSON: " + e); + } + } + + public void testReservedFields() + throws IOException, ClassNotFoundException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + Set overridden = new HashSet<>(); + + // These are not reserved because they're Elasticsearch keywords, not + // field names + overridden.add(ElasticsearchMappings.ANALYZER); + overridden.add(ElasticsearchMappings.COPY_TO); + overridden.add(ElasticsearchMappings.DYNAMIC); + overridden.add(ElasticsearchMappings.ENABLED); + overridden.add(ElasticsearchMappings.NESTED); + overridden.add(ElasticsearchMappings.PROPERTIES); + overridden.add(ElasticsearchMappings.TYPE); + overridden.add(ElasticsearchMappings.WHITESPACE); + + // These are not reserved because they're data types, not field names + overridden.add(Result.TYPE.getPreferredName()); + overridden.add(DataCounts.TYPE.getPreferredName()); + overridden.add(CategoryDefinition.TYPE.getPreferredName()); + overridden.add(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()); + overridden.add(ModelSnapshot.TYPE.getPreferredName()); + overridden.add(Quantiles.TYPE.getPreferredName()); + + Set expected = new HashSet<>(); + + // Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here. + + XContentBuilder builder = ElasticsearchMappings.resultsMapping(Collections.emptyList()); + BufferedInputStream inputStream = + new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + JsonParser parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.categoryDefinitionMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.dataCountsMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + builder = ElasticsearchMappings.modelSnapshotMapping(); + inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + parser = new JsonFactory().createParser(inputStream); + parseJson(parser, expected); + + expected.removeAll(overridden); + + if (ReservedFieldNames.RESERVED_FIELD_NAMES.size() != expected.size()) { + Set diff = new HashSet<>(ReservedFieldNames.RESERVED_FIELD_NAMES); + diff.removeAll(expected); + StringBuilder errorMessage = new StringBuilder("Fields in ReservedFieldNames but not in expected: ").append(diff); + + diff = new HashSet<>(expected); + diff.removeAll(ReservedFieldNames.RESERVED_FIELD_NAMES); + errorMessage.append("\nFields in expected but not in ReservedFieldNames: ").append(diff); + fail(errorMessage.toString()); + } + assertEquals(ReservedFieldNames.RESERVED_FIELD_NAMES.size(), expected.size()); + + for (String s : expected) { + // By comparing like this the failure messages say which string is missing + String reserved = ReservedFieldNames.RESERVED_FIELD_NAMES.contains(s) ? s : null; + assertEquals(s, reserved); + } + } + + @SuppressWarnings("unchecked") + public void testResultMapping() throws IOException { + + XContentBuilder builder = ElasticsearchMappings.resultsMapping( + Arrays.asList("instance", AnomalyRecord.ANOMALY_SCORE.getPreferredName())); + XContentParser parser = createParser(builder); + Map type = (Map) parser.map().get(Result.TYPE.getPreferredName()); + Map properties = (Map) type.get(ElasticsearchMappings.PROPERTIES); + + // check a keyword mapping for the 'instance' field was created + Map instanceMapping = (Map) properties.get("instance"); + assertNotNull(instanceMapping); + String dataType = (String)instanceMapping.get(ElasticsearchMappings.TYPE); + assertEquals(ElasticsearchMappings.KEYWORD, dataType); + + // check anomaly score wasn't overwritten + Map anomalyScoreMapping = (Map) properties.get(AnomalyRecord.ANOMALY_SCORE.getPreferredName()); + assertNotNull(anomalyScoreMapping); + dataType = (String)anomalyScoreMapping.get(ElasticsearchMappings.TYPE); + assertEquals(ElasticsearchMappings.DOUBLE, dataType); + } + +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java new file mode 100644 index 00000000000..97abbb2b787 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.results.Influencer; + +public class InfluencersQueryBuilderTests extends ESTestCase { + + public void testDefaultBuild() throws Exception { + InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder().build(); + + assertEquals(0, query.getFrom()); + assertEquals(InfluencersQueryBuilder.DEFAULT_SIZE, query.getSize()); + assertEquals(false, query.isIncludeInterim()); + assertEquals(0.0, query.getAnomalyScoreFilter(), 0.0001); + assertNull(query.getStart()); + assertNull(query.getEnd()); + assertEquals(Influencer.ANOMALY_SCORE.getPreferredName(), query.getSortField()); + assertFalse(query.isSortDescending()); + } + + public void testAll() { + InfluencersQueryBuilder.InfluencersQuery query = new InfluencersQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .anomalyScoreThreshold(50.0d) + .start("1000") + .end("2000") + .sortField("anomaly_score") + .sortDescending(true) + .build(); + + assertEquals(20, query.getFrom()); + assertEquals(40, query.getSize()); + assertEquals(true, query.isIncludeInterim()); + assertEquals(50.0d, query.getAnomalyScoreFilter(), 0.00001); + assertEquals("1000", query.getStart()); + assertEquals("2000", query.getEnd()); + assertEquals("anomaly_score", query.getSortField()); + assertTrue(query.isSortDescending()); + } + + public void testEqualsHash() { + InfluencersQueryBuilder query = new InfluencersQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .anomalyScoreThreshold(50.0d) + .start("1000") + .end("2000"); + + InfluencersQueryBuilder query2 = new InfluencersQueryBuilder() + .from(20) + .size(40) + .includeInterim(true) + .anomalyScoreThreshold(50.0d) + .start("1000") + .end("2000"); + + assertEquals(query.build(), query2.build()); + assertEquals(query.build().hashCode(), query2.build().hashCode()); + query2.clear(); + assertFalse(query.build().equals(query2.build())); + + query2.from(20) + .size(40) + .includeInterim(true) + .anomalyScoreThreshold(50.0d) + .start("1000") + .end("2000"); + assertEquals(query.build(), query2.build()); + + query2.clear(); + query2.from(20) + .size(40) + .includeInterim(true) + .anomalyScoreThreshold(50.1d) + .start("1000") + .end("2000"); + assertFalse(query.build().equals(query2.build())); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleterTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleterTests.java new file mode 100644 index 00000000000..47535636c24 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleterTests.java @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.delete.DeleteRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.elasticsearch.mock.orig.Mockito.times; +import static org.elasticsearch.mock.orig.Mockito.verify; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; + +public class JobDataDeleterTests extends ESTestCase { + + public void testDeleteResultsFromTime() { + + final long TOTAL_HIT_COUNT = 100L; + final int PER_SCROLL_SEARCH_HIT_COUNT = 20; + + SearchResponse response = createSearchResponseWithHits(TOTAL_HIT_COUNT, PER_SCROLL_SEARCH_HIT_COUNT); + BulkResponse bulkResponse = Mockito.mock(BulkResponse.class); + + Client client = new MockClientBuilder("myCluster") + .prepareSearchExecuteListener(AnomalyDetectorsIndex.jobResultsIndexName("foo"), response) + .prepareSearchScrollExecuteListener(response) + .prepareBulk(bulkResponse).build(); + + JobDataDeleter bulkDeleter = new JobDataDeleter(client, "foo"); + + // because of the mocking this runs in the current thread + bulkDeleter.deleteResultsFromTime(new Date().getTime(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + assertTrue(aBoolean); + } + + @Override + public void onFailure(Exception e) { + fail(e.toString()); + } + }); + + verify(client.prepareBulk(), times((int)TOTAL_HIT_COUNT)).add(any(DeleteRequestBuilder.class)); + + ActionListener bulkListener = new ActionListener() { + @Override + public void onResponse(BulkResponse bulkItemResponses) { + } + + @Override + public void onFailure(Exception e) { + fail(e.toString()); + } + }; + + when(client.prepareBulk().numberOfActions()).thenReturn(new Integer((int)TOTAL_HIT_COUNT)); + bulkDeleter.commit(bulkListener); + + verify(client.prepareBulk(), times(1)).execute(bulkListener); + } + + public void testDeleteModelSnapShot() { + String jobId = "foo"; + ModelSnapshot snapshot = new ModelSnapshot(jobId); + snapshot.setSnapshotDocCount(5); + snapshot.setSnapshotId("snap-1"); + + BulkResponse bulkResponse = Mockito.mock(BulkResponse.class); + Client client = new MockClientBuilder("myCluster").prepareBulk(bulkResponse).build(); + + JobDataDeleter bulkDeleter = new JobDataDeleter(client, jobId); + bulkDeleter.deleteModelSnapshot(snapshot); + verify(client, times(5)) + .prepareDelete(eq(AnomalyDetectorsIndex.jobStateIndexName()), eq(ModelState.TYPE.getPreferredName()), anyString()); + verify(client, times(1)) + .prepareDelete(eq(AnomalyDetectorsIndex.jobResultsIndexName(jobId)), eq(ModelSnapshot.TYPE.getPreferredName()), + eq("snap-1")); + } + + private SearchResponse createSearchResponseWithHits(long totalHitCount, int hitsPerSearchResult) { + SearchHits hits = mockSearchHits(totalHitCount, hitsPerSearchResult); + SearchResponse searchResponse = Mockito.mock(SearchResponse.class); + when(searchResponse.getHits()).thenReturn(hits); + when(searchResponse.getScrollId()).thenReturn("scroll1"); + return searchResponse; + } + + private SearchHits mockSearchHits(long totalHitCount, int hitsPerSearchResult) { + + List hitList = new ArrayList<>(); + for (int i=0; i<20; i++) { + SearchHit hit = new SearchHit(123, "mockSeachHit-" + i, + new Text("mockSearchHit"), Collections.emptyMap()); + hitList.add(hit); + } + + return new SearchHits(hitList.toArray(new SearchHit[0]), totalHitCount, 1); + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java new file mode 100644 index 00000000000..1fc5fb968fa --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java @@ -0,0 +1,1223 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.MultiSearchRequest; +import org.elasticsearch.action.search.MultiSearchResponse; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.action.DeleteJobAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.job.config.AnalysisLimits; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; +import org.elasticsearch.xpack.ml.notifications.AuditActivity; +import org.elasticsearch.xpack.ml.notifications.AuditMessage; +import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; +import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.elasticsearch.xpack.ml.job.results.PerPartitionMaxProbabilities; +import org.elasticsearch.xpack.ml.job.results.Result; +import org.elasticsearch.xpack.ml.notifications.Auditor; +import org.mockito.ArgumentCaptor; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.elasticsearch.xpack.ml.job.config.JobTests.buildJobBuilder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class JobProviderTests extends ESTestCase { + private static final String CLUSTER_NAME = "myCluster"; + private static final String JOB_ID = "foo"; + private static final String STATE_INDEX_NAME = ".ml-state"; + + public void testGetQuantiles_GivenNoQuantilesForJob() throws Exception { + GetResponse getResponse = createGetResponse(false, null); + + Client client = getMockedClient(getResponse); + JobProvider provider = createProvider(client); + + Quantiles[] holder = new Quantiles[1]; + provider.getQuantiles(JOB_ID, quantiles -> holder[0] = quantiles, RuntimeException::new); + Quantiles quantiles = holder[0]; + assertNull(quantiles); + } + + public void testGetQuantiles_GivenQuantilesHaveNonEmptyState() throws Exception { + Map source = new HashMap<>(); + source.put(Job.ID.getPreferredName(), "foo"); + source.put(Quantiles.TIMESTAMP.getPreferredName(), 0L); + source.put(Quantiles.QUANTILE_STATE.getPreferredName(), "state"); + GetResponse getResponse = createGetResponse(true, source); + + Client client = getMockedClient(getResponse); + JobProvider provider = createProvider(client); + + Quantiles[] holder = new Quantiles[1]; + provider.getQuantiles(JOB_ID, quantiles -> holder[0] = quantiles, RuntimeException::new); + Quantiles quantiles = holder[0]; + assertNotNull(quantiles); + assertEquals("state", quantiles.getQuantileState()); + } + + public void testGetQuantiles_GivenQuantilesHaveEmptyState() throws Exception { + Map source = new HashMap<>(); + source.put(Job.ID.getPreferredName(), "foo"); + source.put(Quantiles.TIMESTAMP.getPreferredName(), new Date(0L).getTime()); + source.put(Quantiles.QUANTILE_STATE.getPreferredName(), ""); + GetResponse getResponse = createGetResponse(true, source); + + Client client = getMockedClient(getResponse); + JobProvider provider = createProvider(client); + + Quantiles[] holder = new Quantiles[1]; + provider.getQuantiles(JOB_ID, quantiles -> holder[0] = quantiles, RuntimeException::new); + Quantiles quantiles = holder[0]; + assertNotNull(quantiles); + assertEquals("", quantiles.getQuantileState()); + } + + public void testMlResultsIndexSettings() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + JobProvider provider = createProvider(clientBuilder.build()); + Settings settings = provider.mlResultsIndexSettings().build(); + + assertEquals("1", settings.get("index.number_of_shards")); + assertEquals("0", settings.get("index.number_of_replicas")); + assertEquals("async", settings.get("index.translog.durability")); + assertEquals("true", settings.get("index.mapper.dynamic")); + assertEquals("all_field_values", settings.get("index.query.default_field")); + } + + public void testCreateJobResultsIndex() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + clientBuilder.createIndexRequest(AnomalyDetectorsIndex.jobResultsIndexName("foo"), captor); + + Job.Builder job = buildJobBuilder("foo"); + JobProvider provider = createProvider(clientBuilder.build()); + + provider.createJobResultIndex(job.build(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + CreateIndexRequest request = captor.getValue(); + assertNotNull(request); + assertEquals(provider.mlResultsIndexSettings().build(), request.settings()); + assertTrue(request.mappings().containsKey(Result.TYPE.getPreferredName())); + assertTrue(request.mappings().containsKey(CategoryDefinition.TYPE.getPreferredName())); + assertTrue(request.mappings().containsKey(DataCounts.TYPE.getPreferredName())); + assertTrue(request.mappings().containsKey(ModelSnapshot.TYPE.getPreferredName())); + assertEquals(4, request.mappings().size()); + + clientBuilder.verifyIndexCreated(AnomalyDetectorsIndex.jobResultsIndexName("foo")); + } + + @Override + public void onFailure(Exception e) { + fail(e.toString()); + } + }); + } + + public void testCreateJobRelatedIndicies_createsAliasIfIndexNameIsSet() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + clientBuilder.createIndexRequest(AnomalyDetectorsIndex.jobResultsIndexName("foo"), captor); + clientBuilder.prepareAlias(AnomalyDetectorsIndex.jobResultsIndexName("bar"), AnomalyDetectorsIndex.jobResultsIndexName("foo")); + + Job.Builder job = buildJobBuilder("foo"); + job.setIndexName("bar"); + Client client = clientBuilder.build(); + JobProvider provider = createProvider(client); + + provider.createJobResultIndex(job.build(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + verify(client.admin().indices(), times(1)).prepareAliases(); + } + + @Override + public void onFailure(Exception e) { + fail(e.toString()); + } + }); + } + + public void testCreateJobRelatedIndicies_doesntCreateAliasIfIndexNameIsSameAsJobId() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + clientBuilder.createIndexRequest(AnomalyDetectorsIndex.jobResultsIndexName("foo"), captor); + + Job.Builder job = buildJobBuilder("foo"); + job.setIndexName("foo"); + Client client = clientBuilder.build(); + JobProvider provider = createProvider(client); + + provider.createJobResultIndex(job.build(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + verify(client.admin().indices(), never()).prepareAliases(); + } + + @Override + public void onFailure(Exception e) { + fail(e.toString()); + } + }); + } + + public void testMlAuditIndexSettings() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + JobProvider provider = createProvider(clientBuilder.build()); + Settings settings = provider.mlResultsIndexSettings().build(); + + assertEquals("1", settings.get("index.number_of_shards")); + assertEquals("0", settings.get("index.number_of_replicas")); + assertEquals("async", settings.get("index.translog.durability")); + assertEquals("true", settings.get("index.mapper.dynamic")); + } + + public void testCreateAuditMessageIndex() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + clientBuilder.createIndexRequest(Auditor.NOTIFICATIONS_INDEX, captor); + + JobProvider provider = createProvider(clientBuilder.build()); + + provider.createNotificationMessageIndex((result, error) -> { + assertTrue(result); + CreateIndexRequest request = captor.getValue(); + assertNotNull(request); + assertEquals(provider.mlNotificationIndexSettings().build(), request.settings()); + assertTrue(request.mappings().containsKey(AuditMessage.TYPE.getPreferredName())); + assertTrue(request.mappings().containsKey(AuditActivity.TYPE.getPreferredName())); + assertEquals(2, request.mappings().size()); + + clientBuilder.verifyIndexCreated(Auditor.NOTIFICATIONS_INDEX); + }); + } + + public void testCreateMetaIndex() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + clientBuilder.createIndexRequest(JobProvider.ML_META_INDEX, captor); + + JobProvider provider = createProvider(clientBuilder.build()); + + provider.createMetaIndex((result, error) -> { + assertTrue(result); + CreateIndexRequest request = captor.getValue(); + assertNotNull(request); + assertEquals(provider.mlNotificationIndexSettings().build(), request.settings()); + assertEquals(0, request.mappings().size()); + + clientBuilder.verifyIndexCreated(JobProvider.ML_META_INDEX); + }); + } + + public void testMlStateIndexSettings() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + JobProvider provider = createProvider(clientBuilder.build()); + Settings settings = provider.mlResultsIndexSettings().build(); + + assertEquals("1", settings.get("index.number_of_shards")); + assertEquals("0", settings.get("index.number_of_replicas")); + assertEquals("async", settings.get("index.translog.durability")); + } + + public void testCreateJobStateIndex() { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + clientBuilder.createIndexRequest(AnomalyDetectorsIndex.jobStateIndexName(), captor); + + Job.Builder job = buildJobBuilder("foo"); + JobProvider provider = createProvider(clientBuilder.build()); + + provider.createJobStateIndex((result, error) -> { + assertTrue(result); + CreateIndexRequest request = captor.getValue(); + assertNotNull(request); + assertEquals(provider.mlStateIndexSettings().build(), request.settings()); + assertTrue(request.mappings().containsKey(CategorizerState.TYPE)); + assertTrue(request.mappings().containsKey(Quantiles.TYPE.getPreferredName())); + assertTrue(request.mappings().containsKey(ModelState.TYPE.getPreferredName())); + assertEquals(3, request.mappings().size()); + }); + } + + public void testCreateJob() throws InterruptedException, ExecutionException { + Job.Builder job = buildJobBuilder("marscapone"); + job.setDescription("This is a very cheesy job"); + AnalysisLimits limits = new AnalysisLimits(9878695309134L, null); + job.setAnalysisLimits(limits); + + ArgumentCaptor captor = ArgumentCaptor.forClass(CreateIndexRequest.class); + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME) + .createIndexRequest(AnomalyDetectorsIndex.jobResultsIndexName(job.getId()), captor); + + Client client = clientBuilder.build(); + JobProvider provider = createProvider(client); + AtomicReference resultHolder = new AtomicReference<>(); + provider.createJobResultIndex(job.build(), new ActionListener() { + @Override + public void onResponse(Boolean aBoolean) { + resultHolder.set(aBoolean); + } + + @Override + public void onFailure(Exception e) { + + } + }); + assertNotNull(resultHolder.get()); + assertTrue(resultHolder.get()); + } + + public void testDeleteJob() throws InterruptedException, ExecutionException, IOException { + @SuppressWarnings("unchecked") + ActionListener actionListener = mock(ActionListener.class); + String jobId = "ThisIsMyJob"; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse(); + Client client = clientBuilder.build(); + JobProvider provider = createProvider(client); + clientBuilder.resetIndices(); + clientBuilder.addIndicesExistsResponse(AnomalyDetectorsIndex.jobResultsIndexName(jobId), true) + .addIndicesDeleteResponse(AnomalyDetectorsIndex.jobResultsIndexName(jobId), true, + false, actionListener); + clientBuilder.build(); + + provider.deleteJobRelatedIndices(jobId, actionListener); + + ArgumentCaptor responseCaptor = ArgumentCaptor.forClass(DeleteJobAction.Response.class); + verify(actionListener).onResponse(responseCaptor.capture()); + assertTrue(responseCaptor.getValue().isAcknowledged()); + } + + public void testDeleteJob_InvalidIndex() throws InterruptedException, ExecutionException, IOException { + @SuppressWarnings("unchecked") + ActionListener actionListener = mock(ActionListener.class); + String jobId = "ThisIsMyJob"; + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse(); + Client client = clientBuilder.build(); + JobProvider provider = createProvider(client); + clientBuilder.resetIndices(); + clientBuilder.addIndicesExistsResponse(AnomalyDetectorsIndex.jobResultsIndexName(jobId), true) + .addIndicesDeleteResponse(AnomalyDetectorsIndex.jobResultsIndexName(jobId), true, + true, actionListener); + clientBuilder.build(); + + provider.deleteJobRelatedIndices(jobId, actionListener); + + ArgumentCaptor exceptionCaptor = ArgumentCaptor.forClass(Exception.class); + verify(actionListener).onFailure(exceptionCaptor.capture()); + assertThat(exceptionCaptor.getValue(), instanceOf(InterruptedException.class)); + } + + public void testBuckets_OneBucketNoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("job_id", "foo"); + map.put("timestamp", now.getTime()); + map.put("bucket_span", 22); + source.add(map); + + QueryBuilder[] queryBuilderHolder = new QueryBuilder[1]; + SearchResponse response = createSearchResponse(true, source); + int from = 0; + int size = 10; + Client client = getMockedClient(queryBuilder -> {queryBuilderHolder[0] = queryBuilder;}, response); + JobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(0.0) + .normalizedProbabilityThreshold(1.0); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.buckets(jobId, bq.build(), r -> holder[0] = r, e -> {throw new RuntimeException(e);}); + QueryPage buckets = holder[0]; + assertEquals(1L, buckets.count()); + QueryBuilder query = queryBuilderHolder[0]; + String queryString = query.toString(); + assertTrue( + queryString.matches("(?s).*max_normalized_probability[^}]*from. : 1\\.0.*must_not[^}]*term[^}]*is_interim.*value. : .true" + + ".*")); + } + + public void testBuckets_OneBucketInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("job_id", "foo"); + map.put("timestamp", now.getTime()); + map.put("bucket_span", 22); + source.add(map); + + QueryBuilder[] queryBuilderHolder = new QueryBuilder[1]; + SearchResponse response = createSearchResponse(true, source); + int from = 99; + int size = 17; + + Client client = getMockedClient(queryBuilder -> queryBuilderHolder[0] = queryBuilder, response); + JobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(5.1) + .normalizedProbabilityThreshold(10.9).includeInterim(true); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.buckets(jobId, bq.build(), r -> holder[0] = r, e -> {throw new RuntimeException(e);}); + QueryPage buckets = holder[0]; + assertEquals(1L, buckets.count()); + QueryBuilder query = queryBuilderHolder[0]; + String queryString = query.toString(); + assertTrue(queryString.matches("(?s).*max_normalized_probability[^}]*from. : 10\\.9.*")); + assertTrue(queryString.matches("(?s).*anomaly_score[^}]*from. : 5\\.1.*")); + assertFalse(queryString.matches("(?s).*is_interim.*")); + } + + public void testBuckets_UsingBuilder() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("job_id", "foo"); + map.put("timestamp", now.getTime()); + map.put("bucket_span", 22); + source.add(map); + + QueryBuilder[] queryBuilderHolder = new QueryBuilder[1]; + SearchResponse response = createSearchResponse(true, source); + int from = 99; + int size = 17; + + Client client = getMockedClient(queryBuilder -> queryBuilderHolder[0] = queryBuilder, response); + JobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder(); + bq.from(from); + bq.size(size); + bq.anomalyScoreThreshold(5.1); + bq.normalizedProbabilityThreshold(10.9); + bq.includeInterim(true); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.buckets(jobId, bq.build(), r -> holder[0] = r, e -> {throw new RuntimeException(e);}); + QueryPage buckets = holder[0]; + assertEquals(1L, buckets.count()); + QueryBuilder query = queryBuilderHolder[0]; + String queryString = query.toString(); + assertTrue(queryString.matches("(?s).*max_normalized_probability[^}]*from. : 10\\.9.*")); + assertTrue(queryString.matches("(?s).*anomaly_score[^}]*from. : 5\\.1.*")); + assertFalse(queryString.matches("(?s).*is_interim.*")); + } + + public void testBucket_NoBucketNoExpandNoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Long timestamp = 98765432123456789L; + List> source = new ArrayList<>(); + + SearchResponse response = createSearchResponse(false, source); + + Client client = getMockedClient(queryBuilder -> {}, response); + JobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder(); + bq.timestamp(Long.toString(timestamp)); + Exception[] holder = new Exception[1]; + provider.buckets(jobId, bq.build(), q -> {}, e -> {holder[0] = e;}); + assertEquals(ResourceNotFoundException.class, holder[0].getClass()); + } + + public void testBucket_OneBucketNoExpandNoInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("job_id", "foo"); + map.put("timestamp", now.getTime()); + map.put("bucket_span", 22); + source.add(map); + + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(queryBuilder -> {}, response); + JobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder(); + bq.timestamp(Long.toString(now.getTime())); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] bucketHolder = new QueryPage[1]; + provider.buckets(jobId, bq.build(), q -> {bucketHolder[0] = q;}, e -> {}); + assertThat(bucketHolder[0].count(), equalTo(1L)); + Bucket b = bucketHolder[0].results().get(0); + assertEquals(now, b.getTimestamp()); + } + + public void testBucket_OneBucketNoExpandInterim() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("job_id", "foo"); + map.put("timestamp", now.getTime()); + map.put("bucket_span", 22); + map.put("is_interim", true); + source.add(map); + + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(queryBuilder -> {}, response); + JobProvider provider = createProvider(client); + + BucketsQueryBuilder bq = new BucketsQueryBuilder(); + bq.timestamp(Long.toString(now.getTime())); + + Exception[] holder = new Exception[1]; + provider.buckets(jobId, bq.build(), q -> {}, e -> {holder[0] = e;}); + assertEquals(ResourceNotFoundException.class, holder[0].getClass()); + } + + public void testRecords() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("typical", 22.4); + recordMap1.put("actual", 33.3); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("function", "irritable"); + recordMap1.put("bucket_span", 22); + recordMap1.put("sequence_num", 1); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("typical", 1122.4); + recordMap2.put("actual", 933.3); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("function", "irrascible"); + recordMap2.put("bucket_span", 22); + recordMap2.put("sequence_num", 2); + source.add(recordMap1); + source.add(recordMap2); + + int from = 14; + int size = 2; + String sortfield = "minefield"; + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(qb -> {}, response); + JobProvider provider = createProvider(client); + + RecordsQueryBuilder rqb = new RecordsQueryBuilder().from(from).size(size).epochStart(String.valueOf(now.getTime())) + .epochEnd(String.valueOf(now.getTime())).includeInterim(true).sortField(sortfield).anomalyScoreThreshold(11.1) + .normalizedProbability(2.2); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.records(jobId, rqb.build(), page -> holder[0] = page, RuntimeException::new); + QueryPage recordPage = holder[0]; + assertEquals(2L, recordPage.count()); + List records = recordPage.results(); + assertEquals(22.4, records.get(0).getTypical().get(0), 0.000001); + assertEquals(33.3, records.get(0).getActual().get(0), 0.000001); + assertEquals("irritable", records.get(0).getFunction()); + assertEquals(1122.4, records.get(1).getTypical().get(0), 0.000001); + assertEquals(933.3, records.get(1).getActual().get(0), 0.000001); + assertEquals("irrascible", records.get(1).getFunction()); + } + + public void testRecords_UsingBuilder() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("typical", 22.4); + recordMap1.put("actual", 33.3); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("function", "irritable"); + recordMap1.put("bucket_span", 22); + recordMap1.put("sequence_num", 1); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("typical", 1122.4); + recordMap2.put("actual", 933.3); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("function", "irrascible"); + recordMap2.put("bucket_span", 22); + recordMap2.put("sequence_num", 2); + source.add(recordMap1); + source.add(recordMap2); + + int from = 14; + int size = 2; + String sortfield = "minefield"; + SearchResponse response = createSearchResponse(true, source); + + Client client = getMockedClient(qb -> {}, response); + JobProvider provider = createProvider(client); + + RecordsQueryBuilder rqb = new RecordsQueryBuilder(); + rqb.from(from); + rqb.size(size); + rqb.epochStart(String.valueOf(now.getTime())); + rqb.epochEnd(String.valueOf(now.getTime())); + rqb.includeInterim(true); + rqb.sortField(sortfield); + rqb.anomalyScoreThreshold(11.1); + rqb.normalizedProbability(2.2); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.records(jobId, rqb.build(), page -> holder[0] = page, RuntimeException::new); + QueryPage recordPage = holder[0]; + assertEquals(2L, recordPage.count()); + List records = recordPage.results(); + assertEquals(22.4, records.get(0).getTypical().get(0), 0.000001); + assertEquals(33.3, records.get(0).getActual().get(0), 0.000001); + assertEquals("irritable", records.get(0).getFunction()); + assertEquals(1122.4, records.get(1).getTypical().get(0), 0.000001); + assertEquals(933.3, records.get(1).getActual().get(0), 0.000001); + assertEquals("irrascible", records.get(1).getFunction()); + } + + public void testBucketRecords() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + Bucket bucket = mock(Bucket.class); + when(bucket.getTimestamp()).thenReturn(now); + + List> source = new ArrayList<>(); + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("typical", 22.4); + recordMap1.put("actual", 33.3); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("function", "irritable"); + recordMap1.put("bucket_span", 22); + recordMap1.put("sequence_num", 1); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("typical", 1122.4); + recordMap2.put("actual", 933.3); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("function", "irrascible"); + recordMap2.put("bucket_span", 22); + recordMap2.put("sequence_num", 2); + source.add(recordMap1); + source.add(recordMap2); + + int from = 14; + int size = 2; + String sortfield = "minefield"; + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(qb -> {}, response); + JobProvider provider = createProvider(client); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.bucketRecords(jobId, bucket, from, size, true, sortfield, true, "", page -> holder[0] = page, RuntimeException::new); + QueryPage recordPage = holder[0]; + assertEquals(2L, recordPage.count()); + List records = recordPage.results(); + + assertEquals(22.4, records.get(0).getTypical().get(0), 0.000001); + assertEquals(33.3, records.get(0).getActual().get(0), 0.000001); + assertEquals("irritable", records.get(0).getFunction()); + assertEquals(1122.4, records.get(1).getTypical().get(0), 0.000001); + assertEquals(933.3, records.get(1).getActual().get(0), 0.000001); + assertEquals("irrascible", records.get(1).getFunction()); + } + + public void testexpandBucket() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + Bucket bucket = new Bucket("foo", now, 22); + + List> source = new ArrayList<>(); + for (int i = 0; i < 400; i++) { + Map recordMap = new HashMap<>(); + recordMap.put("job_id", "foo"); + recordMap.put("typical", 22.4 + i); + recordMap.put("actual", 33.3 + i); + recordMap.put("timestamp", now.getTime()); + recordMap.put("function", "irritable"); + recordMap.put("bucket_span", 22); + recordMap.put("sequence_num", i + 1); + source.add(recordMap); + } + + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(qb -> {}, response); + JobProvider provider = createProvider(client); + + Integer[] holder = new Integer[1]; + provider.expandBucket(jobId, false, bucket, null, 0, records -> holder[0] = records, RuntimeException::new); + int records = holder[0]; + assertEquals(400L, records); + } + + public void testexpandBucket_WithManyRecords() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + Date now = new Date(); + Bucket bucket = new Bucket("foo", now, 22); + + List> source = new ArrayList<>(); + for (int i = 0; i < 600; i++) { + Map recordMap = new HashMap<>(); + recordMap.put("job_id", "foo"); + recordMap.put("typical", 22.4 + i); + recordMap.put("actual", 33.3 + i); + recordMap.put("timestamp", now.getTime()); + recordMap.put("function", "irritable"); + recordMap.put("bucket_span", 22); + recordMap.put("sequence_num", i + 1); + source.add(recordMap); + } + + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(qb -> {}, response); + JobProvider provider = createProvider(client); + + Integer[] holder = new Integer[1]; + provider.expandBucket(jobId, false, bucket, null, 0, records -> holder[0] = records, RuntimeException::new); + int records = holder[0]; + + // This is not realistic, but is an artifact of the fact that the mock + // query returns all the records, not a subset + assertEquals(1200L, records); + } + + public void testCategoryDefinitions() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + String terms = "the terms and conditions are not valid here"; + List> source = new ArrayList<>(); + + Map map = new HashMap<>(); + map.put("job_id", "foo"); + map.put("category_id", String.valueOf(map.hashCode())); + map.put("terms", terms); + + source.add(map); + + SearchResponse response = createSearchResponse(true, source); + int from = 0; + int size = 10; + Client client = getMockedClient(q -> {}, response); + + JobProvider provider = createProvider(client); + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.categoryDefinitions(jobId, null, from, size, r -> {holder[0] = r;}, + e -> {throw new RuntimeException(e);}); + QueryPage categoryDefinitions = holder[0]; + assertEquals(1L, categoryDefinitions.count()); + assertEquals(terms, categoryDefinitions.results().get(0).getTerms()); + } + + public void testCategoryDefinition() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentification"; + String terms = "the terms and conditions are not valid here"; + + Map source = new HashMap<>(); + String categoryId = String.valueOf(source.hashCode()); + source.put("job_id", "foo"); + source.put("category_id", categoryId); + source.put("terms", terms); + + SearchResponse response = createSearchResponse(true, Collections.singletonList(source)); + Client client = getMockedClient(q -> {}, response); + JobProvider provider = createProvider(client); + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.categoryDefinitions(jobId, categoryId, null, null, + r -> {holder[0] = r;}, e -> {throw new RuntimeException(e);}); + QueryPage categoryDefinitions = holder[0]; + assertEquals(1L, categoryDefinitions.count()); + assertEquals(terms, categoryDefinitions.results().get(0).getTerms()); + } + + public void testInfluencers_NoInterim() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("probability", 0.555); + recordMap1.put("influencer_field_name", "Builder"); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("influencer_field_value", "Bob"); + recordMap1.put("initial_anomaly_score", 22.2); + recordMap1.put("anomaly_score", 22.6); + recordMap1.put("bucket_span", 123); + recordMap1.put("sequence_num", 1); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("probability", 0.99); + recordMap2.put("influencer_field_name", "Builder"); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("influencer_field_value", "James"); + recordMap2.put("initial_anomaly_score", 5.0); + recordMap2.put("anomaly_score", 5.0); + recordMap2.put("bucket_span", 123); + recordMap2.put("sequence_num", 2); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + QueryBuilder[] qbHolder = new QueryBuilder[1]; + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(q -> qbHolder[0] = q, response); + JobProvider provider = createProvider(client); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + InfluencersQuery query = new InfluencersQueryBuilder().from(from).size(size).includeInterim(false).build(); + provider.influencers(jobId, query, page -> holder[0] = page, RuntimeException::new); + QueryPage page = holder[0]; + assertEquals(2L, page.count()); + + String queryString = qbHolder[0].toString(); + assertTrue(queryString.matches("(?s).*must_not[^}]*term[^}]*is_interim.*value. : .true.*")); + + List records = page.results(); + assertEquals("foo", records.get(0).getJobId()); + assertEquals("Bob", records.get(0).getInfluencerFieldValue()); + assertEquals("Builder", records.get(0).getInfluencerFieldName()); + assertEquals(now, records.get(0).getTimestamp()); + assertEquals(0.555, records.get(0).getProbability(), 0.00001); + assertEquals(22.6, records.get(0).getAnomalyScore(), 0.00001); + assertEquals(22.2, records.get(0).getInitialAnomalyScore(), 0.00001); + + assertEquals("James", records.get(1).getInfluencerFieldValue()); + assertEquals("Builder", records.get(1).getInfluencerFieldName()); + assertEquals(now, records.get(1).getTimestamp()); + assertEquals(0.99, records.get(1).getProbability(), 0.00001); + assertEquals(5.0, records.get(1).getAnomalyScore(), 0.00001); + assertEquals(5.0, records.get(1).getInitialAnomalyScore(), 0.00001); + } + + public void testInfluencers_WithInterim() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("probability", 0.555); + recordMap1.put("influencer_field_name", "Builder"); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("influencer_field_value", "Bob"); + recordMap1.put("initial_anomaly_score", 22.2); + recordMap1.put("anomaly_score", 22.6); + recordMap1.put("bucket_span", 123); + recordMap1.put("sequence_num", 1); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("probability", 0.99); + recordMap2.put("influencer_field_name", "Builder"); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("influencer_field_value", "James"); + recordMap2.put("initial_anomaly_score", 5.0); + recordMap2.put("anomaly_score", 5.0); + recordMap2.put("bucket_span", 123); + recordMap2.put("sequence_num", 2); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + QueryBuilder[] qbHolder = new QueryBuilder[1]; + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(q -> qbHolder[0] = q, response); + JobProvider provider = createProvider(client); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + InfluencersQuery query = new InfluencersQueryBuilder().from(from).size(size).start("0").end("0").sortField("sort") + .sortDescending(true).anomalyScoreThreshold(0.0).includeInterim(true).build(); + provider.influencers(jobId, query, page -> holder[0] = page, RuntimeException::new); + QueryPage page = holder[0]; + assertEquals(2L, page.count()); + + String queryString = qbHolder[0].toString(); + assertFalse(queryString.matches("(?s).*isInterim.*")); + + List records = page.results(); + assertEquals("Bob", records.get(0).getInfluencerFieldValue()); + assertEquals("Builder", records.get(0).getInfluencerFieldName()); + assertEquals(now, records.get(0).getTimestamp()); + assertEquals(0.555, records.get(0).getProbability(), 0.00001); + assertEquals(22.6, records.get(0).getAnomalyScore(), 0.00001); + assertEquals(22.2, records.get(0).getInitialAnomalyScore(), 0.00001); + + assertEquals("James", records.get(1).getInfluencerFieldValue()); + assertEquals("Builder", records.get(1).getInfluencerFieldName()); + assertEquals(now, records.get(1).getTimestamp()); + assertEquals(0.99, records.get(1).getProbability(), 0.00001); + assertEquals(5.0, records.get(1).getAnomalyScore(), 0.00001); + assertEquals(5.0, records.get(1).getInitialAnomalyScore(), 0.00001); + } + + public void testModelSnapshots() throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("description", "snapshot1"); + recordMap1.put("restore_priority", 1); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("snapshot_doc_count", 5); + recordMap1.put("latest_record_time_stamp", now.getTime()); + recordMap1.put("latest_result_time_stamp", now.getTime()); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("description", "snapshot2"); + recordMap2.put("restore_priority", 999); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("snapshot_doc_count", 6); + recordMap2.put("latest_record_time_stamp", now.getTime()); + recordMap2.put("latest_result_time_stamp", now.getTime()); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(qb -> {}, response); + JobProvider provider = createProvider(client); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] holder = new QueryPage[1]; + provider.modelSnapshots(jobId, from, size, r -> holder[0] = r, RuntimeException::new); + QueryPage page = holder[0]; + assertEquals(2L, page.count()); + List snapshots = page.results(); + + assertEquals("foo", snapshots.get(0).getJobId()); + assertEquals(now, snapshots.get(0).getTimestamp()); + assertEquals(now, snapshots.get(0).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(0).getLatestResultTimeStamp()); + assertEquals("snapshot1", snapshots.get(0).getDescription()); + assertEquals(1L, snapshots.get(0).getRestorePriority()); + assertEquals(5, snapshots.get(0).getSnapshotDocCount()); + + assertEquals(now, snapshots.get(1).getTimestamp()); + assertEquals(now, snapshots.get(1).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(1).getLatestResultTimeStamp()); + assertEquals("snapshot2", snapshots.get(1).getDescription()); + assertEquals(999L, snapshots.get(1).getRestorePriority()); + assertEquals(6, snapshots.get(1).getSnapshotDocCount()); + } + + public void testModelSnapshots_WithDescription() + throws InterruptedException, ExecutionException, IOException { + String jobId = "TestJobIdentificationForInfluencers"; + Date now = new Date(); + List> source = new ArrayList<>(); + + Map recordMap1 = new HashMap<>(); + recordMap1.put("job_id", "foo"); + recordMap1.put("description", "snapshot1"); + recordMap1.put("restore_priority", 1); + recordMap1.put("timestamp", now.getTime()); + recordMap1.put("snapshot_doc_count", 5); + recordMap1.put("latest_record_time_stamp", now.getTime()); + recordMap1.put("latest_result_time_stamp", now.getTime()); + Map recordMap2 = new HashMap<>(); + recordMap2.put("job_id", "foo"); + recordMap2.put("description", "snapshot2"); + recordMap2.put("restore_priority", 999); + recordMap2.put("timestamp", now.getTime()); + recordMap2.put("snapshot_doc_count", 6); + recordMap2.put("latest_record_time_stamp", now.getTime()); + recordMap2.put("latest_result_time_stamp", now.getTime()); + source.add(recordMap1); + source.add(recordMap2); + + int from = 4; + int size = 3; + QueryBuilder[] qbHolder = new QueryBuilder[1]; + SearchResponse response = createSearchResponse(true, source); + Client client = getMockedClient(qb -> qbHolder[0] = qb, response); + JobProvider provider = createProvider(client); + + @SuppressWarnings({"unchecked", "rawtypes"}) + QueryPage[] hodor = new QueryPage[1]; + provider.modelSnapshots(jobId, from, size, null, null, "sortfield", true, "snappyId", "description1", + p -> hodor[0] = p, RuntimeException::new); + QueryPage page = hodor[0]; + assertEquals(2L, page.count()); + List snapshots = page.results(); + + assertEquals(now, snapshots.get(0).getTimestamp()); + assertEquals(now, snapshots.get(0).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(0).getLatestResultTimeStamp()); + assertEquals("snapshot1", snapshots.get(0).getDescription()); + assertEquals(1L, snapshots.get(0).getRestorePriority()); + assertEquals(5, snapshots.get(0).getSnapshotDocCount()); + + assertEquals(now, snapshots.get(1).getTimestamp()); + assertEquals(now, snapshots.get(1).getLatestRecordTimeStamp()); + assertEquals(now, snapshots.get(1).getLatestResultTimeStamp()); + assertEquals("snapshot2", snapshots.get(1).getDescription()); + assertEquals(999L, snapshots.get(1).getRestorePriority()); + assertEquals(6, snapshots.get(1).getSnapshotDocCount()); + + String queryString = qbHolder[0].toString(); + assertTrue(queryString.matches("(?s).*snapshot_id.*value. : .snappyId.*description.*value. : .description1.*")); + } + + public void testMergePartitionScoresIntoBucket() throws InterruptedException, ExecutionException { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + + JobProvider provider = createProvider(clientBuilder.build()); + + List partitionMaxProbs = new ArrayList<>(); + + List records = new ArrayList<>(); + records.add(createAnomalyRecord("partitionValue1", new Date(2), 1.0)); + records.add(createAnomalyRecord("partitionValue2", new Date(2), 4.0)); + partitionMaxProbs.add(new PerPartitionMaxProbabilities(records)); + + records.clear(); + records.add(createAnomalyRecord("partitionValue1", new Date(3), 2.0)); + records.add(createAnomalyRecord("partitionValue2", new Date(3), 1.0)); + partitionMaxProbs.add(new PerPartitionMaxProbabilities(records)); + + records.clear(); + records.add(createAnomalyRecord("partitionValue1", new Date(5), 3.0)); + records.add(createAnomalyRecord("partitionValue2", new Date(5), 2.0)); + partitionMaxProbs.add(new PerPartitionMaxProbabilities(records)); + + List buckets = new ArrayList<>(); + buckets.add(createBucketAtEpochTime(1)); + buckets.add(createBucketAtEpochTime(2)); + buckets.add(createBucketAtEpochTime(3)); + buckets.add(createBucketAtEpochTime(4)); + buckets.add(createBucketAtEpochTime(5)); + buckets.add(createBucketAtEpochTime(6)); + + provider.mergePartitionScoresIntoBucket(partitionMaxProbs, buckets, "partitionValue1"); + assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); + assertEquals(1.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); + assertEquals(2.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); + assertEquals(3.0, buckets.get(4).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(5).getMaxNormalizedProbability(), 0.001); + + provider.mergePartitionScoresIntoBucket(partitionMaxProbs, buckets, "partitionValue2"); + assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); + assertEquals(4.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); + assertEquals(1.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); + assertEquals(2.0, buckets.get(4).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(5).getMaxNormalizedProbability(), 0.001); + } + + private AnomalyRecord createAnomalyRecord(String partitionFieldValue, Date timestamp, double normalizedProbability) { + AnomalyRecord record = new AnomalyRecord("foo", timestamp, 600, 42); + record.setPartitionFieldValue(partitionFieldValue); + record.setNormalizedProbability(normalizedProbability); + return record; + } + + public void testMergePartitionScoresIntoBucket_WithEmptyScoresList() throws InterruptedException, ExecutionException { + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); + + JobProvider provider = createProvider(clientBuilder.build()); + + List scores = new ArrayList<>(); + + List buckets = new ArrayList<>(); + buckets.add(createBucketAtEpochTime(1)); + buckets.add(createBucketAtEpochTime(2)); + buckets.add(createBucketAtEpochTime(3)); + buckets.add(createBucketAtEpochTime(4)); + + provider.mergePartitionScoresIntoBucket(scores, buckets, "partitionValue"); + assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); + assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); + } + + public void testRestoreStateToStream() throws Exception { + Map categorizerState = new HashMap<>(); + categorizerState.put("catName", "catVal"); + GetResponse categorizerStateGetResponse1 = createGetResponse(true, categorizerState); + GetResponse categorizerStateGetResponse2 = createGetResponse(false, null); + Map modelState = new HashMap<>(); + modelState.put("modName", "modVal1"); + GetResponse modelStateGetResponse1 = createGetResponse(true, modelState); + modelState.put("modName", "modVal2"); + GetResponse modelStateGetResponse2 = createGetResponse(true, modelState); + + MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME).addClusterStatusYellowResponse() + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), CategorizerState.TYPE, JOB_ID + "_1", categorizerStateGetResponse1) + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), CategorizerState.TYPE, JOB_ID + "_2", categorizerStateGetResponse2) + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ModelState.TYPE.getPreferredName(), "123_1", modelStateGetResponse1) + .prepareGet(AnomalyDetectorsIndex.jobStateIndexName(), ModelState.TYPE.getPreferredName(), "123_2", modelStateGetResponse2); + + JobProvider provider = createProvider(clientBuilder.build()); + + ModelSnapshot modelSnapshot = new ModelSnapshot(randomAsciiOfLengthBetween(1, 20)); + modelSnapshot.setSnapshotId("123"); + modelSnapshot.setSnapshotDocCount(2); + + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + + provider.restoreStateToStream(JOB_ID, modelSnapshot, stream); + + String[] restoreData = stream.toString(StandardCharsets.UTF_8.name()).split("\0"); + assertEquals(3, restoreData.length); + assertEquals("{\"catName\":\"catVal\"}", restoreData[0]); + assertEquals("{\"modName\":\"modVal1\"}", restoreData[1]); + assertEquals("{\"modName\":\"modVal2\"}", restoreData[2]); + } + + private Bucket createBucketAtEpochTime(long epoch) { + Bucket b = new Bucket("foo", new Date(epoch), 123); + b.setMaxNormalizedProbability(10.0); + return b; + } + + private JobProvider createProvider(Client client) { + return new JobProvider(client, 0); + } + + private static GetResponse createGetResponse(boolean exists, Map source) throws IOException { + GetResponse getResponse = mock(GetResponse.class); + when(getResponse.isExists()).thenReturn(exists); + when(getResponse.getSourceAsBytesRef()).thenReturn(XContentFactory.jsonBuilder().map(source).bytes()); + return getResponse; + } + + private static SearchResponse createSearchResponse(boolean exists, List> source) throws IOException { + SearchResponse response = mock(SearchResponse.class); + List list = new ArrayList<>(); + + for (Map map : source) { + Map _source = new HashMap<>(map); + + Map fields = new HashMap<>(); + fields.put("field_1", new SearchHitField("field_1", Arrays.asList("foo"))); + fields.put("field_2", new SearchHitField("field_2", Arrays.asList("foo"))); + + SearchHit hit = new SearchHit(123, String.valueOf(map.hashCode()), new Text("foo"), fields) + .sourceRef(XContentFactory.jsonBuilder().map(_source).bytes()); + + list.add(hit); + } + SearchHits hits = new SearchHits(list.toArray(new SearchHit[0]), source.size(), 1); + when(response.getHits()).thenReturn(hits); + + return response; + } + + private Client getMockedClient(Consumer queryBuilderConsumer, SearchResponse response) { + Client client = mock(Client.class); + doAnswer(invocationOnMock -> { + MultiSearchRequest multiSearchRequest = (MultiSearchRequest) invocationOnMock.getArguments()[0]; + queryBuilderConsumer.accept(multiSearchRequest.requests().get(0).source().query()); + @SuppressWarnings("unchecked") + ActionListener actionListener = (ActionListener) invocationOnMock.getArguments()[1]; + MultiSearchResponse mresponse = + new MultiSearchResponse(new MultiSearchResponse.Item[]{new MultiSearchResponse.Item(response, null)}); + actionListener.onResponse(mresponse); + return null; + }).when(client).multiSearch(any(), any()); + doAnswer(invocationOnMock -> { + SearchRequest searchRequest = (SearchRequest) invocationOnMock.getArguments()[0]; + queryBuilderConsumer.accept(searchRequest.source().query()); + @SuppressWarnings("unchecked") + ActionListener actionListener = (ActionListener) invocationOnMock.getArguments()[1]; + actionListener.onResponse(response); + return null; + }).when(client).search(any(), any()); + return client; + } + + private Client getMockedClient(GetResponse response) { + Client client = mock(Client.class); + @SuppressWarnings("unchecked") + ActionFuture actionFuture = mock(ActionFuture.class); + when(client.get(any())).thenReturn(actionFuture); + when(actionFuture.actionGet()).thenReturn(response); + + doAnswer(invocationOnMock -> { + @SuppressWarnings("unchecked") + ActionListener actionListener = (ActionListener) invocationOnMock.getArguments()[1]; + actionListener.onResponse(response); + return null; + }).when(client).get(any(), any()); + return client; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java new file mode 100644 index 00000000000..97ed255dc78 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersisterTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.process.normalizer.BucketNormalizable; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; + +import java.util.Date; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class JobRenormalizedResultsPersisterTests extends ESTestCase { + + public void testUpdateBucket() { + BucketNormalizable bn = createBucketNormalizable(); + JobRenormalizedResultsPersister persister = createJobRenormalizedResultsPersister(); + persister.updateBucket(bn); + + assertEquals(3, persister.getBulkRequest().numberOfActions()); + assertEquals("foo-index", persister.getBulkRequest().requests().get(0).index()); + } + + public void testExecuteRequestResetsBulkRequest() { + BucketNormalizable bn = createBucketNormalizable(); + JobRenormalizedResultsPersister persister = createJobRenormalizedResultsPersister(); + persister.updateBucket(bn); + persister.executeRequest("foo"); + assertEquals(0, persister.getBulkRequest().numberOfActions()); + } + + private JobRenormalizedResultsPersister createJobRenormalizedResultsPersister() { + BulkResponse bulkResponse = mock(BulkResponse.class); + when(bulkResponse.hasFailures()).thenReturn(false); + + Client client = new MockClientBuilder("cluster").bulk(bulkResponse).build(); + return new JobRenormalizedResultsPersister(Settings.EMPTY, client); + } + + private BucketNormalizable createBucketNormalizable() { + Date now = new Date(); + Bucket bucket = new Bucket("foo", now, 1); + int sequenceNum = 0; + bucket.addBucketInfluencer(new BucketInfluencer("foo", now, 1, sequenceNum++)); + bucket.addBucketInfluencer(new BucketInfluencer("foo", now, 1, sequenceNum++)); + return new BucketNormalizable(bucket, "foo-index"); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java new file mode 100644 index 00000000000..e392df7bd10 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; +import org.elasticsearch.xpack.ml.job.results.Bucket; +import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; +import org.elasticsearch.xpack.ml.job.results.Influencer; +import org.mockito.ArgumentCaptor; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + + +public class JobResultsPersisterTests extends ESTestCase { + + private static final String JOB_ID = "foo"; + + public void testPersistBucket_OneRecord() throws IOException { + ArgumentCaptor captor = ArgumentCaptor.forClass(BulkRequest.class); + Client client = mockClient(captor); + Bucket bucket = new Bucket("foo", new Date(), 123456); + bucket.setAnomalyScore(99.9); + bucket.setEventCount(57); + bucket.setInitialAnomalyScore(88.8); + bucket.setMaxNormalizedProbability(42.0); + bucket.setProcessingTimeMs(8888); + bucket.setRecordCount(1); + + BucketInfluencer bi = new BucketInfluencer(JOB_ID, new Date(), 600, 1); + bi.setAnomalyScore(14.15); + bi.setInfluencerFieldName("biOne"); + bi.setInitialAnomalyScore(18.12); + bi.setProbability(0.0054); + bi.setRawAnomalyScore(19.19); + bucket.addBucketInfluencer(bi); + + // We are adding a record but it shouldn't be persisted as part of the bucket + AnomalyRecord record = new AnomalyRecord(JOB_ID, new Date(), 600, 2); + record.setAnomalyScore(99.8); + bucket.setRecords(Arrays.asList(record)); + + JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client); + persister.bulkPersisterBuilder(JOB_ID).persistBucket(bucket).executeRequest(); + BulkRequest bulkRequest = captor.getValue(); + assertEquals(2, bulkRequest.numberOfActions()); + + String s = ((IndexRequest)bulkRequest.requests().get(0)).source().utf8ToString(); + assertTrue(s.matches(".*anomaly_score.:99\\.9.*")); + assertTrue(s.matches(".*initial_anomaly_score.:88\\.8.*")); + assertTrue(s.matches(".*max_normalized_probability.:42\\.0.*")); + assertTrue(s.matches(".*record_count.:1.*")); + assertTrue(s.matches(".*event_count.:57.*")); + assertTrue(s.matches(".*bucket_span.:123456.*")); + assertTrue(s.matches(".*processing_time_ms.:8888.*")); + // There should NOT be any nested records + assertFalse(s.matches(".*records*")); + + s = ((IndexRequest)bulkRequest.requests().get(1)).source().utf8ToString(); + assertTrue(s.matches(".*probability.:0\\.0054.*")); + assertTrue(s.matches(".*influencer_field_name.:.biOne.*")); + assertTrue(s.matches(".*initial_anomaly_score.:18\\.12.*")); + assertTrue(s.matches(".*anomaly_score.:14\\.15.*")); + assertTrue(s.matches(".*raw_anomaly_score.:19\\.19.*")); + } + + public void testPersistRecords() throws IOException { + ArgumentCaptor captor = ArgumentCaptor.forClass(BulkRequest.class); + Client client = mockClient(captor); + + List records = new ArrayList<>(); + AnomalyRecord r1 = new AnomalyRecord(JOB_ID, new Date(), 42, 1); + records.add(r1); + List actuals = new ArrayList<>(); + actuals.add(5.0); + actuals.add(5.1); + r1.setActual(actuals); + r1.setAnomalyScore(99.8); + r1.setByFieldName("byName"); + r1.setByFieldValue("byValue"); + r1.setCorrelatedByFieldValue("testCorrelations"); + r1.setDetectorIndex(3); + r1.setFieldName("testFieldName"); + r1.setFunction("testFunction"); + r1.setFunctionDescription("testDescription"); + r1.setInitialNormalizedProbability(23.4); + r1.setNormalizedProbability(0.005); + r1.setOverFieldName("overName"); + r1.setOverFieldValue("overValue"); + r1.setPartitionFieldName("partName"); + r1.setPartitionFieldValue("partValue"); + r1.setProbability(0.1); + List typicals = new ArrayList<>(); + typicals.add(0.44); + typicals.add(998765.3); + r1.setTypical(typicals); + + JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client); + persister.bulkPersisterBuilder(JOB_ID).persistRecords(records).executeRequest(); + BulkRequest bulkRequest = captor.getValue(); + assertEquals(1, bulkRequest.numberOfActions()); + + String s = ((IndexRequest) bulkRequest.requests().get(0)).source().utf8ToString(); + assertTrue(s.matches(".*detector_index.:3.*")); + assertTrue(s.matches(".*\"probability\":0\\.1.*")); + assertTrue(s.matches(".*\"anomaly_score\":99\\.8.*")); + assertTrue(s.matches(".*\"normalized_probability\":0\\.005.*")); + assertTrue(s.matches(".*initial_normalized_probability.:23.4.*")); + assertTrue(s.matches(".*bucket_span.:42.*")); + assertTrue(s.matches(".*by_field_name.:.byName.*")); + assertTrue(s.matches(".*by_field_value.:.byValue.*")); + assertTrue(s.matches(".*correlated_by_field_value.:.testCorrelations.*")); + assertTrue(s.matches(".*typical.:.0\\.44,998765\\.3.*")); + assertTrue(s.matches(".*actual.:.5\\.0,5\\.1.*")); + assertTrue(s.matches(".*field_name.:.testFieldName.*")); + assertTrue(s.matches(".*function.:.testFunction.*")); + assertTrue(s.matches(".*function_description.:.testDescription.*")); + assertTrue(s.matches(".*partition_field_name.:.partName.*")); + assertTrue(s.matches(".*partition_field_value.:.partValue.*")); + assertTrue(s.matches(".*over_field_name.:.overName.*")); + assertTrue(s.matches(".*over_field_value.:.overValue.*")); + } + + public void testPersistInfluencers() throws IOException { + ArgumentCaptor captor = ArgumentCaptor.forClass(BulkRequest.class); + Client client = mockClient(captor); + + List influencers = new ArrayList<>(); + Influencer inf = new Influencer(JOB_ID, "infName1", "infValue1", new Date(), 600, 1); + inf.setAnomalyScore(16); + inf.setInitialAnomalyScore(55.5); + inf.setProbability(0.4); + influencers.add(inf); + + JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client); + persister.bulkPersisterBuilder(JOB_ID).persistInfluencers(influencers).executeRequest(); + BulkRequest bulkRequest = captor.getValue(); + assertEquals(1, bulkRequest.numberOfActions()); + + String s = ((IndexRequest) bulkRequest.requests().get(0)).source().utf8ToString(); + assertTrue(s.matches(".*probability.:0\\.4.*")); + assertTrue(s.matches(".*influencer_field_name.:.infName1.*")); + assertTrue(s.matches(".*influencer_field_value.:.infValue1.*")); + assertTrue(s.matches(".*initial_anomaly_score.:55\\.5.*")); + assertTrue(s.matches(".*anomaly_score.:16\\.0.*")); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private Client mockClient(ArgumentCaptor captor) { + Client client = mock(Client.class); + ActionFuture future = mock(ActionFuture.class); + when(future.actionGet()).thenReturn(new BulkResponse(new BulkItemResponse[0], 0L)); + when(client.bulk(captor.capture())).thenReturn(future); + return client; + } +} diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockBatchedDocumentsIterator.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockBatchedDocumentsIterator.java new file mode 100644 index 00000000000..f2eccaa9db1 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockBatchedDocumentsIterator.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.client.Client; +import org.elasticsearch.search.SearchHit; + +import java.util.Deque; +import java.util.List; +import java.util.NoSuchElementException; + +import static org.mockito.Mockito.mock; + +public class MockBatchedDocumentsIterator extends BatchedDocumentsIterator { + private final List> batches; + private int index; + private boolean wasTimeRangeCalled; + private String interimFieldName; + + public MockBatchedDocumentsIterator(List> batches) { + super(mock(Client.class), "foo"); + this.batches = batches; + index = 0; + wasTimeRangeCalled = false; + interimFieldName = ""; + } + + @Override + public BatchedDocumentsIterator timeRange(long startEpochMs, long endEpochMs) { + wasTimeRangeCalled = true; + return this; + } + + @Override + public BatchedDocumentsIterator includeInterim(String interimFieldName) { + this.interimFieldName = interimFieldName; + return this; + } + + @Override + public Deque next() { + if ((!wasTimeRangeCalled) || !hasNext()) { + throw new NoSuchElementException(); + } + return batches.get(index++); + } + + @Override + protected String getType() { + return null; + } + + @Override + protected T map(SearchHit hit) { + return null; + } + + @Override + public boolean hasNext() { + return index != batches.size(); + } + + /** + * If includeInterim has not been called this is an empty string + */ + public String getInterimFieldName() { + return interimFieldName; + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java new file mode 100644 index 00000000000..3bb0e796187 --- /dev/null +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java @@ -0,0 +1,353 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; +import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; +import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; +import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetRequestBuilder; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchScrollRequestBuilder; +import org.elasticsearch.action.support.WriteRequest.RefreshPolicy; +import org.elasticsearch.action.update.UpdateRequestBuilder; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.AdminClient; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.ClusterAdminClient; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xpack.ml.action.DeleteJobAction; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class MockClientBuilder { + @Mock + private Client client; + + @Mock + private AdminClient adminClient; + @Mock + private ClusterAdminClient clusterAdminClient; + @Mock + private IndicesAdminClient indicesAdminClient; + + public MockClientBuilder(String clusterName) { + client = mock(Client.class); + adminClient = mock(AdminClient.class); + clusterAdminClient = mock(ClusterAdminClient.class); + indicesAdminClient = mock(IndicesAdminClient.class); + + when(client.admin()).thenReturn(adminClient); + when(adminClient.cluster()).thenReturn(clusterAdminClient); + when(adminClient.indices()).thenReturn(indicesAdminClient); + Settings settings = Settings.builder().put("cluster.name", clusterName).build(); + when(client.settings()).thenReturn(settings); + } + + @SuppressWarnings({ "unchecked" }) + public MockClientBuilder addClusterStatusYellowResponse() throws InterruptedException, ExecutionException { + ListenableActionFuture actionFuture = mock(ListenableActionFuture.class); + ClusterHealthRequestBuilder clusterHealthRequestBuilder = mock(ClusterHealthRequestBuilder.class); + + when(clusterAdminClient.prepareHealth()).thenReturn(clusterHealthRequestBuilder); + when(clusterHealthRequestBuilder.setWaitForYellowStatus()).thenReturn(clusterHealthRequestBuilder); + when(clusterHealthRequestBuilder.execute()).thenReturn(actionFuture); + when(actionFuture.actionGet()).thenReturn(mock(ClusterHealthResponse.class)); + return this; + } + + @SuppressWarnings({ "unchecked" }) + public MockClientBuilder addClusterStatusYellowResponse(String index) throws InterruptedException, ExecutionException { + ListenableActionFuture actionFuture = mock(ListenableActionFuture.class); + ClusterHealthRequestBuilder clusterHealthRequestBuilder = mock(ClusterHealthRequestBuilder.class); + + when(clusterAdminClient.prepareHealth(index)).thenReturn(clusterHealthRequestBuilder); + when(clusterHealthRequestBuilder.setWaitForYellowStatus()).thenReturn(clusterHealthRequestBuilder); + when(clusterHealthRequestBuilder.execute()).thenReturn(actionFuture); + when(actionFuture.actionGet()).thenReturn(mock(ClusterHealthResponse.class)); + return this; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public MockClientBuilder addIndicesExistsResponse(String index, boolean exists) throws InterruptedException, ExecutionException { + ActionFuture actionFuture = mock(ActionFuture.class); + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(IndicesExistsRequest.class); + + when(indicesAdminClient.exists(requestCaptor.capture())).thenReturn(actionFuture); + doAnswer(invocation -> { + IndicesExistsRequest request = (IndicesExistsRequest) invocation.getArguments()[0]; + return request.indices()[0].equals(index) ? actionFuture : null; + }).when(indicesAdminClient).exists(any(IndicesExistsRequest.class)); + when(actionFuture.get()).thenReturn(new IndicesExistsResponse(exists)); + when(actionFuture.actionGet()).thenReturn(new IndicesExistsResponse(exists)); + return this; + } + + @SuppressWarnings({ "unchecked" }) + public MockClientBuilder addIndicesDeleteResponse(String index, boolean exists, boolean exception, + ActionListener actionListener) throws InterruptedException, ExecutionException, IOException { + DeleteIndexResponse response = DeleteIndexAction.INSTANCE.newResponse(); + StreamInput si = mock(StreamInput.class); + // this looks complicated but Mockito can't mock the final method + // DeleteIndexResponse.isAcknowledged() and the only way to create + // one with a true response is reading from a stream. + when(si.readByte()).thenReturn((byte) 0x01); + response.readFrom(si); + + doAnswer(invocation -> { + DeleteIndexRequest deleteIndexRequest = (DeleteIndexRequest) invocation.getArguments()[0]; + assertArrayEquals(new String[] { index }, deleteIndexRequest.indices()); + if (exception) { + actionListener.onFailure(new InterruptedException()); + } else { + actionListener.onResponse(new DeleteJobAction.Response(true)); + } + return null; + }).when(indicesAdminClient).delete(any(DeleteIndexRequest.class), any(ActionListener.class)); + return this; + } + + public MockClientBuilder prepareGet(String index, String type, String id, GetResponse response) { + GetRequestBuilder getRequestBuilder = mock(GetRequestBuilder.class); + when(getRequestBuilder.get()).thenReturn(response); + when(getRequestBuilder.setFetchSource(false)).thenReturn(getRequestBuilder); + when(client.prepareGet(index, type, id)).thenReturn(getRequestBuilder); + return this; + } + + public MockClientBuilder prepareCreate(String index) { + CreateIndexRequestBuilder createIndexRequestBuilder = mock(CreateIndexRequestBuilder.class); + CreateIndexResponse response = mock(CreateIndexResponse.class); + when(createIndexRequestBuilder.setSettings(any(Settings.Builder.class))).thenReturn(createIndexRequestBuilder); + when(createIndexRequestBuilder.addMapping(any(String.class), any(XContentBuilder.class))).thenReturn(createIndexRequestBuilder); + when(createIndexRequestBuilder.get()).thenReturn(response); + when(indicesAdminClient.prepareCreate(eq(index))).thenReturn(createIndexRequestBuilder); + return this; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + public MockClientBuilder createIndexRequest(String index, ArgumentCaptor requestCapture) { + + doAnswer(invocation -> { + ((ActionListener) invocation.getArguments()[1]).onResponse(mock(CreateIndexResponse.class)); + return null; + }).when(indicesAdminClient).create(requestCapture.capture(), any(ActionListener.class)); + return this; + } + + @SuppressWarnings("unchecked") + public MockClientBuilder prepareSearchExecuteListener(String index, SearchResponse response) { + SearchRequestBuilder builder = mock(SearchRequestBuilder.class); + when(builder.setTypes(anyString())).thenReturn(builder); + when(builder.addSort(any(SortBuilder.class))).thenReturn(builder); + when(builder.setFetchSource(anyBoolean())).thenReturn(builder); + when(builder.setScroll(anyString())).thenReturn(builder); + when(builder.addDocValueField(any(String.class))).thenReturn(builder); + when(builder.addSort(any(String.class), any(SortOrder.class))).thenReturn(builder); + when(builder.setQuery(any())).thenReturn(builder); + when(builder.setSize(anyInt())).thenReturn(builder); + + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocationOnMock) throws Throwable { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[0]; + listener.onResponse(response); + return null; + } + }).when(builder).execute(any()); + + when(client.prepareSearch(eq(index))).thenReturn(builder); + + return this; + } + + @SuppressWarnings("unchecked") + public MockClientBuilder prepareSearchScrollExecuteListener(SearchResponse response) { + SearchScrollRequestBuilder builder = mock(SearchScrollRequestBuilder.class); + when(builder.setScroll(anyString())).thenReturn(builder); + when(builder.setScrollId(anyString())).thenReturn(builder); + + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocationOnMock) throws Throwable { + ActionListener listener = (ActionListener) invocationOnMock.getArguments()[0]; + listener.onResponse(response); + return null; + } + }).when(builder).execute(any()); + + when(client.prepareSearchScroll(anyString())).thenReturn(builder); + + return this; + } + + public MockClientBuilder prepareSearch(String index, String type, int from, int size, SearchResponse response, + ArgumentCaptor filter) { + SearchRequestBuilder builder = mock(SearchRequestBuilder.class); + when(builder.setTypes(eq(type))).thenReturn(builder); + when(builder.addSort(any(SortBuilder.class))).thenReturn(builder); + when(builder.setQuery(filter.capture())).thenReturn(builder); + when(builder.setPostFilter(filter.capture())).thenReturn(builder); + when(builder.setFrom(eq(from))).thenReturn(builder); + when(builder.setSize(eq(size))).thenReturn(builder); + when(builder.setFetchSource(eq(true))).thenReturn(builder); + when(builder.addDocValueField(any(String.class))).thenReturn(builder); + when(builder.addSort(any(String.class), any(SortOrder.class))).thenReturn(builder); + when(builder.get()).thenReturn(response); + when(client.prepareSearch(eq(index))).thenReturn(builder); + return this; + } + + public MockClientBuilder prepareSearchAnySize(String index, String type, SearchResponse response, ArgumentCaptor filter) { + SearchRequestBuilder builder = mock(SearchRequestBuilder.class); + when(builder.setTypes(eq(type))).thenReturn(builder); + when(builder.addSort(any(SortBuilder.class))).thenReturn(builder); + when(builder.setQuery(filter.capture())).thenReturn(builder); + when(builder.setPostFilter(filter.capture())).thenReturn(builder); + when(builder.setFrom(any(Integer.class))).thenReturn(builder); + when(builder.setSize(any(Integer.class))).thenReturn(builder); + when(builder.setFetchSource(eq(true))).thenReturn(builder); + when(builder.addDocValueField(any(String.class))).thenReturn(builder); + when(builder.addSort(any(String.class), any(SortOrder.class))).thenReturn(builder); + when(builder.get()).thenReturn(response); + when(client.prepareSearch(eq(index))).thenReturn(builder); + return this; + } + + @SuppressWarnings("unchecked") + public MockClientBuilder prepareIndex(String index, String type, String responseId, ArgumentCaptor getSource) { + IndexRequestBuilder builder = mock(IndexRequestBuilder.class); + ListenableActionFuture actionFuture = mock(ListenableActionFuture.class); + IndexResponse response = mock(IndexResponse.class); + when(response.getId()).thenReturn(responseId); + + when(client.prepareIndex(eq(index), eq(type))).thenReturn(builder); + when(client.prepareIndex(eq(index), eq(type), any(String.class))).thenReturn(builder); + when(builder.setSource(getSource.capture())).thenReturn(builder); + when(builder.setRefreshPolicy(eq(RefreshPolicy.IMMEDIATE))).thenReturn(builder); + when(builder.execute()).thenReturn(actionFuture); + when(actionFuture.actionGet()).thenReturn(response); + return this; + } + + @SuppressWarnings("unchecked") + public MockClientBuilder prepareAlias(String indexName, String alias) { + IndicesAliasesRequestBuilder aliasesRequestBuilder = mock(IndicesAliasesRequestBuilder.class); + when(aliasesRequestBuilder.addAlias(eq(indexName), eq(alias))).thenReturn(aliasesRequestBuilder); + when(indicesAdminClient.prepareAliases()).thenReturn(aliasesRequestBuilder); + doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocationOnMock) throws Throwable { + ActionListener listener = + (ActionListener) invocationOnMock.getArguments()[0]; + listener.onResponse(mock(IndicesAliasesResponse.class)); + return null; + } + }).when(aliasesRequestBuilder).execute(any()); + return this; + } + + @SuppressWarnings("unchecked") + public MockClientBuilder prepareBulk(BulkResponse response) { + ListenableActionFuture actionFuture = mock(ListenableActionFuture.class); + BulkRequestBuilder builder = mock(BulkRequestBuilder.class); + when(client.prepareBulk()).thenReturn(builder); + when(builder.execute()).thenReturn(actionFuture); + when(actionFuture.actionGet()).thenReturn(response); + return this; + } + + @SuppressWarnings("unchecked") + public MockClientBuilder bulk(BulkResponse response) { + ActionFuture actionFuture = mock(ActionFuture.class); + when(client.bulk(any(BulkRequest.class))).thenReturn(actionFuture); + when(actionFuture.actionGet()).thenReturn(response); + return this; + } + + public MockClientBuilder prepareUpdateScript(String index, String type, String id, ArgumentCaptor