diff --git a/plugin/bwc-snapshot-dummy-projects/build.gradle b/plugin/bwc-snapshot-dummy-projects/build.gradle index ba676471d62..c664cb173ad 100644 --- a/plugin/bwc-snapshot-dummy-projects/build.gradle +++ b/plugin/bwc-snapshot-dummy-projects/build.gradle @@ -33,6 +33,7 @@ subprojects { */ Object esCheckoutPath = """${-> if (Os.isFamily(Os.FAMILY_WINDOWS)) { + esCheckoutDir.mkdirs() NodeInfo.getShortPathName(esCheckoutDir.toString()) } else { esCheckoutDir.toString() @@ -41,6 +42,7 @@ subprojects { File xpackCheckoutDir = file("${esCheckoutDir}-extra/x-pack-elasticsearch") Object xpackCheckoutPath = """${-> if (Os.isFamily(Os.FAMILY_WINDOWS)) { + xpackCheckoutDir.mkdirs() NodeInfo.getShortPathName(xpackCheckoutDir.toString()) } else { xpackCheckoutDir.toString() diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 8d3e5e8c135..f6bd76593e6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -53,6 +52,7 @@ import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.ml.action.CloseJobAction; import org.elasticsearch.xpack.ml.action.DeleteCalendarAction; +import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction; import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.ml.action.DeleteExpiredDataAction; import org.elasticsearch.xpack.ml.action.DeleteFilterAction; @@ -118,8 +118,10 @@ import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarAction; +import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarJobAction; import org.elasticsearch.xpack.ml.rest.calendar.RestGetCalendarsAction; import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarAction; +import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarJobAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestDeleteDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedStatsAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedsAction; @@ -467,7 +469,9 @@ public class MachineLearning implements ActionPlugin { new RestForecastJobAction(settings, restController), new RestGetCalendarsAction(settings, restController), new RestPutCalendarAction(settings, restController), - new RestDeleteCalendarAction(settings, restController) + new RestDeleteCalendarAction(settings, restController), + new RestDeleteCalendarJobAction(settings, restController), + new RestPutCalendarJobAction(settings, restController) ); } @@ -516,7 +520,8 @@ public class MachineLearning implements ActionPlugin { new ActionHandler<>(ForecastJobAction.INSTANCE, ForecastJobAction.TransportAction.class), new ActionHandler<>(GetCalendarsAction.INSTANCE, GetCalendarsAction.TransportAction.class), new ActionHandler<>(PutCalendarAction.INSTANCE, PutCalendarAction.TransportAction.class), - new ActionHandler<>(DeleteCalendarAction.INSTANCE, DeleteCalendarAction.TransportAction.class) + new ActionHandler<>(DeleteCalendarAction.INSTANCE, DeleteCalendarAction.TransportAction.class), + new ActionHandler<>(UpdateCalendarJobAction.INSTANCE, UpdateCalendarJobAction.TransportAction.class) ); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java index 95e79cea92e..cc8153c0777 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; @@ -32,6 +33,12 @@ public final class MlMetaIndex { builder.startObject(TYPE); ElasticsearchMappings.addDefaultMapping(builder); builder.startObject(ElasticsearchMappings.PROPERTIES) + .startObject(Calendar.ID.getPreferredName()) + .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) + .endObject() + .startObject(Calendar.JOB_IDS.getPreferredName()) + .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) + .endObject() .startObject(SpecialEvent.START_TIME.getPreferredName()) .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) .endObject() diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java index fb7cba02392..d6bf90936f6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java @@ -81,6 +81,10 @@ public class MlMetadata implements MetaData.Custom { return jobs; } + public boolean isGroupOrJob(String id) { + return groupOrJobLookup.isGroupOrJob(id); + } + public Set expandJobIds(String expression, boolean allowNoJobs) { return groupOrJobLookup.expandJobIds(expression, allowNoJobs); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java index aafb33d2aa1..c547745bd79 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.ml.MlMetaIndex; import org.elasticsearch.xpack.ml.action.util.PageParams; import org.elasticsearch.xpack.ml.action.util.QueryPage; import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import java.io.IOException; @@ -212,16 +213,16 @@ public class GetCalendarsAction extends Action { - private final Client client; + private final JobProvider jobProvider; @Inject public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - Client client) { + JobProvider jobProvider) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); - this.client = client; + this.jobProvider = jobProvider; } @Override @@ -239,76 +240,24 @@ public class GetCalendarsAction extends Action listener) { - GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId)); - executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { - @Override - public void onResponse(GetResponse getDocResponse) { - try { - QueryPage calendars; - if (getDocResponse.isExists()) { - BytesReference docSource = getDocResponse.getSourceAsBytesRef(); - - try (XContentParser parser = - XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource)) { - Calendar calendar = Calendar.PARSER.apply(parser, null).build(); - calendars = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD); - - Response response = new Response(calendars); - listener.onResponse(response); - } - } else { - this.onFailure(QueryPage.emptyQueryPage(Calendar.RESULTS_FIELD)); - } - - } catch (Exception e) { - this.onFailure(e); - } - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + jobProvider.calendar(calendarId, ActionListener.wrap( + calendar -> { + QueryPage page = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD); + listener.onResponse(new Response(page)); + }, + listener::onFailure + )); } private void getCalendars(PageParams pageParams, ActionListener listener) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() - .from(pageParams.getFrom()) - .size(pageParams.getSize()) - .sort(Calendar.ID.getPreferredName()) - .query(QueryBuilders.termQuery(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE)); - - SearchRequest searchRequest = new SearchRequest(MlMetaIndex.INDEX_NAME) - .indicesOptions(JobProvider.addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS)) - .source(sourceBuilder); - - executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, new ActionListener() { - @Override - public void onResponse(SearchResponse response) { - List docs = new ArrayList<>(); - for (SearchHit hit : response.getHits().getHits()) { - BytesReference docSource = hit.getSourceRef(); - try (XContentParser parser = XContentFactory.xContent(docSource).createParser( - NamedXContentRegistry.EMPTY, docSource)) { - docs.add(Calendar.PARSER.apply(parser, null).build()); - } catch (IOException e) { - this.onFailure(e); - } - } - - Response getResponse = new Response( - new QueryPage<>(docs, docs.size(), Calendar.RESULTS_FIELD)); - listener.onResponse(getResponse); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, - client::search); + CalendarQueryBuilder query = new CalendarQueryBuilder().pageParams(pageParams).sort(true); + jobProvider.calendars(query, ActionListener.wrap( + calendars -> { + listener.onResponse(new Response(calendars)); + }, + listener::onFailure + )); } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java index c8270866e39..6ba0aa5eaf3 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java @@ -20,7 +20,9 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,13 +36,18 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ml.MlMetaIndex; +import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.watcher.support.Exceptions; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.ClientHelper.ML_ORIGIN; @@ -162,36 +169,62 @@ public class PutCalendarAction extends Action { private final Client client; + private final ClusterService clusterService; @Inject public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, ClusterService clusterService) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); this.client = client; + this.clusterService = clusterService; } @Override protected void doExecute(Request request, ActionListener listener) { - final Calendar calendar = request.getCalendar(); + Calendar calendar = request.getCalendar(); + + checkJobsExist(calendar.getJobIds(), listener::onFailure); + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { indexRequest.source(calendar.toXContent(builder, @@ -218,5 +251,17 @@ public class PutCalendarAction extends Action jobIds, Consumer errorHandler) { + ClusterState state = clusterService.state(); + MlMetadata mlMetadata = state.getMetaData().custom(MlMetadata.TYPE); + for (String jobId: jobIds) { + Set jobs = mlMetadata.expandJobIds(jobId, true); + if (jobs.isEmpty()) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + return; + } + } + } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java new file mode 100644 index 00000000000..7753cfb4923 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.MlMetadata; +import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +public class UpdateCalendarJobAction extends Action { + public static final UpdateCalendarJobAction INSTANCE = new UpdateCalendarJobAction(); + public static final String NAME = "cluster:admin/xpack/ml/calendars/jobs/update"; + + private UpdateCalendarJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public PutCalendarAction.Response newResponse() { + return new PutCalendarAction.Response(); + } + + public static class Request extends ActionRequest { + + private String calendarId; + private Set jobIdsToAdd; + private Set jobIdsToRemove; + + Request() { + } + + public Request(String calendarId, Set jobIdsToAdd, Set jobIdsToRemove) { + this.calendarId = ExceptionsHelper.requireNonNull(calendarId, Calendar.ID.getPreferredName()); + this.jobIdsToAdd = ExceptionsHelper.requireNonNull(jobIdsToAdd, "job_ids_to_add"); + this.jobIdsToRemove = ExceptionsHelper.requireNonNull(jobIdsToRemove, "job_ids_to_remove"); + } + + public String getCalendarId() { + return calendarId; + } + + public Set getJobIdsToAdd() { + return jobIdsToAdd; + } + + public Set getJobIdsToRemove() { + return jobIdsToRemove; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + calendarId = in.readString(); + jobIdsToAdd = new HashSet<>(in.readList(StreamInput::readString)); + jobIdsToRemove = new HashSet<>(in.readList(StreamInput::readString)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(calendarId); + out.writeStringList(new ArrayList<>(jobIdsToAdd)); + out.writeStringList(new ArrayList<>(jobIdsToRemove)); + } + + @Override + public int hashCode() { + return Objects.hash(calendarId, jobIdsToAdd, jobIdsToRemove); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(calendarId, other.calendarId) && Objects.equals(jobIdsToAdd, other.jobIdsToAdd) + && Objects.equals(jobIdsToRemove, other.jobIdsToRemove); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final ClusterService clusterService; + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, + TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, + indexNameExpressionResolver, Request::new); + this.clusterService = clusterService; + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + ClusterState state = clusterService.state(); + MlMetadata mlMetadata = state.getMetaData().custom(MlMetadata.TYPE); + for (String jobToAdd: request.getJobIdsToAdd()) { + if (mlMetadata.isGroupOrJob(jobToAdd) == false) { + listener.onFailure(ExceptionsHelper.missingJobException(jobToAdd)); + return; + } + } + + for (String jobToRemove: request.getJobIdsToRemove()) { + if (mlMetadata.isGroupOrJob(jobToRemove) == false) { + listener.onFailure(ExceptionsHelper.missingJobException(jobToRemove)); + return; + } + } + + jobProvider.updateCalendar(request.getCalendarId(), request.getJobIdsToAdd(), request.getJobIdsToRemove(), + c -> listener.onResponse(new PutCalendarAction.Response(c)), listener::onFailure); + } + } +} + diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java index ed1d2c5f093..5a99fc6c6ef 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java @@ -13,10 +13,8 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.ml.MlMetaIndex; -import org.elasticsearch.xpack.ml.job.config.MlFilter; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -70,7 +68,7 @@ public class Calendar implements ToXContentObject, Writeable { } public List getJobIds() { - return new ArrayList<>(jobIds); + return Collections.unmodifiableList(jobIds); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java index 339ca80351a..f65c33f2c9e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java @@ -150,7 +150,7 @@ public class SpecialEvent implements ToXContentObject, Writeable { conditions.add(RuleCondition.createTime(Operator.LT, bucketEndTime)); DetectionRule.Builder builder = new DetectionRule.Builder(conditions); - builder.setRuleAction(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS); + builder.setActions(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING); builder.setConditionsConnective(Connective.AND); return builder.build(); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 3b69c97f404..e1114a0ed0b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -313,7 +313,7 @@ public class JobManager extends AbstractComponent { String jobId = request.getJobId(); logger.debug("Deleting job '" + jobId + "'"); - // Step 3. When the job has been removed from the cluster state, return a response + // Step 4. When the job has been removed from the cluster state, return a response // ------- CheckedConsumer apiResponseHandler = jobDeleted -> { if (jobDeleted) { @@ -325,7 +325,7 @@ public class JobManager extends AbstractComponent { } }; - // Step 2. When the physical storage has been deleted, remove from Cluster State + // Step 3. When the physical storage has been deleted, remove from Cluster State // ------- CheckedConsumer deleteJobStateHandler = response -> clusterService.submitStateUpdateTask("delete-job-" + jobId, new AckedClusterStateUpdateTask(request, ActionListener.wrap(apiResponseHandler, actionListener::onFailure)) { @@ -351,11 +351,18 @@ public class JobManager extends AbstractComponent { } }); + + // Step 2. Remove the job from any calendars + CheckedConsumer removeFromCalendarsHandler = response -> { + jobProvider.removeJobFromCalendars(jobId, ActionListener.wrap(deleteJobStateHandler::accept, + actionListener::onFailure )); + }; + + // Step 1. Delete the physical storage // This task manages the physical deletion of the job state and results - task.delete(jobId, client, clusterService.state(), deleteJobStateHandler::accept, actionListener::onFailure); - + task.delete(jobId, client, clusterService.state(), removeFromCalendarsHandler, actionListener::onFailure); } public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionListener actionListener, diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java index 8b244bc47c7..537593bd8cb 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java @@ -21,8 +21,10 @@ import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,11 +34,11 @@ import java.util.stream.Collectors; public class DetectionRule implements ToXContentObject, Writeable { public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule"); - public static final ParseField RULE_ACTION_FIELD = new ParseField("rule_action"); + public static final ParseField ACTIONS_FIELD = new ParseField("actions"); public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name"); public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value"); public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective"); - public static final ParseField RULE_CONDITIONS_FIELD = new ParseField("rule_conditions"); + public static final ParseField CONDITIONS_FIELD = new ParseField("conditions"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ObjectParser METADATA_PARSER = @@ -51,12 +53,7 @@ public class DetectionRule implements ToXContentObject, Writeable { for (MlParserType parserType : MlParserType.values()) { ObjectParser parser = PARSERS.get(parserType); assert parser != null; - parser.declareField(Builder::setRuleAction, p -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return RuleAction.fromString(p.text()); - } - throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, RULE_ACTION_FIELD, ValueType.STRING); + parser.declareStringArray(Builder::setActions, ACTIONS_FIELD); parser.declareString(Builder::setTargetFieldName, TARGET_FIELD_NAME_FIELD); parser.declareString(Builder::setTargetFieldValue, TARGET_FIELD_VALUE_FIELD); parser.declareField(Builder::setConditionsConnective, p -> { @@ -65,33 +62,38 @@ public class DetectionRule implements ToXContentObject, Writeable { } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING); - parser.declareObjectArray(Builder::setRuleConditions, (p, c) -> - RuleCondition.PARSERS.get(parserType).apply(p, c), RULE_CONDITIONS_FIELD); + parser.declareObjectArray(Builder::setConditions, (p, c) -> + RuleCondition.PARSERS.get(parserType).apply(p, c), CONDITIONS_FIELD); } } - private final RuleAction ruleAction; + private final EnumSet actions; private final String targetFieldName; private final String targetFieldValue; private final Connective conditionsConnective; - private final List ruleConditions; + private final List conditions; - private DetectionRule(RuleAction ruleAction, @Nullable String targetFieldName, @Nullable String targetFieldValue, - Connective conditionsConnective, List ruleConditions) { - this.ruleAction = Objects.requireNonNull(ruleAction); + private DetectionRule(EnumSet actions, @Nullable String targetFieldName, @Nullable String targetFieldValue, + Connective conditionsConnective, List conditions) { + this.actions = Objects.requireNonNull(actions); this.targetFieldName = targetFieldName; this.targetFieldValue = targetFieldValue; this.conditionsConnective = Objects.requireNonNull(conditionsConnective); - this.ruleConditions = Collections.unmodifiableList(ruleConditions); + this.conditions = Collections.unmodifiableList(conditions); } public DetectionRule(StreamInput in) throws IOException { - ruleAction = RuleAction.readFromStream(in); + int actionsCount = in.readVInt(); + actions = EnumSet.noneOf(RuleAction.class); + for (int i = 0; i < actionsCount; ++i) { + actions.add(RuleAction.readFromStream(in)); + } + conditionsConnective = Connective.readFromStream(in); int size = in.readVInt(); - ruleConditions = new ArrayList<>(size); + conditions = new ArrayList<>(size); for (int i = 0; i < size; i++) { - ruleConditions.add(new RuleCondition(in)); + conditions.add(new RuleCondition(in)); } targetFieldName = in.readOptionalString(); targetFieldValue = in.readOptionalString(); @@ -99,10 +101,14 @@ public class DetectionRule implements ToXContentObject, Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - ruleAction.writeTo(out); + out.writeVInt(actions.size()); + for (RuleAction action : actions) { + action.writeTo(out); + } + conditionsConnective.writeTo(out); - out.writeVInt(ruleConditions.size()); - for (RuleCondition condition : ruleConditions) { + out.writeVInt(conditions.size()); + for (RuleCondition condition : conditions) { condition.writeTo(out); } out.writeOptionalString(targetFieldName); @@ -112,9 +118,9 @@ public class DetectionRule implements ToXContentObject, Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(RULE_ACTION_FIELD.getPreferredName(), ruleAction); + builder.field(ACTIONS_FIELD.getPreferredName(), actions); builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective); - builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions); + builder.field(CONDITIONS_FIELD.getPreferredName(), conditions); if (targetFieldName != null) { builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName); } @@ -125,8 +131,8 @@ public class DetectionRule implements ToXContentObject, Writeable { return builder; } - public RuleAction getRuleAction() { - return ruleAction; + public EnumSet getActions() { + return actions; } @Nullable @@ -143,12 +149,12 @@ public class DetectionRule implements ToXContentObject, Writeable { return conditionsConnective; } - public List getRuleConditions() { - return ruleConditions; + public List getConditions() { + return conditions; } public Set extractReferencedFilters() { - return ruleConditions.stream().map(RuleCondition::getValueFilter).filter(Objects::nonNull).collect(Collectors.toSet()); + return conditions.stream().map(RuleCondition::getFilterId).filter(Objects::nonNull).collect(Collectors.toSet()); } @Override @@ -162,34 +168,46 @@ public class DetectionRule implements ToXContentObject, Writeable { } DetectionRule other = (DetectionRule) obj; - return Objects.equals(ruleAction, other.ruleAction) + return Objects.equals(actions, other.actions) && Objects.equals(targetFieldName, other.targetFieldName) && Objects.equals(targetFieldValue, other.targetFieldValue) && Objects.equals(conditionsConnective, other.conditionsConnective) - && Objects.equals(ruleConditions, other.ruleConditions); + && Objects.equals(conditions, other.conditions); } @Override public int hashCode() { - return Objects.hash(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions); + return Objects.hash(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions); } public static class Builder { - private RuleAction ruleAction = RuleAction.FILTER_RESULTS; + private EnumSet actions = EnumSet.of(RuleAction.FILTER_RESULTS); private String targetFieldName; private String targetFieldValue; private Connective conditionsConnective = Connective.OR; - private List ruleConditions = Collections.emptyList(); + private List conditions = Collections.emptyList(); - public Builder(List ruleConditions) { - this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_CONDITIONS_FIELD.getPreferredName()); + public Builder(List conditions) { + this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName()); } private Builder() { } - public Builder setRuleAction(RuleAction ruleAction) { - this.ruleAction = ExceptionsHelper.requireNonNull(ruleAction, RULE_ACTION_FIELD.getPreferredName()); + public Builder setActions(List actions) { + this.actions.clear(); + actions.stream().map(RuleAction::fromString).forEach(this.actions::add); + return this; + } + + public Builder setActions(EnumSet actions) { + this.actions = Objects.requireNonNull(actions, ACTIONS_FIELD.getPreferredName()); + return this; + } + + public Builder setActions(RuleAction... actions) { + this.actions.clear(); + Arrays.stream(actions).forEach(this.actions::add); return this; } @@ -208,8 +226,8 @@ public class DetectionRule implements ToXContentObject, Writeable { return this; } - public Builder setRuleConditions(List ruleConditions) { - this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_ACTION_FIELD.getPreferredName()); + public Builder setConditions(List conditions) { + this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName()); return this; } @@ -218,18 +236,18 @@ public class DetectionRule implements ToXContentObject, Writeable { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue); throw ExceptionsHelper.badRequestException(msg); } - if (ruleConditions == null || ruleConditions.isEmpty()) { + if (conditions == null || conditions.isEmpty()) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION); throw ExceptionsHelper.badRequestException(msg); } - for (RuleCondition condition : ruleConditions) { - if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) { + for (RuleCondition condition : conditions) { + if (condition.getType() == RuleConditionType.CATEGORICAL && targetFieldName != null) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName()); throw ExceptionsHelper.badRequestException(msg); } } - return new DetectionRule(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions); + return new DetectionRule(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions); } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java index c167e75089a..726fef8c91a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java @@ -83,7 +83,7 @@ public class Detector implements ToXContentObject, Writeable { public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name"); public static final ParseField USE_NULL_FIELD = new ParseField("use_null"); public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent"); - public static final ParseField DETECTOR_RULES_FIELD = new ParseField("detector_rules"); + public static final ParseField RULES_FIELD = new ParseField("rules"); public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly @@ -110,8 +110,8 @@ public class Detector implements ToXContentObject, Writeable { } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING); - parser.declareObjectArray(Builder::setDetectorRules, (p, c) -> - DetectionRule.PARSERS.get(parserType).apply(p, c).build(), DETECTOR_RULES_FIELD); + parser.declareObjectArray(Builder::setRules, (p, c) -> + DetectionRule.PARSERS.get(parserType).apply(p, c).build(), RULES_FIELD); parser.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX); } } @@ -329,7 +329,7 @@ public class Detector implements ToXContentObject, Writeable { private final String partitionFieldName; private final boolean useNull; private final ExcludeFrequent excludeFrequent; - private final List detectorRules; + private final List rules; private final int detectorIndex; public Detector(StreamInput in) throws IOException { @@ -341,7 +341,7 @@ public class Detector implements ToXContentObject, Writeable { partitionFieldName = in.readOptionalString(); useNull = in.readBoolean(); excludeFrequent = in.readBoolean() ? ExcludeFrequent.readFromStream(in) : null; - detectorRules = in.readList(DetectionRule::new); + rules = in.readList(DetectionRule::new); if (in.getVersion().onOrAfter(Version.V_5_5_0)) { detectorIndex = in.readInt(); } else { @@ -365,7 +365,7 @@ public class Detector implements ToXContentObject, Writeable { } else { out.writeBoolean(false); } - out.writeList(detectorRules); + out.writeList(rules); if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeInt(detectorIndex); } @@ -394,7 +394,7 @@ public class Detector implements ToXContentObject, Writeable { if (excludeFrequent != null) { builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent); } - builder.field(DETECTOR_RULES_FIELD.getPreferredName(), detectorRules); + builder.field(RULES_FIELD.getPreferredName(), rules); // negative means "unknown", which should only happen for a 5.4 job if (detectorIndex >= 0 // no point writing this to cluster state, as the indexes will get reassigned on reload anyway @@ -406,7 +406,7 @@ public class Detector implements ToXContentObject, Writeable { } private Detector(String detectorDescription, String function, String fieldName, String byFieldName, String overFieldName, - String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List detectorRules, + String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List rules, int detectorIndex) { this.function = function; this.fieldName = fieldName; @@ -415,7 +415,7 @@ public class Detector implements ToXContentObject, Writeable { this.partitionFieldName = partitionFieldName; this.useNull = useNull; this.excludeFrequent = excludeFrequent; - this.detectorRules = Collections.unmodifiableList(detectorRules); + this.rules = Collections.unmodifiableList(rules); this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this); this.detectorIndex = detectorIndex; } @@ -491,8 +491,8 @@ public class Detector implements ToXContentObject, Writeable { return excludeFrequent; } - public List getDetectorRules() { - return detectorRules; + public List getRules() { + return rules; } /** @@ -514,8 +514,8 @@ public class Detector implements ToXContentObject, Writeable { } public Set extractReferencedFilters() { - return detectorRules == null ? Collections.emptySet() - : detectorRules.stream().map(DetectionRule::extractReferencedFilters) + return rules == null ? Collections.emptySet() + : rules.stream().map(DetectionRule::extractReferencedFilters) .flatMap(Set::stream).collect(Collectors.toSet()); } @@ -556,15 +556,14 @@ public class Detector implements ToXContentObject, Writeable { Objects.equals(this.partitionFieldName, that.partitionFieldName) && Objects.equals(this.useNull, that.useNull) && Objects.equals(this.excludeFrequent, that.excludeFrequent) && - Objects.equals(this.detectorRules, that.detectorRules) && + Objects.equals(this.rules, that.rules) && this.detectorIndex == that.detectorIndex; } @Override public int hashCode() { - return Objects.hash(detectorDescription, function, fieldName, byFieldName, - overFieldName, partitionFieldName, useNull, excludeFrequent, - detectorRules, detectorIndex); + return Objects.hash(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, useNull, + excludeFrequent, rules, detectorIndex); } public static class Builder { @@ -587,7 +586,7 @@ public class Detector implements ToXContentObject, Writeable { private String partitionFieldName; private boolean useNull = false; private ExcludeFrequent excludeFrequent; - private List detectorRules = Collections.emptyList(); + private List rules = Collections.emptyList(); // negative means unknown, and is expected for v5.4 jobs private int detectorIndex = -1; @@ -603,8 +602,7 @@ public class Detector implements ToXContentObject, Writeable { partitionFieldName = detector.partitionFieldName; useNull = detector.useNull; excludeFrequent = detector.excludeFrequent; - detectorRules = new ArrayList<>(detector.detectorRules.size()); - detectorRules.addAll(detector.getDetectorRules()); + rules = new ArrayList<>(detector.getRules()); detectorIndex = detector.detectorIndex; } @@ -645,8 +643,8 @@ public class Detector implements ToXContentObject, Writeable { this.excludeFrequent = excludeFrequent; } - public void setDetectorRules(List detectorRules) { - this.detectorRules = detectorRules; + public void setRules(List rules) { + this.rules = rules; } public void setDetectorIndex(int detectorIndex) { @@ -704,12 +702,12 @@ public class Detector implements ToXContentObject, Writeable { } String function = this.function == null ? Detector.METRIC : this.function; - if (detectorRules.isEmpty() == false) { + if (rules.isEmpty() == false) { if (FUNCTIONS_WITHOUT_RULE_SUPPORT.contains(function)) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function); throw ExceptionsHelper.badRequestException(msg); } - for (DetectionRule rule : detectorRules) { + for (DetectionRule rule : rules) { checkScoping(rule); } } @@ -764,7 +762,7 @@ public class Detector implements ToXContentObject, Writeable { } return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, - useNull, excludeFrequent, detectorRules, detectorIndex); + useNull, excludeFrequent, rules, detectorIndex); } public List extractAnalysisFields() { @@ -802,7 +800,7 @@ public class Detector implements ToXContentObject, Writeable { String targetFieldName = rule.getTargetFieldName(); checkTargetFieldNameIsValid(extractAnalysisFields(), targetFieldName); List validOptions = getValidFieldNameOptions(rule); - for (RuleCondition condition : rule.getRuleConditions()) { + for (RuleCondition condition : rule.getConditions()) { if (!validOptions.contains(condition.getFieldName())) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, validOptions, condition.getFieldName()); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java index dc304149769..85096c49a34 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java @@ -315,7 +315,7 @@ public class JobUpdate implements Writeable, ToXContentObject { detectorbuilder.setDetectorDescription(dd.getDescription()); } if (dd.getRules() != null) { - detectorbuilder.setDetectorRules(dd.getRules()); + detectorbuilder.setRules(dd.getRules()); } ac.getDetectors().set(dd.getDetectorIndex(), detectorbuilder.build()); } @@ -435,13 +435,11 @@ public class JobUpdate implements Writeable, ToXContentObject { new ConstructingObjectParser<>("detector_update", a -> new DetectorUpdate((int) a[0], (String) a[1], (List) a[2])); - public static final ParseField RULES = new ParseField("rules"); - static { PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), Detector.DETECTOR_INDEX); PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION); - PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), - (parser, parseFieldMatcher) -> DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), RULES); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (parser, parseFieldMatcher) -> + DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), Detector.RULES_FIELD); } private int detectorIndex; @@ -495,7 +493,7 @@ public class JobUpdate implements Writeable, ToXContentObject { builder.field(Job.DESCRIPTION.getPreferredName(), description); } if (rules != null) { - builder.field(RULES.getPreferredName(), rules); + builder.field(Detector.RULES_FIELD.getPreferredName(), rules); } builder.endObject(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java index 20efbf01ac6..2a96cc45c4a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java @@ -14,8 +14,7 @@ import java.util.Locale; public enum RuleAction implements Writeable { FILTER_RESULTS, - SKIP_SAMPLING, - SKIP_SAMPLING_AND_FILTER_RESULTS; + SKIP_SAMPLING; /** * Case-insensitive from string method. diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java index b23ed163066..a950189931f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java @@ -26,11 +26,10 @@ import java.util.Map; import java.util.Objects; public class RuleCondition implements ToXContentObject, Writeable { - public static final ParseField CONDITION_TYPE_FIELD = new ParseField("condition_type"); + public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition"); public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name"); public static final ParseField FIELD_VALUE_FIELD = new ParseField("field_value"); - public static final ParseField VALUE_FILTER_FIELD = new ParseField("value_filter"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ConstructingObjectParser METADATA_PARSER = @@ -53,60 +52,60 @@ public class RuleCondition implements ToXContentObject, Writeable { return RuleConditionType.fromString(p.text()); } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, CONDITION_TYPE_FIELD, ValueType.STRING); + }, TYPE_FIELD, ValueType.STRING); parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_NAME_FIELD); parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_VALUE_FIELD); parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), Condition.PARSER, Condition.CONDITION_FIELD); - parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), VALUE_FILTER_FIELD); + parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), MlFilter.ID); } } - private final RuleConditionType conditionType; + private final RuleConditionType type; private final String fieldName; private final String fieldValue; private final Condition condition; - private final String valueFilter; + private final String filterId; public RuleCondition(StreamInput in) throws IOException { - conditionType = RuleConditionType.readFromStream(in); + type = RuleConditionType.readFromStream(in); condition = in.readOptionalWriteable(Condition::new); fieldName = in.readOptionalString(); fieldValue = in.readOptionalString(); - valueFilter = in.readOptionalString(); + filterId = in.readOptionalString(); } @Override public void writeTo(StreamOutput out) throws IOException { - conditionType.writeTo(out); + type.writeTo(out); out.writeOptionalWriteable(condition); out.writeOptionalString(fieldName); out.writeOptionalString(fieldValue); - out.writeOptionalString(valueFilter); + out.writeOptionalString(filterId); } - RuleCondition(RuleConditionType conditionType, String fieldName, String fieldValue, Condition condition, String valueFilter) { - this.conditionType = conditionType; + RuleCondition(RuleConditionType type, String fieldName, String fieldValue, Condition condition, String filterId) { + this.type = type; this.fieldName = fieldName; this.fieldValue = fieldValue; this.condition = condition; - this.valueFilter = valueFilter; + this.filterId = filterId; verifyFieldsBoundToType(this); verifyFieldValueRequiresFieldName(this); } public RuleCondition(RuleCondition ruleCondition) { - this.conditionType = ruleCondition.conditionType; + this.type = ruleCondition.type; this.fieldName = ruleCondition.fieldName; this.fieldValue = ruleCondition.fieldValue; this.condition = ruleCondition.condition; - this.valueFilter = ruleCondition.valueFilter; + this.filterId = ruleCondition.filterId; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CONDITION_TYPE_FIELD.getPreferredName(), conditionType); + builder.field(TYPE_FIELD.getPreferredName(), type); if (condition != null) { builder.field(Condition.CONDITION_FIELD.getPreferredName(), condition); } @@ -116,15 +115,15 @@ public class RuleCondition implements ToXContentObject, Writeable { if (fieldValue != null) { builder.field(FIELD_VALUE_FIELD.getPreferredName(), fieldValue); } - if (valueFilter != null) { - builder.field(VALUE_FILTER_FIELD.getPreferredName(), valueFilter); + if (filterId != null) { + builder.field(MlFilter.ID.getPreferredName(), filterId); } builder.endObject(); return builder; } - public RuleConditionType getConditionType() { - return conditionType; + public RuleConditionType getType() { + return type; } /** @@ -153,8 +152,8 @@ public class RuleCondition implements ToXContentObject, Writeable { * The unique identifier of a filter. Required when the rule type is * categorical. Should be null for all other types. */ - public String getValueFilter() { - return valueFilter; + public String getFilterId() { + return filterId; } @Override @@ -168,14 +167,14 @@ public class RuleCondition implements ToXContentObject, Writeable { } RuleCondition other = (RuleCondition) obj; - return Objects.equals(conditionType, other.conditionType) && Objects.equals(fieldName, other.fieldName) + return Objects.equals(type, other.type) && Objects.equals(fieldName, other.fieldName) && Objects.equals(fieldValue, other.fieldValue) && Objects.equals(condition, other.condition) - && Objects.equals(valueFilter, other.valueFilter); + && Objects.equals(filterId, other.filterId); } @Override public int hashCode() { - return Objects.hash(conditionType, fieldName, fieldValue, condition, valueFilter); + return Objects.hash(type, fieldName, fieldValue, condition, filterId); } public static RuleCondition createCategorical(String fieldName, String valueFilter) { @@ -195,7 +194,7 @@ public class RuleCondition implements ToXContentObject, Writeable { } private static void verifyFieldsBoundToType(RuleCondition ruleCondition) throws ElasticsearchParseException { - switch (ruleCondition.getConditionType()) { + switch (ruleCondition.getType()) { case CATEGORICAL: verifyCategorical(ruleCondition); break; @@ -215,7 +214,7 @@ public class RuleCondition implements ToXContentObject, Writeable { private static void verifyCategorical(RuleCondition ruleCondition) throws ElasticsearchParseException { checkCategoricalHasNoField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); checkCategoricalHasNoField(RuleCondition.FIELD_VALUE_FIELD.getPreferredName(), ruleCondition.getFieldValue()); - checkCategoricalHasField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter()); + checkCategoricalHasField(MlFilter.ID.getPreferredName(), ruleCondition.getFilterId()); } private static void checkCategoricalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { @@ -233,7 +232,7 @@ public class RuleCondition implements ToXContentObject, Writeable { } private static void verifyNumerical(RuleCondition ruleCondition) throws ElasticsearchParseException { - checkNumericalHasNoField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter()); + checkNumericalHasNoField(MlFilter.ID.getPreferredName(), ruleCondition.getFilterId()); checkNumericalHasField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); if (ruleCondition.getFieldName() != null && ruleCondition.getFieldValue() == null) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java index 0bfc184d7c7..65aeec387cf 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java @@ -59,6 +59,10 @@ public class GroupOrJobLookup { return new GroupOrJobResolver().expand(expression, allowNoJobs); } + public boolean isGroupOrJob(String id) { + return groupOrJobLookup.containsKey(id); + } + private class GroupOrJobResolver extends NameResolver { private GroupOrJobResolver() { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java new file mode 100644 index 00000000000..7b50fd349ff --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.calendars.Calendar; + +public class CalendarQueryBuilder { + + private PageParams pageParams = new PageParams(0, 10000); + private String jobId; + private boolean sort = false; + + /** + * Page the query result + * @param params The page parameters + * @return this + */ + public CalendarQueryBuilder pageParams(PageParams params) { + this.pageParams = params; + return this; + } + + /** + * Query only calendars used by this job + * @param jobId The job Id + * @return this + */ + public CalendarQueryBuilder jobId(String jobId) { + this.jobId = jobId; + return this; + } + + /** + * Sort results by calendar_id + * @param sort Sort if true + * @return this + */ + public CalendarQueryBuilder sort(boolean sort) { + this.sort = sort; + return this; + } + + public SearchSourceBuilder build() { + QueryBuilder qb; + if (jobId != null) { + qb = new BoolQueryBuilder() + .filter(new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE)) + .filter(new TermsQueryBuilder(Calendar.JOB_IDS.getPreferredName(), jobId)); + } else { + qb = new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE); + } + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(qb); + + if (sort) { + sourceBuilder.sort(Calendar.ID.getPreferredName()); + } + + sourceBuilder.from(pageParams.getFrom()).size(pageParams.getSize()); + + return sourceBuilder; + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index 108756836c2..66a90e3e969 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -18,6 +18,11 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; @@ -25,6 +30,9 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlock; @@ -38,6 +46,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -62,6 +71,7 @@ import org.elasticsearch.xpack.ml.action.GetCategoriesAction; import org.elasticsearch.xpack.ml.action.GetInfluencersAction; import org.elasticsearch.xpack.ml.action.GetRecordsAction; import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.MlFilter; @@ -87,13 +97,16 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.ClientHelper.clientWithOrigin; @@ -998,6 +1011,136 @@ public class JobProvider { , client::search); } + public void updateCalendar(String calendarId, Set jobIdsToAdd, Set jobIdsToRemove, + Consumer handler, Consumer errorHandler) { + + ActionListener getCalendarListener = ActionListener.wrap( + calendar -> { + Set currentJobs = new HashSet<>(calendar.getJobIds()); + currentJobs.addAll(jobIdsToAdd); + currentJobs.removeAll(jobIdsToRemove); + Calendar updatedCalendar = new Calendar(calendar.getId(), new ArrayList<>(currentJobs)); + + UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, updatedCalendar.documentId()); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + updateRequest.doc(updatedCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise calendar with id [" + updatedCalendar.getId() + "]", e); + } + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, updateRequest, + ActionListener.wrap( + response -> { + handler.accept(updatedCalendar); + }, + errorHandler) + , client::update); + + }, + errorHandler + ); + + calendar(calendarId, getCalendarListener); + } + + public void calendars(CalendarQueryBuilder queryBuilder, ActionListener> listener) { + SearchRequest searchRequest = client.prepareSearch(MlMetaIndex.INDEX_NAME) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(queryBuilder.build()).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + List calendars = new ArrayList<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + calendars.add(parseSearchHit(hit, Calendar.PARSER, listener::onFailure).build()); + } + + listener.onResponse(new QueryPage(calendars, response.getHits().getTotalHits(), + Calendar.RESULTS_FIELD)); + }, + listener::onFailure) + , client::search); + } + + public void removeJobFromCalendars(String jobId, ActionListener listener) { + + ActionListener updateCalandarsListener = ActionListener.wrap( + r -> { + if (r.hasFailures()) { + listener.onResponse(false); + } + listener.onResponse(true); + }, + listener::onFailure + ); + + ActionListener> getCalendarsListener = ActionListener.wrap( + r -> { + BulkRequestBuilder bulkUpdate = client.prepareBulk(); + bulkUpdate.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + r.results().stream() + .map(c -> { + Set ids = new HashSet<>(c.getJobIds()); + ids.remove(jobId); + return new Calendar(c.getId(), new ArrayList<>(ids)); + }).forEach(c -> { + UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, + c.documentId()); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + updateRequest.doc(c.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise calendar with id [" + c.getId() + "]", e); + } + bulkUpdate.add(updateRequest); + }); + + if (bulkUpdate.numberOfActions() > 0) { + executeAsyncWithOrigin(client, ML_ORIGIN, BulkAction.INSTANCE, bulkUpdate.request(), updateCalandarsListener); + } else { + listener.onResponse(true); + } + }, + listener::onFailure + ); + + CalendarQueryBuilder query = new CalendarQueryBuilder().jobId(jobId); + calendars(query, getCalendarsListener); + } + + public void calendar(String calendarId, ActionListener listener) { + GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId)); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getDocResponse) { + try { + if (getDocResponse.isExists()) { + BytesReference docSource = getDocResponse.getSourceAsBytesRef(); + + try (XContentParser parser = + XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource)) { + Calendar calendar = Calendar.PARSER.apply(parser, null).build(); + listener.onResponse(calendar); + } + } else { + this.onFailure(new ResourceNotFoundException("No calendar with id [" + calendarId + "]")); + } + } catch (Exception e) { + this.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, + client::get); + } + private void handleLatestModelSizeStats(String jobId, ModelSizeStats latestModelSizeStats, Consumer handler, Consumer errorHandler) { if (latestModelSizeStats != null) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java index 3d107d39ff1..c25635478dc 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java @@ -107,8 +107,8 @@ public class FieldConfigWriter { StringBuilder contents) throws IOException { List rules = new ArrayList<>(); - if (detector.getDetectorRules() != null) { - rules.addAll(detector.getDetectorRules()); + if (detector.getRules() != null) { + rules.addAll(detector.getRules()); } rules.addAll(specialEvents); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java index 2e59fb8c514..6bad2ea8609 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java @@ -35,5 +35,4 @@ public class RestDeleteCalendarAction extends BaseRestHandler { DeleteCalendarAction.Request request = new DeleteCalendarAction.Request(restRequest.param(Calendar.ID.getPreferredName())); return channel -> client.execute(DeleteCalendarAction.INSTANCE, request, new AcknowledgedRestListener<>(channel)); } - } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java new file mode 100644 index 00000000000..749987755b9 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.calendar; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction; +import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Collections; + +public class RestDeleteCalendarJobAction extends BaseRestHandler { + + public RestDeleteCalendarJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, + MachineLearning.BASE_PATH + "calendars/{" + Calendar.ID.getPreferredName() + "}/jobs/{" + + Job.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_delete_calendar_job_action"; + } + + @Override + protected BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String calendarId = restRequest.param(Calendar.ID.getPreferredName()); + String jobId = restRequest.param(Job.ID.getPreferredName()); + UpdateCalendarJobAction.Request request = + new UpdateCalendarJobAction.Request(calendarId, Collections.emptySet(), Collections.singleton(jobId)); + return channel -> client.execute(UpdateCalendarJobAction.INSTANCE, request, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java new file mode 100644 index 00000000000..adc9da8876c --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.calendar; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction; +import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Collections; + +public class RestPutCalendarJobAction extends BaseRestHandler { + + public RestPutCalendarJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, + MachineLearning.BASE_PATH + "calendars/{" + Calendar.ID.getPreferredName() + "}/jobs/{" + + Job.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_put_calendar_job_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String calendarId = restRequest.param(Calendar.ID.getPreferredName()); + String jobId = restRequest.param(Job.ID.getPreferredName()); + UpdateCalendarJobAction.Request putCalendarRequest = + new UpdateCalendarJobAction.Request(calendarId, Collections.singleton(jobId), Collections.emptySet()); + return channel -> client.execute(UpdateCalendarJobAction.INSTANCE, putCalendarRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/plugin/src/main/resources/monitoring-beats.json b/plugin/src/main/resources/monitoring-beats.json index 6ee5af734d8..3541ffd2bcd 100644 --- a/plugin/src/main/resources/monitoring-beats.json +++ b/plugin/src/main/resources/monitoring-beats.json @@ -19,6 +19,9 @@ "type": "date", "format": "date_time" }, + "interval_ms": { + "type": "long" + }, "type": { "type": "keyword" }, diff --git a/plugin/src/main/resources/monitoring-kibana.json b/plugin/src/main/resources/monitoring-kibana.json index e1f057c7d0d..51e9f7cdd19 100644 --- a/plugin/src/main/resources/monitoring-kibana.json +++ b/plugin/src/main/resources/monitoring-kibana.json @@ -19,6 +19,9 @@ "type": "date", "format": "date_time" }, + "interval_ms": { + "type": "long" + }, "type": { "type": "keyword" }, diff --git a/plugin/src/main/resources/monitoring-logstash.json b/plugin/src/main/resources/monitoring-logstash.json index 90ee37cc6e0..296fd59ed15 100644 --- a/plugin/src/main/resources/monitoring-logstash.json +++ b/plugin/src/main/resources/monitoring-logstash.json @@ -19,6 +19,9 @@ "type": "date", "format": "date_time" }, + "interval_ms": { + "type": "long" + }, "type": { "type": "keyword" }, diff --git a/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java b/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java index ce3b2f1c883..995ce8e7f34 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java @@ -17,10 +17,12 @@ import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingRequestBuilder; import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingResponse; import org.elasticsearch.xpack.security.authc.ldap.LdapRealm; @@ -42,6 +44,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -191,6 +194,13 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase super.deleteSecurityIndex(); } + @Override + public Set excludeTemplates() { + Set templates = Sets.newHashSet(super.excludeTemplates()); + templates.add(SecurityLifecycleService.SECURITY_TEMPLATE_NAME); // don't remove the security index template + return templates; + } + private List getRoleMappingContent(Function contentFunction) { return getRoleMappingContent(contentFunction, AbstractAdLdapRealmTestCase.roleMappings); } diff --git a/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index 96999294ce3..14f8d61c520 100644 --- a/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -15,6 +15,8 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.client.SecurityClient; import org.elasticsearch.xpack.security.user.ElasticUser; @@ -26,6 +28,7 @@ import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.Set; /** * Test case with method to handle the starting and stopping the stores for native users and roles @@ -59,6 +62,13 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase { .build(); } + @Override + public Set excludeTemplates() { + Set templates = Sets.newHashSet(super.excludeTemplates()); + templates.add(SecurityLifecycleService.SECURITY_TEMPLATE_NAME); // don't remove the security index template + return templates; + } + private SecureString reservedPassword = SecuritySettingsSource.TEST_PASSWORD_SECURE_STRING; protected SecureString getReservedPassword() { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java new file mode 100644 index 00000000000..231e264ef35 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.ml.calendars.CalendarTests; + +public class PutCalendarActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected PutCalendarAction.Response createTestInstance() { + return new PutCalendarAction.Response(CalendarTests.testInstance()); + } + + @Override + protected PutCalendarAction.Response createBlankInstance() { + return new PutCalendarAction.Response(); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java new file mode 100644 index 00000000000..c56483441b1 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.util.HashSet; +import java.util.Set; + +public class UpdateCalendarJobActionResquestTests extends AbstractStreamableTestCase { + + @Override + protected UpdateCalendarJobAction.Request createTestInstance() { + int addSize = randomIntBetween(0, 2); + Set toAdd = new HashSet<>(); + for (int i=0; i toRemove = new HashSet<>(); + for (int i=0; i { - @Override - protected Calendar createTestInstance() { + public static Calendar testInstance() { int size = randomInt(10); List items = new ArrayList<>(size); for (int i = 0; i < size; i++) { @@ -28,6 +27,11 @@ public class CalendarTests extends AbstractSerializingTestCase { return new Calendar(randomAlphaOfLengthBetween(1, 20), items); } + @Override + protected Calendar createTestInstance() { + return testInstance(); + } + @Override protected Writeable.Reader instanceReader() { return Calendar::new; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java index 148b6200d78..6ecd8496ac8 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; public class SpecialEventTests extends AbstractSerializingTestCase { @@ -60,14 +61,14 @@ public class SpecialEventTests extends AbstractSerializingTestCase DetectionRule rule = event.toDetectionRule(TimeValue.timeValueSeconds(bucketSpanSecs)); assertEquals(Connective.AND, rule.getConditionsConnective()); - assertEquals(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS, rule.getRuleAction()); + assertEquals(rule.getActions(), EnumSet.of(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING)); assertNull(rule.getTargetFieldName()); assertNull(rule.getTargetFieldValue()); - List conditions = rule.getRuleConditions(); + List conditions = rule.getConditions(); assertEquals(2, conditions.size()); - assertEquals(RuleConditionType.TIME, conditions.get(0).getConditionType()); - assertEquals(RuleConditionType.TIME, conditions.get(1).getConditionType()); + assertEquals(RuleConditionType.TIME, conditions.get(0).getType()); + assertEquals(RuleConditionType.TIME, conditions.get(1).getType()); assertEquals(Operator.GTE, conditions.get(0).getCondition().getOperator()); assertEquals(Operator.LT, conditions.get(1).getCondition().getOperator()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java index d22b34e5369..0933f8780f4 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java @@ -61,7 +61,7 @@ public class DetectionRulesIT extends MlNativeAutodetectIntegTestCase { DetectionRule rule = new DetectionRule.Builder(Arrays.asList(condition1, condition2, condition3)).build(); Detector.Builder detector = new Detector.Builder("max", "value"); - detector.setDetectorRules(Arrays.asList(rule)); + detector.setRules(Arrays.asList(rule)); detector.setByFieldName("by_field"); AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java index 96f932c86f0..850ea920857 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java @@ -25,6 +25,8 @@ import org.elasticsearch.xpack.XPackSingleNodeTestCase; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MlMetaIndex; import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.ml.job.config.Connective; @@ -36,6 +38,7 @@ import org.elasticsearch.xpack.ml.job.config.MlFilter; import org.elasticsearch.xpack.ml.job.config.RuleAction; import org.elasticsearch.xpack.ml.job.config.RuleCondition; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; @@ -56,9 +59,21 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.hamcrest.core.Is.is; public class JobProviderIT extends XPackSingleNodeTestCase { @@ -97,6 +112,160 @@ public class JobProviderIT extends XPackSingleNodeTestCase { }); } + public void testGetCalandarByJobId() throws Exception { + List calendars = new ArrayList<>(); + calendars.add(new Calendar("empty calendar", Collections.emptyList())); + calendars.add(new Calendar("foo calendar", Collections.singletonList("foo"))); + calendars.add(new Calendar("foo bar calendar", Arrays.asList("foo", "bar"))); + calendars.add(new Calendar("cat calendar", Collections.singletonList("cat"))); + calendars.add(new Calendar("cat foo calendar", Arrays.asList("cat", "foo"))); + indexCalendars(calendars); + + List queryResult = getCalendars("ted"); + assertThat(queryResult, is(empty())); + + queryResult = getCalendars("foo"); + assertThat(queryResult, hasSize(3)); + Long matchedCount = queryResult.stream().filter( + c -> c.getId().equals("foo calendar") || c.getId().equals("foo bar calendar") || c.getId().equals("cat foo calendar")) + .collect(Collectors.counting()); + assertEquals(new Long(3), matchedCount); + + queryResult = getCalendars("bar"); + assertThat(queryResult, hasSize(1)); + assertEquals("foo bar calendar", queryResult.get(0).getId()); + } + + public void testUpdateCalendar() throws Exception { + String calendarId = "empty calendar"; + Calendar emptyCal = new Calendar(calendarId, Collections.emptyList()); + indexCalendars(Collections.singletonList(emptyCal)); + + Set addedIds = new HashSet<>(); + addedIds.add("foo"); + addedIds.add("bar"); + updateCalendar(calendarId, addedIds, Collections.emptySet()); + + Calendar updated = getCalendar(calendarId); + assertEquals(calendarId, updated.getId()); + assertEquals(addedIds, new HashSet<>(updated.getJobIds())); + + Set removedIds = new HashSet<>(); + removedIds.add("foo"); + updateCalendar(calendarId, Collections.emptySet(), removedIds); + + updated = getCalendar(calendarId); + assertEquals(calendarId, updated.getId()); + assertEquals(1, updated.getJobIds().size()); + assertEquals("bar", updated.getJobIds().get(0)); + } + + public void testRemoveJobFromCalendar() throws Exception { + List calendars = new ArrayList<>(); + calendars.add(new Calendar("empty calendar", Collections.emptyList())); + calendars.add(new Calendar("foo calendar", Collections.singletonList("foo"))); + calendars.add(new Calendar("foo bar calendar", Arrays.asList("foo", "bar"))); + calendars.add(new Calendar("cat calendar", Collections.singletonList("cat"))); + calendars.add(new Calendar("cat foo calendar", Arrays.asList("cat", "foo"))); + indexCalendars(calendars); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + jobProvider.removeJobFromCalendars("bar", ActionListener.wrap( + r -> latch.countDown(), + exceptionHolder::set)); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + List updatedCalendars = getCalendars(null); + assertEquals(5, updatedCalendars.size()); + for (Calendar cal: updatedCalendars) { + assertThat("bar", not(isIn(cal.getJobIds()))); + } + + Calendar catFoo = getCalendar("cat foo calendar"); + assertThat(catFoo.getJobIds(), contains("cat", "foo")); + + CountDownLatch latch2 = new CountDownLatch(1); + exceptionHolder = new AtomicReference<>(); + jobProvider.removeJobFromCalendars("cat", ActionListener.wrap( + r -> latch2.countDown(), + exceptionHolder::set)); + + latch2.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + updatedCalendars = getCalendars(null); + assertEquals(5, updatedCalendars.size()); + for (Calendar cal: updatedCalendars) { + assertThat("bar", not(isIn(cal.getJobIds()))); + assertThat("cat", not(isIn(cal.getJobIds()))); + } + } + + private List getCalendars(String jobId) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference> result = new AtomicReference<>(); + + CalendarQueryBuilder query = new CalendarQueryBuilder(); + + if (jobId != null) { + query.jobId(jobId); + } + jobProvider.calendars(query, ActionListener.wrap( + r -> { + latch.countDown(); + result.set(r); + }, + exceptionHolder::set)); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + return result.get().results(); + } + + private void updateCalendar(String calendarId, Set idsToAdd, Set idsToRemove) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + jobProvider.updateCalendar(calendarId, idsToAdd, idsToRemove, + r -> latch.countDown(), + exceptionHolder::set); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + client().admin().indices().prepareRefresh(MlMetaIndex.INDEX_NAME).get(); + } + + private Calendar getCalendar(String calendarId) throws Exception { + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference calendarHolder = new AtomicReference<>(); + jobProvider.calendar(calendarId, ActionListener.wrap( + c -> { latch.countDown(); calendarHolder.set(c); }, + exceptionHolder::set) + ); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + return calendarHolder.get(); + } + public void testSpecialEvents() throws Exception { List events = new ArrayList<>(); events.add(new SpecialEvent("A_and_B_downtime", "downtime", createZonedDateTime(1000L), createZonedDateTime(2000L), @@ -262,10 +431,10 @@ public class JobProviderIT extends XPackSingleNodeTestCase { } DetectionRule.Builder rule = new DetectionRule.Builder(conditions) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setConditionsConnective(Connective.OR); - detector.setDetectorRules(Collections.singletonList(rule.build())); + detector.setRules(Collections.singletonList(rule.build())); } return new AnalysisConfig.Builder(Collections.singletonList(detector.build())); @@ -321,7 +490,8 @@ public class JobProviderIT extends XPackSingleNodeTestCase { for (MlFilter filter : filters) { IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - indexRequest.source(filter.toXContent(builder, ToXContent.EMPTY_PARAMS)); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + indexRequest.source(filter.toXContent(builder, params)); bulkRequest.add(indexRequest); } } @@ -341,7 +511,21 @@ public class JobProviderIT extends XPackSingleNodeTestCase { private void indexQuantiles(Quantiles quantiles) { JobResultsPersister persister = new JobResultsPersister(nodeSettings(), client()); persister.persistQuantiles(quantiles); + } + private void indexCalendars(List calendars) throws IOException { + BulkRequestBuilder bulkRequest = client().prepareBulk(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + for (Calendar calendar: calendars) { + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId()); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + indexRequest.source(calendar.toXContent(builder, params)); + bulkRequest.add(indexRequest); + } + } + bulkRequest.execute().actionGet(); } private ZonedDateTime createZonedDateTime(long epochMs) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java index 5ebdc4f86ce..16c9242f9d7 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java @@ -461,9 +461,9 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase ruleConditions = new ArrayList<>(size); for (int i = 0; i < size; i++) { // no need for random condition (it is already tested) ruleConditions.addAll(createRule(Double.toString(randomDouble()))); } - return new DetectionRule.Builder(ruleConditions) - .setRuleAction(ruleAction) - .setTargetFieldName(targetFieldName) - .setTargetFieldValue(targetFieldValue) - .setConditionsConnective(connective) - .build(); + DetectionRule.Builder builder = new DetectionRule.Builder(ruleConditions); + + if (randomBoolean()) { + EnumSet actions = EnumSet.noneOf(RuleAction.class); + int actionsCount = randomIntBetween(1, RuleAction.values().length); + for (int i = 0; i < actionsCount; ++i) { + actions.add(randomFrom(RuleAction.values())); + } + builder.setActions(actions); + } + + if (randomBoolean()) { + builder.setConditionsConnective(randomFrom(Connective.values())); + } + + if (randomBoolean()) { + builder.setTargetFieldName(randomAlphaOfLengthBetween(1, 20)); + builder.setTargetFieldValue(randomAlphaOfLengthBetween(1, 20)); + } + + return builder.build(); } @Override @@ -125,16 +133,16 @@ public class DetectionRuleTests extends AbstractSerializingTestCase ruleConditions = instance.getRuleConditions(); - RuleAction ruleAction = instance.getRuleAction(); + List conditions = instance.getConditions(); + EnumSet actions = instance.getActions(); String targetFieldName = instance.getTargetFieldName(); String targetFieldValue = instance.getTargetFieldValue(); Connective connective = instance.getConditionsConnective(); switch (between(0, 3)) { case 0: - ruleConditions = new ArrayList<>(ruleConditions); - ruleConditions.addAll(createRule(Double.toString(randomDouble()))); + conditions = new ArrayList<>(conditions); + conditions.addAll(createRule(Double.toString(randomDouble()))); break; case 1: targetFieldName = randomAlphaOfLengthBetween(5, 10); @@ -156,7 +164,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase { Condition condition = new Condition(Operator.GT, "5"); DetectionRule rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field") + .setActions(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field") .setTargetFieldValue("targetValue") .setConditionsConnective(Connective.AND) .build(); - builder.setDetectorRules(Collections.singletonList(rule)); + builder.setRules(Collections.singletonList(rule)); detector2 = builder.build(); assertFalse(detector1.equals(detector2)); } @@ -84,22 +84,22 @@ public class DetectorTests extends AbstractSerializingTestCase { Condition condition = new Condition(Operator.GT, "5"); DetectionRule rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setTargetFieldName("over_field") .setTargetFieldValue("targetValue") .setConditionsConnective(Connective.AND) .build(); - builder.setDetectorRules(Collections.singletonList(rule)); + builder.setRules(Collections.singletonList(rule)); builder.setByFieldName(null); detector = builder.build(); assertEquals(Collections.singletonList("over_field"), detector.extractAnalysisFields()); builder = new Detector.Builder(detector); rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setConditionsConnective(Connective.AND) .build(); - builder.setDetectorRules(Collections.singletonList(rule)); + builder.setRules(Collections.singletonList(rule)); builder.setOverFieldName(null); detector = builder.build(); assertTrue(detector.extractAnalysisFields().isEmpty()); @@ -107,7 +107,7 @@ public class DetectorTests extends AbstractSerializingTestCase { public void testExtractReferencedLists() { Detector.Builder builder = createDetector(); - builder.setDetectorRules(Arrays.asList( + builder.setRules(Arrays.asList( new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list1"))).build(), new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list2"))).build())); @@ -140,12 +140,12 @@ public class DetectorTests extends AbstractSerializingTestCase { Condition condition = new Condition(Operator.GT, "5"); DetectionRule rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by_field", "val", condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setTargetFieldName("over_field") .setTargetFieldValue("targetValue") .setConditionsConnective(Connective.AND) .build(); - detector.setDetectorRules(Collections.singletonList(rule)); + detector.setRules(Collections.singletonList(rule)); return detector; } @@ -176,15 +176,15 @@ public class DetectorTests extends AbstractSerializingTestCase { } if (randomBoolean()) { int size = randomInt(10); - List detectorRules = new ArrayList<>(size); + List rules = new ArrayList<>(size); for (int i = 0; i < size; i++) { // no need for random DetectionRule (it is already tested) Condition condition = new Condition(Operator.GT, "5"); - detectorRules.add(new DetectionRule.Builder( + rules.add(new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) .setTargetFieldName(fieldName).build()); } - detector.setDetectorRules(detectorRules); + detector.setRules(rules); } if (randomBoolean()) { detector.setUseNull(randomBoolean()); @@ -508,14 +508,14 @@ public class DetectorTests extends AbstractSerializingTestCase { } } - public void testVerify_GivenInvalidDetectionRuleTargetFieldName() { + public void testVerify_GivenInvalidRuleTargetFieldName() { Detector.Builder detector = new Detector.Builder("mean", "metricVale"); detector.setByFieldName("metricName"); detector.setPartitionFieldName("instance"); RuleCondition ruleCondition = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null); DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instancE").build(); - detector.setDetectorRules(Collections.singletonList(rule)); + detector.setRules(Collections.singletonList(rule)); ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); @@ -524,14 +524,14 @@ public class DetectorTests extends AbstractSerializingTestCase { e.getMessage()); } - public void testVerify_GivenValidDetectionRule() { + public void testVerify_GivenValidRule() { Detector.Builder detector = new Detector.Builder("mean", "metricVale"); detector.setByFieldName("metricName"); detector.setPartitionFieldName("instance"); RuleCondition ruleCondition = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null); DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instance").build(); - detector.setDetectorRules(Collections.singletonList(rule)); + detector.setRules(Collections.singletonList(rule)); detector.build(); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java index fa03921f282..7ce8e74ef23 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java @@ -165,7 +165,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase { updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); assertEquals(detectorUpdate.getRules(), - updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorRules()); + updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getRules()); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java index f46bf5a2482..b371471227a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java @@ -11,14 +11,12 @@ import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; -public class -RuleActionTests extends ESTestCase { +public class RuleActionTests extends ESTestCase { public void testForString() { assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("filter_results")); assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("FILTER_RESULTS")); assertEquals(RuleAction.SKIP_SAMPLING, RuleAction.fromString("SKip_sampLing")); - assertEquals(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS, RuleAction.fromString("skip_sampling_and_filter_results")); } public void testToString() { @@ -34,9 +32,9 @@ RuleActionTests extends ESTestCase { } try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(2); + out.writeVInt(1); try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS)); + assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING)); } } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java index 0a38319e169..84e8e2face9 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java @@ -52,7 +52,7 @@ public class RuleConditionTests extends AbstractSerializingTestCase new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, null)); - assertEquals("Invalid detector rule: a categorical rule_condition requires value_filter to be set", e.getMessage()); + assertEquals("Invalid detector rule: a categorical rule_condition requires filter_id to be set", e.getMessage()); } - public void testVerify_GivenNumericalActualWithValueFilter() { + public void testVerify_GivenNumericalActualWithFilterId() { ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); } public void testVerify_GivenNumericalActualWithoutCondition() { @@ -146,10 +146,10 @@ public class RuleConditionTests extends AbstractSerializingTestCase new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); } public void testVerify_GivenNumericalTypicalWithoutCondition() { @@ -158,10 +158,10 @@ public class RuleConditionTests extends AbstractSerializingTestCase new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); } public void testVerify_GivenNumericalDiffAbsWithoutCondition() { @@ -220,12 +220,12 @@ public class RuleConditionTests extends AbstractSerializingTestCase jobs = new ArrayList<>(); + jobs.add(mockJob("foo-1", Arrays.asList("foo-group", "ones"))); + jobs.add(mockJob("foo-2", Arrays.asList("foo-group", "twos"))); + jobs.add(mockJob("bar-1", Arrays.asList("bar-group", "ones"))); + jobs.add(mockJob("nogroup", Collections.emptyList())); + GroupOrJobLookup groupOrJobLookup = new GroupOrJobLookup(jobs); + + assertTrue(groupOrJobLookup.isGroupOrJob("foo-1")); + assertTrue(groupOrJobLookup.isGroupOrJob("twos")); + assertTrue(groupOrJobLookup.isGroupOrJob("nogroup")); + assertFalse(groupOrJobLookup.isGroupOrJob("missing")); + } + private static Job mockJob(String jobId, List groups) { Job job = mock(Job.class); when(job.getId()).thenReturn(jobId); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java index 0d9b4387de6..2de995d5ffb 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.ml.job.process.autodetect.writer; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.config.Condition; import org.elasticsearch.xpack.ml.job.config.Connective; import org.elasticsearch.xpack.ml.job.config.DetectionRule; @@ -22,13 +21,8 @@ import org.mockito.InOrder; import org.mockito.Mockito; import java.io.IOException; -import java.time.Instant; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Optional; import java.util.stream.IntStream; @@ -202,11 +196,11 @@ public class ControlMsgToProcessWriterTests extends ESTestCase { inOrder.verify(lengthEncodedWriter).writeNumFields(4); inOrder.verify(lengthEncodedWriter, times(3)).writeField(""); inOrder.verify(lengthEncodedWriter).writeField("u[detectorRules]\ndetectorIndex=2\n" + - "rulesJson=[{\"rule_action\":\"filter_results\",\"conditions_connective\":\"and\",\"rule_conditions\":" + - "[{\"condition_type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," + + "rulesJson=[{\"actions\":[\"filter_results\"],\"conditions_connective\":\"and\",\"conditions\":" + + "[{\"type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," + "\"target_field_name\":\"targetField1\",\"target_field_value\":\"targetValue\"}," + - "{\"rule_action\":\"filter_results\",\"conditions_connective\":\"and\",\"rule_conditions\":[" + - "{\"condition_type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," + + "{\"actions\":[\"filter_results\"],\"conditions_connective\":\"and\",\"conditions\":[" + + "{\"type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," + "\"target_field_name\":\"targetField2\",\"target_field_value\":\"targetValue\"}]"); verifyNoMoreInteractions(lengthEncodedWriter); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java index 3a56d59e633..7d83f2589b8 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java @@ -26,7 +26,6 @@ import org.mockito.ArgumentCaptor; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.StringReader; import java.nio.charset.StandardCharsets; @@ -36,7 +35,6 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Date; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; @@ -195,7 +193,7 @@ public class FieldConfigWriterTests extends ESTestCase { RuleCondition ruleCondition = RuleCondition.createNumerical (RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricValue", new Condition(Operator.LT, "5")); DetectionRule rule = new DetectionRule.Builder(Arrays.asList(ruleCondition)).setTargetFieldName("instance").build(); - detector.setDetectorRules(Arrays.asList(rule)); + detector.setRules(Arrays.asList(rule)); AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Arrays.asList(detector.build())); analysisConfig = builder.build(); @@ -251,12 +249,12 @@ public class FieldConfigWriterTests extends ESTestCase { createFieldConfigWriter().write(); verify(writer).write("detector.0.clause = count\n" + - "detector.0.rules = [{\"rule_action\":\"skip_sampling_and_filter_results\",\"conditions_connective\":\"and\"," + - "\"rule_conditions\":[{\"condition_type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1511395200\"}}," + - "{\"condition_type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515369600\"}}]}," + - "{\"rule_action\":\"skip_sampling_and_filter_results\",\"conditions_connective\":\"and\"," + - "\"rule_conditions\":[{\"condition_type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1519603200\"}}," + - "{\"condition_type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1519862400\"}}]}]" + + "detector.0.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," + + "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1511395200\"}}," + + "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515369600\"}}]}," + + "{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," + + "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1519603200\"}}," + + "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1519862400\"}}]}]" + "\n"); verifyNoMoreInteractions(writer); diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json new file mode 100644 index 00000000000..43dc1b94789 --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json @@ -0,0 +1,22 @@ +{ + "xpack.ml.delete_calendar_job": { + "methods": [ "DELETE" ], + "url": { + "path": "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}", + "paths": [ "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}" ], + "parts": { + "calendar_id": { + "type" : "string", + "required" : true, + "description" : "The ID of the calendar to modify" + }, + "job_id": { + "type": "string", + "required": true, + "description": "The ID of the job to remove from the calendar" + } + } + }, + "body": null + } +} diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json new file mode 100644 index 00000000000..2abf870058c --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json @@ -0,0 +1,22 @@ +{ + "xpack.ml.put_calendar_job": { + "methods": [ "PUT" ], + "url": { + "path": "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}", + "paths": [ "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}" ], + "parts": { + "calendar_id": { + "type": "string", + "required": true, + "description": "The ID of the calendar to modify" + }, + "job_id": { + "type": "string", + "required": true, + "description": "The ID of the job to add to the calendar" + } + } + }, + "body": null + } +} \ No newline at end of file diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml index 216cb521b5d..f39d88b6f2f 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml @@ -1,25 +1,52 @@ --- "Test calendar CRUD": + - do: + xpack.ml.put_job: + job_id: cal-job + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "cal-job" } + + - do: + xpack.ml.put_job: + job_id: cal-job2 + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "cal-job2" } + + - do: xpack.ml.put_calendar: calendar_id: "advent" body: > { - "job_ids": ["abc", "xyz"] + "job_ids": ["cal-job", "cal-job2"] } - match: { calendar_id: advent } - - match: { job_ids.0: abc } - - match: { job_ids.1: xyz } + - match: { job_ids.0: cal-job } + - match: { job_ids.1: cal-job2 } - do: xpack.ml.get_calendars: calendar_id: "advent" - - match: { count: 1 } + - match: { count: 1 } - match: calendars.0: calendar_id: "advent" - job_ids: ["abc", "xyz"] + job_ids: ["cal-job", "cal-job2"] - is_false: type - do: @@ -27,7 +54,7 @@ calendar_id: "Dogs of the Year" body: > { - "job_ids": ["abc2"] + "job_ids": ["cal-job"] } - do: @@ -51,6 +78,15 @@ xpack.ml.get_calendars: calendar_id: "Dogs of the Year" + - do: + catch: missing + xpack.ml.put_calendar: + calendar_id: "new cal with unknown job" + body: > + { + "job_ids": ["cal-job", "unknown-job"] + } + --- "Test PageParams": - do: @@ -66,14 +102,16 @@ - do: xpack.ml.get_calendars: from: 2 - - match: { count: 1 } + - match: { count: 3 } + - length: { calendars: 1} - match: { calendars.0.calendar_id: Calendar3 } - do: xpack.ml.get_calendars: from: 1 size: 1 - - match: { count: 1 } + - match: { count: 3 } + - length: { calendars: 1} - match: { calendars.0.calendar_id: Calendar2 } --- @@ -90,10 +128,6 @@ - do: xpack.ml.put_calendar: calendar_id: "Mayan" - body: > - { - "job_ids": ["apocalypse"] - } - do: catch: /version_conflict_engine_exception/ @@ -106,3 +140,97 @@ catch: bad_request xpack.ml.put_calendar: calendar_id: "_all" + +--- +"Test deleted job is removed from calendar": + + - do: + xpack.ml.put_job: + job_id: cal-crud-test-delete + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "cal-crud-test-delete" } + + - do: + xpack.ml.put_calendar: + calendar_id: "delete-test" + body: > + { + "job_ids": ["cal-crud-test-delete"] + } + + - do: + xpack.ml.delete_job: + job_id: cal-crud-test-delete + - match: { acknowledged: true } + + - do: + xpack.ml.get_calendars: + calendar_id: "delete-test" + - match: { count: 1 } + - match: { calendars.0.job_ids: [] } + +--- +"Test update calendar": + + - do: + xpack.ml.put_calendar: + calendar_id: "Wildlife" + + - do: + xpack.ml.put_job: + job_id: tiger + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "tiger" } + + - do: + xpack.ml.put_calendar_job: + calendar_id: "Wildlife" + job_id: "tiger" + - match: { calendar_id: "Wildlife" } + - match: { job_ids.0: "tiger" } + + - do: + xpack.ml.get_calendars: + calendar_id: "Wildlife" + - match: { count: 1 } + - match: { calendars.0.calendar_id: "Wildlife" } + - length: { calendars.0.job_ids: 1 } + - match: { calendars.0.job_ids.0: "tiger" } + + - do: + xpack.ml.delete_calendar_job: + calendar_id: "Wildlife" + job_id: "tiger" + + - do: + xpack.ml.get_calendars: + calendar_id: "Wildlife" + - match: { count: 1 } + - match: { calendars.0.calendar_id: "Wildlife" } + - length: { calendars.0.job_ids: 0 } + + - do: + catch: missing + xpack.ml.put_calendar_job: + calendar_id: "Wildlife" + job_id: "missing job" + + - do: + catch: missing + xpack.ml.delete_calendar_job: + calendar_id: "Wildlife" + job_id: "missing job" diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml index c83c7ea2785..28d215d8cd3 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml @@ -152,12 +152,12 @@ setup: "analysis_config" : { "bucket_span": "3600s", "detectors" :[{"function":"mean","field_name":"airline", - "detector_rules": [ + "rules": [ { - "rule_conditions": [ + "conditions": [ { - "condition_type": "categorical", - "value_filter": "filter-foo" + "type": "categorical", + "filter_id": "filter-foo" } ] } diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index 4b8ce54139a..d86394291cb 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -301,8 +301,8 @@ "groups": ["group-1", "group-2"], "description":"Post update description", "detectors": [{"detector_index": 0, "rules": {"target_field_name": "airline", - "rule_conditions": [ { "condition_type": "numerical_actual", - "condition": {"operator": "gt", "value": "10" } } ] } }, + "conditions": [ { "type": "numerical_actual", + "condition": {"operator": "gt", "value": "10" } } ] } }, {"detector_index": 1, "description": "updated description"}], "model_plot_config": { "enabled": false, @@ -327,7 +327,7 @@ - match: { model_plot_config.terms: "foobar" } - match: { analysis_limits.model_memory_limit: "20mb" } - match: { analysis_config.categorization_filters: ["cat3.*"] } - - match: { analysis_config.detectors.0.detector_rules.0.target_field_name: "airline" } + - match: { analysis_config.detectors.0.rules.0.target_field_name: "airline" } - match: { analysis_config.detectors.0.detector_index: 0 } - match: { analysis_config.detectors.1.detector_description: "updated description" } - match: { analysis_config.detectors.1.detector_index: 1 } diff --git a/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java b/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java index 9339c8ce20e..cdaa94c01bc 100644 --- a/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java +++ b/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexNotFoundException; @@ -141,7 +142,7 @@ public class SecurityTribeTests extends NativeRealmIntegTestCase { public void tearDownTribeNodeAndWipeCluster() throws Exception { if (cluster2 != null) { try { - cluster2.wipe(Collections.emptySet()); + cluster2.wipe(Collections.singleton(SecurityLifecycleService.SECURITY_TEMPLATE_NAME)); try { // this is a hack to clean up the .security index since only the XPackSecurity user or superusers can delete it final Client cluster2Client = cluster2.client().filterWithHeader(Collections.singletonMap("Authorization",