From 8ba08ccea9c0b0f128d0d3bb585e6b2830c78b86 Mon Sep 17 00:00:00 2001 From: Yannick Welsch Date: Mon, 18 Dec 2017 18:13:54 +0100 Subject: [PATCH 1/5] [TEST] Create checkout directory on Windows before trimming path The getShortPathName method can only be used on a directory that actually exists, otherwise it will fail with a cryptic message. Original commit: elastic/x-pack-elasticsearch@44552dcfc8be399b147eb34606dc22145e921824 --- plugin/bwc-snapshot-dummy-projects/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugin/bwc-snapshot-dummy-projects/build.gradle b/plugin/bwc-snapshot-dummy-projects/build.gradle index ba676471d62..c664cb173ad 100644 --- a/plugin/bwc-snapshot-dummy-projects/build.gradle +++ b/plugin/bwc-snapshot-dummy-projects/build.gradle @@ -33,6 +33,7 @@ subprojects { */ Object esCheckoutPath = """${-> if (Os.isFamily(Os.FAMILY_WINDOWS)) { + esCheckoutDir.mkdirs() NodeInfo.getShortPathName(esCheckoutDir.toString()) } else { esCheckoutDir.toString() @@ -41,6 +42,7 @@ subprojects { File xpackCheckoutDir = file("${esCheckoutDir}-extra/x-pack-elasticsearch") Object xpackCheckoutPath = """${-> if (Os.isFamily(Os.FAMILY_WINDOWS)) { + xpackCheckoutDir.mkdirs() NodeInfo.getShortPathName(xpackCheckoutDir.toString()) } else { xpackCheckoutDir.toString() From 3efd35cadf6e5721b1747aae43e6c262378e8f7a Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Tue, 19 Dec 2017 09:14:10 +0100 Subject: [PATCH 2/5] [Monitoring] Add missing mapping for interval_ms (elastic/x-pack-elasticsearch#3339) # 2650 added the mapping for the interval_ms field in the Elasticsearch template but not for Kibana,Logstash and Beats templates. Original commit: elastic/x-pack-elasticsearch@44fb501bb32621d1aa466beee45d6a6c0394a329 --- plugin/src/main/resources/monitoring-beats.json | 3 +++ plugin/src/main/resources/monitoring-kibana.json | 3 +++ plugin/src/main/resources/monitoring-logstash.json | 3 +++ 3 files changed, 9 insertions(+) diff --git a/plugin/src/main/resources/monitoring-beats.json b/plugin/src/main/resources/monitoring-beats.json index 6ee5af734d8..3541ffd2bcd 100644 --- a/plugin/src/main/resources/monitoring-beats.json +++ b/plugin/src/main/resources/monitoring-beats.json @@ -19,6 +19,9 @@ "type": "date", "format": "date_time" }, + "interval_ms": { + "type": "long" + }, "type": { "type": "keyword" }, diff --git a/plugin/src/main/resources/monitoring-kibana.json b/plugin/src/main/resources/monitoring-kibana.json index e1f057c7d0d..51e9f7cdd19 100644 --- a/plugin/src/main/resources/monitoring-kibana.json +++ b/plugin/src/main/resources/monitoring-kibana.json @@ -19,6 +19,9 @@ "type": "date", "format": "date_time" }, + "interval_ms": { + "type": "long" + }, "type": { "type": "keyword" }, diff --git a/plugin/src/main/resources/monitoring-logstash.json b/plugin/src/main/resources/monitoring-logstash.json index 90ee37cc6e0..296fd59ed15 100644 --- a/plugin/src/main/resources/monitoring-logstash.json +++ b/plugin/src/main/resources/monitoring-logstash.json @@ -19,6 +19,9 @@ "type": "date", "format": "date_time" }, + "interval_ms": { + "type": "long" + }, "type": { "type": "keyword" }, From a8997387b79b46d42de16c8e8226bb9e878b42cd Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 19 Dec 2017 13:57:32 +0000 Subject: [PATCH 3/5] [ML] Calendar jobs endpoints (elastic/x-pack-elasticsearch#3320) * Calendar jobs endpoints * Refactor put and delete calendar job to use the same action * Check jobs exist when creating the calendar * Address review comments * Add isGroupOrJobMethod * Increase default page size for calendar query Original commit: elastic/x-pack-elasticsearch@7484799fe952cdf3b22b6045f122a45305057be9 --- .../xpack/ml/MachineLearning.java | 11 +- .../elasticsearch/xpack/ml/MlMetaIndex.java | 7 + .../elasticsearch/xpack/ml/MlMetadata.java | 4 + .../xpack/ml/action/GetCalendarsAction.java | 87 ++------ .../xpack/ml/action/PutCalendarAction.java | 49 ++++- .../ml/action/UpdateCalendarJobAction.java | 168 ++++++++++++++++ .../xpack/ml/calendars/Calendar.java | 4 +- .../xpack/ml/job/JobManager.java | 15 +- .../xpack/ml/job/groups/GroupOrJobLookup.java | 4 + .../job/persistence/CalendarQueryBuilder.java | 71 +++++++ .../xpack/ml/job/persistence/JobProvider.java | 143 ++++++++++++++ .../calendar/RestDeleteCalendarAction.java | 1 - .../calendar/RestDeleteCalendarJobAction.java | 44 +++++ .../calendar/RestPutCalendarJobAction.java | 44 +++++ .../PutCalendarActionResponseTests.java | 22 +++ .../UpdateCalendarJobActionResquestTests.java | 36 ++++ .../xpack/ml/calendars/CalendarTests.java | 8 +- .../xpack/ml/integration/JobProviderIT.java | 186 +++++++++++++++++- .../ml/job/groups/GroupOrJobLookupTests.java | 14 ++ .../api/xpack.ml.delete_calendar_job.json | 22 +++ .../api/xpack.ml.put_calendar_job.json | 22 +++ .../rest-api-spec/test/ml/calendar_crud.yml | 152 ++++++++++++-- 22 files changed, 1017 insertions(+), 97 deletions(-) create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java create mode 100644 plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java create mode 100644 plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java create mode 100644 plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json create mode 100644 plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java index 8d3e5e8c135..f6bd76593e6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -53,6 +52,7 @@ import org.elasticsearch.xpack.XPackPlugin; import org.elasticsearch.xpack.XPackSettings; import org.elasticsearch.xpack.ml.action.CloseJobAction; import org.elasticsearch.xpack.ml.action.DeleteCalendarAction; +import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction; import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.ml.action.DeleteExpiredDataAction; import org.elasticsearch.xpack.ml.action.DeleteFilterAction; @@ -118,8 +118,10 @@ import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction; import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarAction; +import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarJobAction; import org.elasticsearch.xpack.ml.rest.calendar.RestGetCalendarsAction; import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarAction; +import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarJobAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestDeleteDatafeedAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedStatsAction; import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedsAction; @@ -467,7 +469,9 @@ public class MachineLearning implements ActionPlugin { new RestForecastJobAction(settings, restController), new RestGetCalendarsAction(settings, restController), new RestPutCalendarAction(settings, restController), - new RestDeleteCalendarAction(settings, restController) + new RestDeleteCalendarAction(settings, restController), + new RestDeleteCalendarJobAction(settings, restController), + new RestPutCalendarJobAction(settings, restController) ); } @@ -516,7 +520,8 @@ public class MachineLearning implements ActionPlugin { new ActionHandler<>(ForecastJobAction.INSTANCE, ForecastJobAction.TransportAction.class), new ActionHandler<>(GetCalendarsAction.INSTANCE, GetCalendarsAction.TransportAction.class), new ActionHandler<>(PutCalendarAction.INSTANCE, PutCalendarAction.TransportAction.class), - new ActionHandler<>(DeleteCalendarAction.INSTANCE, DeleteCalendarAction.TransportAction.class) + new ActionHandler<>(DeleteCalendarAction.INSTANCE, DeleteCalendarAction.TransportAction.class), + new ActionHandler<>(UpdateCalendarJobAction.INSTANCE, UpdateCalendarJobAction.TransportAction.class) ); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java index 95e79cea92e..cc8153c0777 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ml; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; @@ -32,6 +33,12 @@ public final class MlMetaIndex { builder.startObject(TYPE); ElasticsearchMappings.addDefaultMapping(builder); builder.startObject(ElasticsearchMappings.PROPERTIES) + .startObject(Calendar.ID.getPreferredName()) + .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) + .endObject() + .startObject(Calendar.JOB_IDS.getPreferredName()) + .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD) + .endObject() .startObject(SpecialEvent.START_TIME.getPreferredName()) .field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE) .endObject() diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java index fb7cba02392..d6bf90936f6 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java @@ -81,6 +81,10 @@ public class MlMetadata implements MetaData.Custom { return jobs; } + public boolean isGroupOrJob(String id) { + return groupOrJobLookup.isGroupOrJob(id); + } + public Set expandJobIds(String expression, boolean allowNoJobs) { return groupOrJobLookup.expandJobIds(expression, allowNoJobs); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java index aafb33d2aa1..c547745bd79 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.ml.MlMetaIndex; import org.elasticsearch.xpack.ml.action.util.PageParams; import org.elasticsearch.xpack.ml.action.util.QueryPage; import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import java.io.IOException; @@ -212,16 +213,16 @@ public class GetCalendarsAction extends Action { - private final Client client; + private final JobProvider jobProvider; @Inject public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, - Client client) { + JobProvider jobProvider) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); - this.client = client; + this.jobProvider = jobProvider; } @Override @@ -239,76 +240,24 @@ public class GetCalendarsAction extends Action listener) { - GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId)); - executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener() { - @Override - public void onResponse(GetResponse getDocResponse) { - try { - QueryPage calendars; - if (getDocResponse.isExists()) { - BytesReference docSource = getDocResponse.getSourceAsBytesRef(); - - try (XContentParser parser = - XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource)) { - Calendar calendar = Calendar.PARSER.apply(parser, null).build(); - calendars = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD); - - Response response = new Response(calendars); - listener.onResponse(response); - } - } else { - this.onFailure(QueryPage.emptyQueryPage(Calendar.RESULTS_FIELD)); - } - - } catch (Exception e) { - this.onFailure(e); - } - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + jobProvider.calendar(calendarId, ActionListener.wrap( + calendar -> { + QueryPage page = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD); + listener.onResponse(new Response(page)); + }, + listener::onFailure + )); } private void getCalendars(PageParams pageParams, ActionListener listener) { - SearchSourceBuilder sourceBuilder = new SearchSourceBuilder() - .from(pageParams.getFrom()) - .size(pageParams.getSize()) - .sort(Calendar.ID.getPreferredName()) - .query(QueryBuilders.termQuery(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE)); - - SearchRequest searchRequest = new SearchRequest(MlMetaIndex.INDEX_NAME) - .indicesOptions(JobProvider.addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS)) - .source(sourceBuilder); - - executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, new ActionListener() { - @Override - public void onResponse(SearchResponse response) { - List docs = new ArrayList<>(); - for (SearchHit hit : response.getHits().getHits()) { - BytesReference docSource = hit.getSourceRef(); - try (XContentParser parser = XContentFactory.xContent(docSource).createParser( - NamedXContentRegistry.EMPTY, docSource)) { - docs.add(Calendar.PARSER.apply(parser, null).build()); - } catch (IOException e) { - this.onFailure(e); - } - } - - Response getResponse = new Response( - new QueryPage<>(docs, docs.size(), Calendar.RESULTS_FIELD)); - listener.onResponse(getResponse); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, - client::search); + CalendarQueryBuilder query = new CalendarQueryBuilder().pageParams(pageParams).sort(true); + jobProvider.calendars(query, ActionListener.wrap( + calendars -> { + listener.onResponse(new Response(calendars)); + }, + listener::onFailure + )); } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java index c8270866e39..6ba0aa5eaf3 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java @@ -20,7 +20,9 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,13 +36,18 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.ml.MlMetaIndex; +import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; +import org.elasticsearch.xpack.watcher.support.Exceptions; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.xpack.ClientHelper.ML_ORIGIN; @@ -162,36 +169,62 @@ public class PutCalendarAction extends Action { private final Client client; + private final ClusterService clusterService; @Inject public TransportAction(Settings settings, ThreadPool threadPool, TransportService transportService, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, ClusterService clusterService) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); this.client = client; + this.clusterService = clusterService; } @Override protected void doExecute(Request request, ActionListener listener) { - final Calendar calendar = request.getCalendar(); + Calendar calendar = request.getCalendar(); + + checkJobsExist(calendar.getJobIds(), listener::onFailure); + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { indexRequest.source(calendar.toXContent(builder, @@ -218,5 +251,17 @@ public class PutCalendarAction extends Action jobIds, Consumer errorHandler) { + ClusterState state = clusterService.state(); + MlMetadata mlMetadata = state.getMetaData().custom(MlMetadata.TYPE); + for (String jobId: jobIds) { + Set jobs = mlMetadata.expandJobIds(jobId, true); + if (jobs.isEmpty()) { + errorHandler.accept(ExceptionsHelper.missingJobException(jobId)); + return; + } + } + } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java new file mode 100644 index 00000000000..7753cfb4923 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestBuilder; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.ml.MlMetadata; +import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.persistence.JobProvider; +import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +public class UpdateCalendarJobAction extends Action { + public static final UpdateCalendarJobAction INSTANCE = new UpdateCalendarJobAction(); + public static final String NAME = "cluster:admin/xpack/ml/calendars/jobs/update"; + + private UpdateCalendarJobAction() { + super(NAME); + } + + @Override + public RequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new RequestBuilder(client); + } + + @Override + public PutCalendarAction.Response newResponse() { + return new PutCalendarAction.Response(); + } + + public static class Request extends ActionRequest { + + private String calendarId; + private Set jobIdsToAdd; + private Set jobIdsToRemove; + + Request() { + } + + public Request(String calendarId, Set jobIdsToAdd, Set jobIdsToRemove) { + this.calendarId = ExceptionsHelper.requireNonNull(calendarId, Calendar.ID.getPreferredName()); + this.jobIdsToAdd = ExceptionsHelper.requireNonNull(jobIdsToAdd, "job_ids_to_add"); + this.jobIdsToRemove = ExceptionsHelper.requireNonNull(jobIdsToRemove, "job_ids_to_remove"); + } + + public String getCalendarId() { + return calendarId; + } + + public Set getJobIdsToAdd() { + return jobIdsToAdd; + } + + public Set getJobIdsToRemove() { + return jobIdsToRemove; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(StreamInput in) throws IOException { + super.readFrom(in); + calendarId = in.readString(); + jobIdsToAdd = new HashSet<>(in.readList(StreamInput::readString)); + jobIdsToRemove = new HashSet<>(in.readList(StreamInput::readString)); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(calendarId); + out.writeStringList(new ArrayList<>(jobIdsToAdd)); + out.writeStringList(new ArrayList<>(jobIdsToRemove)); + } + + @Override + public int hashCode() { + return Objects.hash(calendarId, jobIdsToAdd, jobIdsToRemove); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Request other = (Request) obj; + return Objects.equals(calendarId, other.calendarId) && Objects.equals(jobIdsToAdd, other.jobIdsToAdd) + && Objects.equals(jobIdsToRemove, other.jobIdsToRemove); + } + } + + public static class RequestBuilder extends ActionRequestBuilder { + + public RequestBuilder(ElasticsearchClient client) { + super(client, INSTANCE, new Request()); + } + } + + public static class TransportAction extends HandledTransportAction { + + private final ClusterService clusterService; + private final JobProvider jobProvider; + + @Inject + public TransportAction(Settings settings, ThreadPool threadPool, + TransportService transportService, ActionFilters actionFilters, + IndexNameExpressionResolver indexNameExpressionResolver, + ClusterService clusterService, JobProvider jobProvider) { + super(settings, NAME, threadPool, transportService, actionFilters, + indexNameExpressionResolver, Request::new); + this.clusterService = clusterService; + this.jobProvider = jobProvider; + } + + @Override + protected void doExecute(Request request, ActionListener listener) { + ClusterState state = clusterService.state(); + MlMetadata mlMetadata = state.getMetaData().custom(MlMetadata.TYPE); + for (String jobToAdd: request.getJobIdsToAdd()) { + if (mlMetadata.isGroupOrJob(jobToAdd) == false) { + listener.onFailure(ExceptionsHelper.missingJobException(jobToAdd)); + return; + } + } + + for (String jobToRemove: request.getJobIdsToRemove()) { + if (mlMetadata.isGroupOrJob(jobToRemove) == false) { + listener.onFailure(ExceptionsHelper.missingJobException(jobToRemove)); + return; + } + } + + jobProvider.updateCalendar(request.getCalendarId(), request.getJobIdsToAdd(), request.getJobIdsToRemove(), + c -> listener.onResponse(new PutCalendarAction.Response(c)), listener::onFailure); + } + } +} + diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java index ed1d2c5f093..5a99fc6c6ef 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java @@ -13,10 +13,8 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.xpack.ml.MlMetaIndex; -import org.elasticsearch.xpack.ml.job.config.MlFilter; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -70,7 +68,7 @@ public class Calendar implements ToXContentObject, Writeable { } public List getJobIds() { - return new ArrayList<>(jobIds); + return Collections.unmodifiableList(jobIds); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 3b69c97f404..e1114a0ed0b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -313,7 +313,7 @@ public class JobManager extends AbstractComponent { String jobId = request.getJobId(); logger.debug("Deleting job '" + jobId + "'"); - // Step 3. When the job has been removed from the cluster state, return a response + // Step 4. When the job has been removed from the cluster state, return a response // ------- CheckedConsumer apiResponseHandler = jobDeleted -> { if (jobDeleted) { @@ -325,7 +325,7 @@ public class JobManager extends AbstractComponent { } }; - // Step 2. When the physical storage has been deleted, remove from Cluster State + // Step 3. When the physical storage has been deleted, remove from Cluster State // ------- CheckedConsumer deleteJobStateHandler = response -> clusterService.submitStateUpdateTask("delete-job-" + jobId, new AckedClusterStateUpdateTask(request, ActionListener.wrap(apiResponseHandler, actionListener::onFailure)) { @@ -351,11 +351,18 @@ public class JobManager extends AbstractComponent { } }); + + // Step 2. Remove the job from any calendars + CheckedConsumer removeFromCalendarsHandler = response -> { + jobProvider.removeJobFromCalendars(jobId, ActionListener.wrap(deleteJobStateHandler::accept, + actionListener::onFailure )); + }; + + // Step 1. Delete the physical storage // This task manages the physical deletion of the job state and results - task.delete(jobId, client, clusterService.state(), deleteJobStateHandler::accept, actionListener::onFailure); - + task.delete(jobId, client, clusterService.state(), removeFromCalendarsHandler, actionListener::onFailure); } public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionListener actionListener, diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java index 0bfc184d7c7..65aeec387cf 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java @@ -59,6 +59,10 @@ public class GroupOrJobLookup { return new GroupOrJobResolver().expand(expression, allowNoJobs); } + public boolean isGroupOrJob(String id) { + return groupOrJobLookup.containsKey(id); + } + private class GroupOrJobResolver extends NameResolver { private GroupOrJobResolver() { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java new file mode 100644 index 00000000000..7b50fd349ff --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.job.persistence; + +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.xpack.ml.action.util.PageParams; +import org.elasticsearch.xpack.ml.calendars.Calendar; + +public class CalendarQueryBuilder { + + private PageParams pageParams = new PageParams(0, 10000); + private String jobId; + private boolean sort = false; + + /** + * Page the query result + * @param params The page parameters + * @return this + */ + public CalendarQueryBuilder pageParams(PageParams params) { + this.pageParams = params; + return this; + } + + /** + * Query only calendars used by this job + * @param jobId The job Id + * @return this + */ + public CalendarQueryBuilder jobId(String jobId) { + this.jobId = jobId; + return this; + } + + /** + * Sort results by calendar_id + * @param sort Sort if true + * @return this + */ + public CalendarQueryBuilder sort(boolean sort) { + this.sort = sort; + return this; + } + + public SearchSourceBuilder build() { + QueryBuilder qb; + if (jobId != null) { + qb = new BoolQueryBuilder() + .filter(new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE)) + .filter(new TermsQueryBuilder(Calendar.JOB_IDS.getPreferredName(), jobId)); + } else { + qb = new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE); + } + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(qb); + + if (sort) { + sourceBuilder.sort(Calendar.ID.getPreferredName()); + } + + sourceBuilder.from(pageParams.getFrom()).size(pageParams.getSize()); + + return sourceBuilder; + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index 108756836c2..66a90e3e969 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -18,6 +18,11 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.elasticsearch.action.bulk.BulkAction; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.get.GetRequest; +import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.search.MultiSearchRequestBuilder; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; @@ -25,6 +30,9 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlock; @@ -38,6 +46,7 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; @@ -62,6 +71,7 @@ import org.elasticsearch.xpack.ml.action.GetCategoriesAction; import org.elasticsearch.xpack.ml.action.GetInfluencersAction; import org.elasticsearch.xpack.ml.action.GetRecordsAction; import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.MlFilter; @@ -87,13 +97,16 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.ClientHelper.clientWithOrigin; @@ -998,6 +1011,136 @@ public class JobProvider { , client::search); } + public void updateCalendar(String calendarId, Set jobIdsToAdd, Set jobIdsToRemove, + Consumer handler, Consumer errorHandler) { + + ActionListener getCalendarListener = ActionListener.wrap( + calendar -> { + Set currentJobs = new HashSet<>(calendar.getJobIds()); + currentJobs.addAll(jobIdsToAdd); + currentJobs.removeAll(jobIdsToRemove); + Calendar updatedCalendar = new Calendar(calendar.getId(), new ArrayList<>(currentJobs)); + + UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, updatedCalendar.documentId()); + updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + updateRequest.doc(updatedCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise calendar with id [" + updatedCalendar.getId() + "]", e); + } + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, updateRequest, + ActionListener.wrap( + response -> { + handler.accept(updatedCalendar); + }, + errorHandler) + , client::update); + + }, + errorHandler + ); + + calendar(calendarId, getCalendarListener); + } + + public void calendars(CalendarQueryBuilder queryBuilder, ActionListener> listener) { + SearchRequest searchRequest = client.prepareSearch(MlMetaIndex.INDEX_NAME) + .setIndicesOptions(IndicesOptions.lenientExpandOpen()) + .setSource(queryBuilder.build()).request(); + + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, + ActionListener.wrap( + response -> { + List calendars = new ArrayList<>(); + SearchHit[] hits = response.getHits().getHits(); + for (SearchHit hit : hits) { + calendars.add(parseSearchHit(hit, Calendar.PARSER, listener::onFailure).build()); + } + + listener.onResponse(new QueryPage(calendars, response.getHits().getTotalHits(), + Calendar.RESULTS_FIELD)); + }, + listener::onFailure) + , client::search); + } + + public void removeJobFromCalendars(String jobId, ActionListener listener) { + + ActionListener updateCalandarsListener = ActionListener.wrap( + r -> { + if (r.hasFailures()) { + listener.onResponse(false); + } + listener.onResponse(true); + }, + listener::onFailure + ); + + ActionListener> getCalendarsListener = ActionListener.wrap( + r -> { + BulkRequestBuilder bulkUpdate = client.prepareBulk(); + bulkUpdate.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + r.results().stream() + .map(c -> { + Set ids = new HashSet<>(c.getJobIds()); + ids.remove(jobId); + return new Calendar(c.getId(), new ArrayList<>(ids)); + }).forEach(c -> { + UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, + c.documentId()); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + updateRequest.doc(c.toXContent(builder, ToXContent.EMPTY_PARAMS)); + } catch (IOException e) { + throw new IllegalStateException("Failed to serialise calendar with id [" + c.getId() + "]", e); + } + bulkUpdate.add(updateRequest); + }); + + if (bulkUpdate.numberOfActions() > 0) { + executeAsyncWithOrigin(client, ML_ORIGIN, BulkAction.INSTANCE, bulkUpdate.request(), updateCalandarsListener); + } else { + listener.onResponse(true); + } + }, + listener::onFailure + ); + + CalendarQueryBuilder query = new CalendarQueryBuilder().jobId(jobId); + calendars(query, getCalendarsListener); + } + + public void calendar(String calendarId, ActionListener listener) { + GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId)); + executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getRequest, new ActionListener() { + @Override + public void onResponse(GetResponse getDocResponse) { + try { + if (getDocResponse.isExists()) { + BytesReference docSource = getDocResponse.getSourceAsBytesRef(); + + try (XContentParser parser = + XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource)) { + Calendar calendar = Calendar.PARSER.apply(parser, null).build(); + listener.onResponse(calendar); + } + } else { + this.onFailure(new ResourceNotFoundException("No calendar with id [" + calendarId + "]")); + } + } catch (Exception e) { + this.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, + client::get); + } + private void handleLatestModelSizeStats(String jobId, ModelSizeStats latestModelSizeStats, Consumer handler, Consumer errorHandler) { if (latestModelSizeStats != null) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java index 2e59fb8c514..6bad2ea8609 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java @@ -35,5 +35,4 @@ public class RestDeleteCalendarAction extends BaseRestHandler { DeleteCalendarAction.Request request = new DeleteCalendarAction.Request(restRequest.param(Calendar.ID.getPreferredName())); return channel -> client.execute(DeleteCalendarAction.INSTANCE, request, new AcknowledgedRestListener<>(channel)); } - } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java new file mode 100644 index 00000000000..749987755b9 --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.calendar; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction; +import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Collections; + +public class RestDeleteCalendarJobAction extends BaseRestHandler { + + public RestDeleteCalendarJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.DELETE, + MachineLearning.BASE_PATH + "calendars/{" + Calendar.ID.getPreferredName() + "}/jobs/{" + + Job.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_delete_calendar_job_action"; + } + + @Override + protected BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String calendarId = restRequest.param(Calendar.ID.getPreferredName()); + String jobId = restRequest.param(Job.ID.getPreferredName()); + UpdateCalendarJobAction.Request request = + new UpdateCalendarJobAction.Request(calendarId, Collections.emptySet(), Collections.singleton(jobId)); + return channel -> client.execute(UpdateCalendarJobAction.INSTANCE, request, new AcknowledgedRestListener<>(channel)); + } +} diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java new file mode 100644 index 00000000000..adc9da8876c --- /dev/null +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.rest.calendar; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction; +import org.elasticsearch.xpack.ml.calendars.Calendar; +import org.elasticsearch.xpack.ml.job.config.Job; + +import java.io.IOException; +import java.util.Collections; + +public class RestPutCalendarJobAction extends BaseRestHandler { + + public RestPutCalendarJobAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, + MachineLearning.BASE_PATH + "calendars/{" + Calendar.ID.getPreferredName() + "}/jobs/{" + + Job.ID.getPreferredName() + "}", this); + } + + @Override + public String getName() { + return "xpack_ml_put_calendar_job_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + String calendarId = restRequest.param(Calendar.ID.getPreferredName()); + String jobId = restRequest.param(Job.ID.getPreferredName()); + UpdateCalendarJobAction.Request putCalendarRequest = + new UpdateCalendarJobAction.Request(calendarId, Collections.singleton(jobId), Collections.emptySet()); + return channel -> client.execute(UpdateCalendarJobAction.INSTANCE, putCalendarRequest, new RestToXContentListener<>(channel)); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java new file mode 100644 index 00000000000..231e264ef35 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; +import org.elasticsearch.xpack.ml.calendars.CalendarTests; + +public class PutCalendarActionResponseTests extends AbstractStreamableTestCase { + + @Override + protected PutCalendarAction.Response createTestInstance() { + return new PutCalendarAction.Response(CalendarTests.testInstance()); + } + + @Override + protected PutCalendarAction.Response createBlankInstance() { + return new PutCalendarAction.Response(); + } +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java new file mode 100644 index 00000000000..c56483441b1 --- /dev/null +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.action; + +import org.elasticsearch.test.AbstractStreamableTestCase; + +import java.util.HashSet; +import java.util.Set; + +public class UpdateCalendarJobActionResquestTests extends AbstractStreamableTestCase { + + @Override + protected UpdateCalendarJobAction.Request createTestInstance() { + int addSize = randomIntBetween(0, 2); + Set toAdd = new HashSet<>(); + for (int i=0; i toRemove = new HashSet<>(); + for (int i=0; i { - @Override - protected Calendar createTestInstance() { + public static Calendar testInstance() { int size = randomInt(10); List items = new ArrayList<>(size); for (int i = 0; i < size; i++) { @@ -28,6 +27,11 @@ public class CalendarTests extends AbstractSerializingTestCase { return new Calendar(randomAlphaOfLengthBetween(1, 20), items); } + @Override + protected Calendar createTestInstance() { + return testInstance(); + } + @Override protected Writeable.Reader instanceReader() { return Calendar::new; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java index 96f932c86f0..724efa2ab46 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java @@ -25,6 +25,8 @@ import org.elasticsearch.xpack.XPackSingleNodeTestCase; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MlMetaIndex; import org.elasticsearch.xpack.ml.action.PutJobAction; +import org.elasticsearch.xpack.ml.action.util.QueryPage; +import org.elasticsearch.xpack.ml.calendars.Calendar; import org.elasticsearch.xpack.ml.calendars.SpecialEvent; import org.elasticsearch.xpack.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.ml.job.config.Connective; @@ -36,6 +38,7 @@ import org.elasticsearch.xpack.ml.job.config.MlFilter; import org.elasticsearch.xpack.ml.job.config.RuleAction; import org.elasticsearch.xpack.ml.job.config.RuleCondition; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; +import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; @@ -56,9 +59,21 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.hamcrest.core.Is.is; public class JobProviderIT extends XPackSingleNodeTestCase { @@ -97,6 +112,160 @@ public class JobProviderIT extends XPackSingleNodeTestCase { }); } + public void testGetCalandarByJobId() throws Exception { + List calendars = new ArrayList<>(); + calendars.add(new Calendar("empty calendar", Collections.emptyList())); + calendars.add(new Calendar("foo calendar", Collections.singletonList("foo"))); + calendars.add(new Calendar("foo bar calendar", Arrays.asList("foo", "bar"))); + calendars.add(new Calendar("cat calendar", Collections.singletonList("cat"))); + calendars.add(new Calendar("cat foo calendar", Arrays.asList("cat", "foo"))); + indexCalendars(calendars); + + List queryResult = getCalendars("ted"); + assertThat(queryResult, is(empty())); + + queryResult = getCalendars("foo"); + assertThat(queryResult, hasSize(3)); + Long matchedCount = queryResult.stream().filter( + c -> c.getId().equals("foo calendar") || c.getId().equals("foo bar calendar") || c.getId().equals("cat foo calendar")) + .collect(Collectors.counting()); + assertEquals(new Long(3), matchedCount); + + queryResult = getCalendars("bar"); + assertThat(queryResult, hasSize(1)); + assertEquals("foo bar calendar", queryResult.get(0).getId()); + } + + public void testUpdateCalendar() throws Exception { + String calendarId = "empty calendar"; + Calendar emptyCal = new Calendar(calendarId, Collections.emptyList()); + indexCalendars(Collections.singletonList(emptyCal)); + + Set addedIds = new HashSet<>(); + addedIds.add("foo"); + addedIds.add("bar"); + updateCalendar(calendarId, addedIds, Collections.emptySet()); + + Calendar updated = getCalendar(calendarId); + assertEquals(calendarId, updated.getId()); + assertEquals(addedIds, new HashSet<>(updated.getJobIds())); + + Set removedIds = new HashSet<>(); + removedIds.add("foo"); + updateCalendar(calendarId, Collections.emptySet(), removedIds); + + updated = getCalendar(calendarId); + assertEquals(calendarId, updated.getId()); + assertEquals(1, updated.getJobIds().size()); + assertEquals("bar", updated.getJobIds().get(0)); + } + + public void testRemoveJobFromCalendar() throws Exception { + List calendars = new ArrayList<>(); + calendars.add(new Calendar("empty calendar", Collections.emptyList())); + calendars.add(new Calendar("foo calendar", Collections.singletonList("foo"))); + calendars.add(new Calendar("foo bar calendar", Arrays.asList("foo", "bar"))); + calendars.add(new Calendar("cat calendar", Collections.singletonList("cat"))); + calendars.add(new Calendar("cat foo calendar", Arrays.asList("cat", "foo"))); + indexCalendars(calendars); + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + jobProvider.removeJobFromCalendars("bar", ActionListener.wrap( + r -> latch.countDown(), + exceptionHolder::set)); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + List updatedCalendars = getCalendars(null); + assertEquals(5, updatedCalendars.size()); + for (Calendar cal: updatedCalendars) { + assertThat("bar", not(isIn(cal.getJobIds()))); + } + + Calendar catFoo = getCalendar("cat foo calendar"); + assertThat(catFoo.getJobIds(), contains("cat", "foo")); + + CountDownLatch latch2 = new CountDownLatch(1); + exceptionHolder = new AtomicReference<>(); + jobProvider.removeJobFromCalendars("cat", ActionListener.wrap( + r -> latch2.countDown(), + exceptionHolder::set)); + + latch2.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + updatedCalendars = getCalendars(null); + assertEquals(5, updatedCalendars.size()); + for (Calendar cal: updatedCalendars) { + assertThat("bar", not(isIn(cal.getJobIds()))); + assertThat("cat", not(isIn(cal.getJobIds()))); + } + } + + private List getCalendars(String jobId) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference> result = new AtomicReference<>(); + + CalendarQueryBuilder query = new CalendarQueryBuilder(); + + if (jobId != null) { + query.jobId(jobId); + } + jobProvider.calendars(query, ActionListener.wrap( + r -> { + latch.countDown(); + result.set(r); + }, + exceptionHolder::set)); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + return result.get().results(); + } + + private void updateCalendar(String calendarId, Set idsToAdd, Set idsToRemove) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + jobProvider.updateCalendar(calendarId, idsToAdd, idsToRemove, + r -> latch.countDown(), + exceptionHolder::set); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + client().admin().indices().prepareRefresh(MlMetaIndex.INDEX_NAME).get(); + } + + private Calendar getCalendar(String calendarId) throws Exception { + + CountDownLatch latch = new CountDownLatch(1); + AtomicReference exceptionHolder = new AtomicReference<>(); + AtomicReference calendarHolder = new AtomicReference<>(); + jobProvider.calendar(calendarId, ActionListener.wrap( + c -> { latch.countDown(); calendarHolder.set(c); }, + exceptionHolder::set) + ); + + latch.await(); + if (exceptionHolder.get() != null) { + throw exceptionHolder.get(); + } + + return calendarHolder.get(); + } + public void testSpecialEvents() throws Exception { List events = new ArrayList<>(); events.add(new SpecialEvent("A_and_B_downtime", "downtime", createZonedDateTime(1000L), createZonedDateTime(2000L), @@ -321,7 +490,8 @@ public class JobProviderIT extends XPackSingleNodeTestCase { for (MlFilter filter : filters) { IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - indexRequest.source(filter.toXContent(builder, ToXContent.EMPTY_PARAMS)); + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + indexRequest.source(filter.toXContent(builder, params)); bulkRequest.add(indexRequest); } } @@ -341,7 +511,21 @@ public class JobProviderIT extends XPackSingleNodeTestCase { private void indexQuantiles(Quantiles quantiles) { JobResultsPersister persister = new JobResultsPersister(nodeSettings(), client()); persister.persistQuantiles(quantiles); + } + private void indexCalendars(List calendars) throws IOException { + BulkRequestBuilder bulkRequest = client().prepareBulk(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + + for (Calendar calendar: calendars) { + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId()); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true")); + indexRequest.source(calendar.toXContent(builder, params)); + bulkRequest.add(indexRequest); + } + } + bulkRequest.execute().actionGet(); } private ZonedDateTime createZonedDateTime(long epochMs) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookupTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookupTests.java index a8ec07695e6..26cf96f2d85 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookupTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookupTests.java @@ -89,6 +89,20 @@ public class GroupOrJobLookupTests extends ESTestCase { assertThat(groupOrJobLookup.expandJobIds("foo-group,*-2", false), contains("bar-2", "foo-1", "foo-2")); } + public void testIsGroupOrJob() { + List jobs = new ArrayList<>(); + jobs.add(mockJob("foo-1", Arrays.asList("foo-group", "ones"))); + jobs.add(mockJob("foo-2", Arrays.asList("foo-group", "twos"))); + jobs.add(mockJob("bar-1", Arrays.asList("bar-group", "ones"))); + jobs.add(mockJob("nogroup", Collections.emptyList())); + GroupOrJobLookup groupOrJobLookup = new GroupOrJobLookup(jobs); + + assertTrue(groupOrJobLookup.isGroupOrJob("foo-1")); + assertTrue(groupOrJobLookup.isGroupOrJob("twos")); + assertTrue(groupOrJobLookup.isGroupOrJob("nogroup")); + assertFalse(groupOrJobLookup.isGroupOrJob("missing")); + } + private static Job mockJob(String jobId, List groups) { Job job = mock(Job.class); when(job.getId()).thenReturn(jobId); diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json new file mode 100644 index 00000000000..43dc1b94789 --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json @@ -0,0 +1,22 @@ +{ + "xpack.ml.delete_calendar_job": { + "methods": [ "DELETE" ], + "url": { + "path": "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}", + "paths": [ "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}" ], + "parts": { + "calendar_id": { + "type" : "string", + "required" : true, + "description" : "The ID of the calendar to modify" + }, + "job_id": { + "type": "string", + "required": true, + "description": "The ID of the job to remove from the calendar" + } + } + }, + "body": null + } +} diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json new file mode 100644 index 00000000000..2abf870058c --- /dev/null +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json @@ -0,0 +1,22 @@ +{ + "xpack.ml.put_calendar_job": { + "methods": [ "PUT" ], + "url": { + "path": "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}", + "paths": [ "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}" ], + "parts": { + "calendar_id": { + "type": "string", + "required": true, + "description": "The ID of the calendar to modify" + }, + "job_id": { + "type": "string", + "required": true, + "description": "The ID of the job to add to the calendar" + } + } + }, + "body": null + } +} \ No newline at end of file diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml index 216cb521b5d..f39d88b6f2f 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml @@ -1,25 +1,52 @@ --- "Test calendar CRUD": + - do: + xpack.ml.put_job: + job_id: cal-job + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "cal-job" } + + - do: + xpack.ml.put_job: + job_id: cal-job2 + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "cal-job2" } + + - do: xpack.ml.put_calendar: calendar_id: "advent" body: > { - "job_ids": ["abc", "xyz"] + "job_ids": ["cal-job", "cal-job2"] } - match: { calendar_id: advent } - - match: { job_ids.0: abc } - - match: { job_ids.1: xyz } + - match: { job_ids.0: cal-job } + - match: { job_ids.1: cal-job2 } - do: xpack.ml.get_calendars: calendar_id: "advent" - - match: { count: 1 } + - match: { count: 1 } - match: calendars.0: calendar_id: "advent" - job_ids: ["abc", "xyz"] + job_ids: ["cal-job", "cal-job2"] - is_false: type - do: @@ -27,7 +54,7 @@ calendar_id: "Dogs of the Year" body: > { - "job_ids": ["abc2"] + "job_ids": ["cal-job"] } - do: @@ -51,6 +78,15 @@ xpack.ml.get_calendars: calendar_id: "Dogs of the Year" + - do: + catch: missing + xpack.ml.put_calendar: + calendar_id: "new cal with unknown job" + body: > + { + "job_ids": ["cal-job", "unknown-job"] + } + --- "Test PageParams": - do: @@ -66,14 +102,16 @@ - do: xpack.ml.get_calendars: from: 2 - - match: { count: 1 } + - match: { count: 3 } + - length: { calendars: 1} - match: { calendars.0.calendar_id: Calendar3 } - do: xpack.ml.get_calendars: from: 1 size: 1 - - match: { count: 1 } + - match: { count: 3 } + - length: { calendars: 1} - match: { calendars.0.calendar_id: Calendar2 } --- @@ -90,10 +128,6 @@ - do: xpack.ml.put_calendar: calendar_id: "Mayan" - body: > - { - "job_ids": ["apocalypse"] - } - do: catch: /version_conflict_engine_exception/ @@ -106,3 +140,97 @@ catch: bad_request xpack.ml.put_calendar: calendar_id: "_all" + +--- +"Test deleted job is removed from calendar": + + - do: + xpack.ml.put_job: + job_id: cal-crud-test-delete + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "cal-crud-test-delete" } + + - do: + xpack.ml.put_calendar: + calendar_id: "delete-test" + body: > + { + "job_ids": ["cal-crud-test-delete"] + } + + - do: + xpack.ml.delete_job: + job_id: cal-crud-test-delete + - match: { acknowledged: true } + + - do: + xpack.ml.get_calendars: + calendar_id: "delete-test" + - match: { count: 1 } + - match: { calendars.0.job_ids: [] } + +--- +"Test update calendar": + + - do: + xpack.ml.put_calendar: + calendar_id: "Wildlife" + + - do: + xpack.ml.put_job: + job_id: tiger + body: > + { + "analysis_config" : { + "detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}] + }, + "data_description" : { + } + } + - match: { job_id: "tiger" } + + - do: + xpack.ml.put_calendar_job: + calendar_id: "Wildlife" + job_id: "tiger" + - match: { calendar_id: "Wildlife" } + - match: { job_ids.0: "tiger" } + + - do: + xpack.ml.get_calendars: + calendar_id: "Wildlife" + - match: { count: 1 } + - match: { calendars.0.calendar_id: "Wildlife" } + - length: { calendars.0.job_ids: 1 } + - match: { calendars.0.job_ids.0: "tiger" } + + - do: + xpack.ml.delete_calendar_job: + calendar_id: "Wildlife" + job_id: "tiger" + + - do: + xpack.ml.get_calendars: + calendar_id: "Wildlife" + - match: { count: 1 } + - match: { calendars.0.calendar_id: "Wildlife" } + - length: { calendars.0.job_ids: 0 } + + - do: + catch: missing + xpack.ml.put_calendar_job: + calendar_id: "Wildlife" + job_id: "missing job" + + - do: + catch: missing + xpack.ml.delete_calendar_job: + calendar_id: "Wildlife" + job_id: "missing job" From f1f1be3927d3797d906f1470ea44845e92c47ce1 Mon Sep 17 00:00:00 2001 From: jaymode Date: Tue, 19 Dec 2017 08:24:23 -0700 Subject: [PATCH 4/5] Test: tests that use security index should not delete template Tests that rely on the security index and security index template being present should not remove the template between tests as this can cause test failures. The template upgrade service relies on cluster state updates to trigger the template being added after a delete, but there is a scenario where the test will just wait for template that never shows up as there is no cluster state update in that time. Instead of fighting ourselves, we should just leave the template in place. Relates elastic/x-pack-elasticsearch#2915 Relates elastic/x-pack-elasticsearch#2911 Original commit: elastic/x-pack-elasticsearch@3ca4aef0bedc2892f8063553d6744ad12f4e71ce --- .../integration/ldap/AbstractAdLdapRealmTestCase.java | 10 ++++++++++ .../elasticsearch/test/NativeRealmIntegTestCase.java | 10 ++++++++++ .../xpack/security/SecurityTribeTests.java | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java b/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java index ce3b2f1c883..995ce8e7f34 100644 --- a/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java @@ -17,10 +17,12 @@ import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecurityIntegTestCase; import org.elasticsearch.test.SecuritySettingsSource; +import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingRequestBuilder; import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingResponse; import org.elasticsearch.xpack.security.authc.ldap.LdapRealm; @@ -42,6 +44,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -191,6 +194,13 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase super.deleteSecurityIndex(); } + @Override + public Set excludeTemplates() { + Set templates = Sets.newHashSet(super.excludeTemplates()); + templates.add(SecurityLifecycleService.SECURITY_TEMPLATE_NAME); // don't remove the security index template + return templates; + } + private List getRoleMappingContent(Function contentFunction) { return getRoleMappingContent(contentFunction, AbstractAdLdapRealmTestCase.roleMappings); } diff --git a/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java b/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java index 96999294ce3..14f8d61c520 100644 --- a/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java +++ b/plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java @@ -15,6 +15,8 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.xpack.security.SecurityLifecycleService; import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.security.client.SecurityClient; import org.elasticsearch.xpack.security.user.ElasticUser; @@ -26,6 +28,7 @@ import org.junit.Before; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.Set; /** * Test case with method to handle the starting and stopping the stores for native users and roles @@ -59,6 +62,13 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase { .build(); } + @Override + public Set excludeTemplates() { + Set templates = Sets.newHashSet(super.excludeTemplates()); + templates.add(SecurityLifecycleService.SECURITY_TEMPLATE_NAME); // don't remove the security index template + return templates; + } + private SecureString reservedPassword = SecuritySettingsSource.TEST_PASSWORD_SECURE_STRING; protected SecureString getReservedPassword() { diff --git a/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java b/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java index 9339c8ce20e..cdaa94c01bc 100644 --- a/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java +++ b/qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.IndexNotFoundException; @@ -141,7 +142,7 @@ public class SecurityTribeTests extends NativeRealmIntegTestCase { public void tearDownTribeNodeAndWipeCluster() throws Exception { if (cluster2 != null) { try { - cluster2.wipe(Collections.emptySet()); + cluster2.wipe(Collections.singleton(SecurityLifecycleService.SECURITY_TEMPLATE_NAME)); try { // this is a hack to clean up the .security index since only the XPackSecurity user or superusers can delete it final Client cluster2Client = cluster2.client().filterWithHeader(Collections.singletonMap("Authorization", From 08a35d44c62475fe41f4c39533f59bff50d542f5 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 19 Dec 2017 16:28:36 +0000 Subject: [PATCH 5/5] [ML] Support multiple rule actions and renames (elastic/x-pack-elasticsearch#3356) Relates elastic/x-pack-elasticsearch#3325 Original commit: elastic/x-pack-elasticsearch@a7f400aeeb925932e4f93e0a3d511b36aa4b2ba1 --- .../xpack/ml/calendars/SpecialEvent.java | 2 +- .../xpack/ml/job/config/DetectionRule.java | 106 ++++++++++-------- .../xpack/ml/job/config/Detector.java | 50 ++++----- .../xpack/ml/job/config/JobUpdate.java | 10 +- .../xpack/ml/job/config/RuleAction.java | 3 +- .../xpack/ml/job/config/RuleCondition.java | 55 +++++---- .../autodetect/writer/FieldConfigWriter.java | 4 +- .../xpack/ml/calendars/SpecialEventTests.java | 9 +- .../ml/integration/DetectionRulesIT.java | 2 +- .../xpack/ml/integration/JobProviderIT.java | 4 +- .../ml/job/config/AnalysisConfigTests.java | 4 +- .../ml/job/config/DetectionRuleTests.java | 50 +++++---- .../xpack/ml/job/config/DetectorTests.java | 32 +++--- .../xpack/ml/job/config/JobUpdateTests.java | 2 +- .../xpack/ml/job/config/RuleActionTests.java | 8 +- .../ml/job/config/RuleConditionTests.java | 26 ++--- .../ControlMsgToProcessWriterTests.java | 14 +-- .../writer/FieldConfigWriterTests.java | 16 ++- .../rest-api-spec/test/ml/filter_crud.yml | 8 +- .../rest-api-spec/test/ml/jobs_crud.yml | 6 +- 20 files changed, 211 insertions(+), 200 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java index 339ca80351a..f65c33f2c9e 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java @@ -150,7 +150,7 @@ public class SpecialEvent implements ToXContentObject, Writeable { conditions.add(RuleCondition.createTime(Operator.LT, bucketEndTime)); DetectionRule.Builder builder = new DetectionRule.Builder(conditions); - builder.setRuleAction(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS); + builder.setActions(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING); builder.setConditionsConnective(Connective.AND); return builder.build(); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java index 8b244bc47c7..537593bd8cb 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java @@ -21,8 +21,10 @@ import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -32,11 +34,11 @@ import java.util.stream.Collectors; public class DetectionRule implements ToXContentObject, Writeable { public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule"); - public static final ParseField RULE_ACTION_FIELD = new ParseField("rule_action"); + public static final ParseField ACTIONS_FIELD = new ParseField("actions"); public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name"); public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value"); public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective"); - public static final ParseField RULE_CONDITIONS_FIELD = new ParseField("rule_conditions"); + public static final ParseField CONDITIONS_FIELD = new ParseField("conditions"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ObjectParser METADATA_PARSER = @@ -51,12 +53,7 @@ public class DetectionRule implements ToXContentObject, Writeable { for (MlParserType parserType : MlParserType.values()) { ObjectParser parser = PARSERS.get(parserType); assert parser != null; - parser.declareField(Builder::setRuleAction, p -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return RuleAction.fromString(p.text()); - } - throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, RULE_ACTION_FIELD, ValueType.STRING); + parser.declareStringArray(Builder::setActions, ACTIONS_FIELD); parser.declareString(Builder::setTargetFieldName, TARGET_FIELD_NAME_FIELD); parser.declareString(Builder::setTargetFieldValue, TARGET_FIELD_VALUE_FIELD); parser.declareField(Builder::setConditionsConnective, p -> { @@ -65,33 +62,38 @@ public class DetectionRule implements ToXContentObject, Writeable { } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING); - parser.declareObjectArray(Builder::setRuleConditions, (p, c) -> - RuleCondition.PARSERS.get(parserType).apply(p, c), RULE_CONDITIONS_FIELD); + parser.declareObjectArray(Builder::setConditions, (p, c) -> + RuleCondition.PARSERS.get(parserType).apply(p, c), CONDITIONS_FIELD); } } - private final RuleAction ruleAction; + private final EnumSet actions; private final String targetFieldName; private final String targetFieldValue; private final Connective conditionsConnective; - private final List ruleConditions; + private final List conditions; - private DetectionRule(RuleAction ruleAction, @Nullable String targetFieldName, @Nullable String targetFieldValue, - Connective conditionsConnective, List ruleConditions) { - this.ruleAction = Objects.requireNonNull(ruleAction); + private DetectionRule(EnumSet actions, @Nullable String targetFieldName, @Nullable String targetFieldValue, + Connective conditionsConnective, List conditions) { + this.actions = Objects.requireNonNull(actions); this.targetFieldName = targetFieldName; this.targetFieldValue = targetFieldValue; this.conditionsConnective = Objects.requireNonNull(conditionsConnective); - this.ruleConditions = Collections.unmodifiableList(ruleConditions); + this.conditions = Collections.unmodifiableList(conditions); } public DetectionRule(StreamInput in) throws IOException { - ruleAction = RuleAction.readFromStream(in); + int actionsCount = in.readVInt(); + actions = EnumSet.noneOf(RuleAction.class); + for (int i = 0; i < actionsCount; ++i) { + actions.add(RuleAction.readFromStream(in)); + } + conditionsConnective = Connective.readFromStream(in); int size = in.readVInt(); - ruleConditions = new ArrayList<>(size); + conditions = new ArrayList<>(size); for (int i = 0; i < size; i++) { - ruleConditions.add(new RuleCondition(in)); + conditions.add(new RuleCondition(in)); } targetFieldName = in.readOptionalString(); targetFieldValue = in.readOptionalString(); @@ -99,10 +101,14 @@ public class DetectionRule implements ToXContentObject, Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - ruleAction.writeTo(out); + out.writeVInt(actions.size()); + for (RuleAction action : actions) { + action.writeTo(out); + } + conditionsConnective.writeTo(out); - out.writeVInt(ruleConditions.size()); - for (RuleCondition condition : ruleConditions) { + out.writeVInt(conditions.size()); + for (RuleCondition condition : conditions) { condition.writeTo(out); } out.writeOptionalString(targetFieldName); @@ -112,9 +118,9 @@ public class DetectionRule implements ToXContentObject, Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(RULE_ACTION_FIELD.getPreferredName(), ruleAction); + builder.field(ACTIONS_FIELD.getPreferredName(), actions); builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective); - builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions); + builder.field(CONDITIONS_FIELD.getPreferredName(), conditions); if (targetFieldName != null) { builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName); } @@ -125,8 +131,8 @@ public class DetectionRule implements ToXContentObject, Writeable { return builder; } - public RuleAction getRuleAction() { - return ruleAction; + public EnumSet getActions() { + return actions; } @Nullable @@ -143,12 +149,12 @@ public class DetectionRule implements ToXContentObject, Writeable { return conditionsConnective; } - public List getRuleConditions() { - return ruleConditions; + public List getConditions() { + return conditions; } public Set extractReferencedFilters() { - return ruleConditions.stream().map(RuleCondition::getValueFilter).filter(Objects::nonNull).collect(Collectors.toSet()); + return conditions.stream().map(RuleCondition::getFilterId).filter(Objects::nonNull).collect(Collectors.toSet()); } @Override @@ -162,34 +168,46 @@ public class DetectionRule implements ToXContentObject, Writeable { } DetectionRule other = (DetectionRule) obj; - return Objects.equals(ruleAction, other.ruleAction) + return Objects.equals(actions, other.actions) && Objects.equals(targetFieldName, other.targetFieldName) && Objects.equals(targetFieldValue, other.targetFieldValue) && Objects.equals(conditionsConnective, other.conditionsConnective) - && Objects.equals(ruleConditions, other.ruleConditions); + && Objects.equals(conditions, other.conditions); } @Override public int hashCode() { - return Objects.hash(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions); + return Objects.hash(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions); } public static class Builder { - private RuleAction ruleAction = RuleAction.FILTER_RESULTS; + private EnumSet actions = EnumSet.of(RuleAction.FILTER_RESULTS); private String targetFieldName; private String targetFieldValue; private Connective conditionsConnective = Connective.OR; - private List ruleConditions = Collections.emptyList(); + private List conditions = Collections.emptyList(); - public Builder(List ruleConditions) { - this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_CONDITIONS_FIELD.getPreferredName()); + public Builder(List conditions) { + this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName()); } private Builder() { } - public Builder setRuleAction(RuleAction ruleAction) { - this.ruleAction = ExceptionsHelper.requireNonNull(ruleAction, RULE_ACTION_FIELD.getPreferredName()); + public Builder setActions(List actions) { + this.actions.clear(); + actions.stream().map(RuleAction::fromString).forEach(this.actions::add); + return this; + } + + public Builder setActions(EnumSet actions) { + this.actions = Objects.requireNonNull(actions, ACTIONS_FIELD.getPreferredName()); + return this; + } + + public Builder setActions(RuleAction... actions) { + this.actions.clear(); + Arrays.stream(actions).forEach(this.actions::add); return this; } @@ -208,8 +226,8 @@ public class DetectionRule implements ToXContentObject, Writeable { return this; } - public Builder setRuleConditions(List ruleConditions) { - this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_ACTION_FIELD.getPreferredName()); + public Builder setConditions(List conditions) { + this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName()); return this; } @@ -218,18 +236,18 @@ public class DetectionRule implements ToXContentObject, Writeable { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue); throw ExceptionsHelper.badRequestException(msg); } - if (ruleConditions == null || ruleConditions.isEmpty()) { + if (conditions == null || conditions.isEmpty()) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION); throw ExceptionsHelper.badRequestException(msg); } - for (RuleCondition condition : ruleConditions) { - if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) { + for (RuleCondition condition : conditions) { + if (condition.getType() == RuleConditionType.CATEGORICAL && targetFieldName != null) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName()); throw ExceptionsHelper.badRequestException(msg); } } - return new DetectionRule(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions); + return new DetectionRule(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions); } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java index c167e75089a..726fef8c91a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java @@ -83,7 +83,7 @@ public class Detector implements ToXContentObject, Writeable { public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name"); public static final ParseField USE_NULL_FIELD = new ParseField("use_null"); public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent"); - public static final ParseField DETECTOR_RULES_FIELD = new ParseField("detector_rules"); + public static final ParseField RULES_FIELD = new ParseField("rules"); public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly @@ -110,8 +110,8 @@ public class Detector implements ToXContentObject, Writeable { } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING); - parser.declareObjectArray(Builder::setDetectorRules, (p, c) -> - DetectionRule.PARSERS.get(parserType).apply(p, c).build(), DETECTOR_RULES_FIELD); + parser.declareObjectArray(Builder::setRules, (p, c) -> + DetectionRule.PARSERS.get(parserType).apply(p, c).build(), RULES_FIELD); parser.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX); } } @@ -329,7 +329,7 @@ public class Detector implements ToXContentObject, Writeable { private final String partitionFieldName; private final boolean useNull; private final ExcludeFrequent excludeFrequent; - private final List detectorRules; + private final List rules; private final int detectorIndex; public Detector(StreamInput in) throws IOException { @@ -341,7 +341,7 @@ public class Detector implements ToXContentObject, Writeable { partitionFieldName = in.readOptionalString(); useNull = in.readBoolean(); excludeFrequent = in.readBoolean() ? ExcludeFrequent.readFromStream(in) : null; - detectorRules = in.readList(DetectionRule::new); + rules = in.readList(DetectionRule::new); if (in.getVersion().onOrAfter(Version.V_5_5_0)) { detectorIndex = in.readInt(); } else { @@ -365,7 +365,7 @@ public class Detector implements ToXContentObject, Writeable { } else { out.writeBoolean(false); } - out.writeList(detectorRules); + out.writeList(rules); if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeInt(detectorIndex); } @@ -394,7 +394,7 @@ public class Detector implements ToXContentObject, Writeable { if (excludeFrequent != null) { builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent); } - builder.field(DETECTOR_RULES_FIELD.getPreferredName(), detectorRules); + builder.field(RULES_FIELD.getPreferredName(), rules); // negative means "unknown", which should only happen for a 5.4 job if (detectorIndex >= 0 // no point writing this to cluster state, as the indexes will get reassigned on reload anyway @@ -406,7 +406,7 @@ public class Detector implements ToXContentObject, Writeable { } private Detector(String detectorDescription, String function, String fieldName, String byFieldName, String overFieldName, - String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List detectorRules, + String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List rules, int detectorIndex) { this.function = function; this.fieldName = fieldName; @@ -415,7 +415,7 @@ public class Detector implements ToXContentObject, Writeable { this.partitionFieldName = partitionFieldName; this.useNull = useNull; this.excludeFrequent = excludeFrequent; - this.detectorRules = Collections.unmodifiableList(detectorRules); + this.rules = Collections.unmodifiableList(rules); this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this); this.detectorIndex = detectorIndex; } @@ -491,8 +491,8 @@ public class Detector implements ToXContentObject, Writeable { return excludeFrequent; } - public List getDetectorRules() { - return detectorRules; + public List getRules() { + return rules; } /** @@ -514,8 +514,8 @@ public class Detector implements ToXContentObject, Writeable { } public Set extractReferencedFilters() { - return detectorRules == null ? Collections.emptySet() - : detectorRules.stream().map(DetectionRule::extractReferencedFilters) + return rules == null ? Collections.emptySet() + : rules.stream().map(DetectionRule::extractReferencedFilters) .flatMap(Set::stream).collect(Collectors.toSet()); } @@ -556,15 +556,14 @@ public class Detector implements ToXContentObject, Writeable { Objects.equals(this.partitionFieldName, that.partitionFieldName) && Objects.equals(this.useNull, that.useNull) && Objects.equals(this.excludeFrequent, that.excludeFrequent) && - Objects.equals(this.detectorRules, that.detectorRules) && + Objects.equals(this.rules, that.rules) && this.detectorIndex == that.detectorIndex; } @Override public int hashCode() { - return Objects.hash(detectorDescription, function, fieldName, byFieldName, - overFieldName, partitionFieldName, useNull, excludeFrequent, - detectorRules, detectorIndex); + return Objects.hash(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, useNull, + excludeFrequent, rules, detectorIndex); } public static class Builder { @@ -587,7 +586,7 @@ public class Detector implements ToXContentObject, Writeable { private String partitionFieldName; private boolean useNull = false; private ExcludeFrequent excludeFrequent; - private List detectorRules = Collections.emptyList(); + private List rules = Collections.emptyList(); // negative means unknown, and is expected for v5.4 jobs private int detectorIndex = -1; @@ -603,8 +602,7 @@ public class Detector implements ToXContentObject, Writeable { partitionFieldName = detector.partitionFieldName; useNull = detector.useNull; excludeFrequent = detector.excludeFrequent; - detectorRules = new ArrayList<>(detector.detectorRules.size()); - detectorRules.addAll(detector.getDetectorRules()); + rules = new ArrayList<>(detector.getRules()); detectorIndex = detector.detectorIndex; } @@ -645,8 +643,8 @@ public class Detector implements ToXContentObject, Writeable { this.excludeFrequent = excludeFrequent; } - public void setDetectorRules(List detectorRules) { - this.detectorRules = detectorRules; + public void setRules(List rules) { + this.rules = rules; } public void setDetectorIndex(int detectorIndex) { @@ -704,12 +702,12 @@ public class Detector implements ToXContentObject, Writeable { } String function = this.function == null ? Detector.METRIC : this.function; - if (detectorRules.isEmpty() == false) { + if (rules.isEmpty() == false) { if (FUNCTIONS_WITHOUT_RULE_SUPPORT.contains(function)) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function); throw ExceptionsHelper.badRequestException(msg); } - for (DetectionRule rule : detectorRules) { + for (DetectionRule rule : rules) { checkScoping(rule); } } @@ -764,7 +762,7 @@ public class Detector implements ToXContentObject, Writeable { } return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, - useNull, excludeFrequent, detectorRules, detectorIndex); + useNull, excludeFrequent, rules, detectorIndex); } public List extractAnalysisFields() { @@ -802,7 +800,7 @@ public class Detector implements ToXContentObject, Writeable { String targetFieldName = rule.getTargetFieldName(); checkTargetFieldNameIsValid(extractAnalysisFields(), targetFieldName); List validOptions = getValidFieldNameOptions(rule); - for (RuleCondition condition : rule.getRuleConditions()) { + for (RuleCondition condition : rule.getConditions()) { if (!validOptions.contains(condition.getFieldName())) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, validOptions, condition.getFieldName()); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java index dc304149769..85096c49a34 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java @@ -315,7 +315,7 @@ public class JobUpdate implements Writeable, ToXContentObject { detectorbuilder.setDetectorDescription(dd.getDescription()); } if (dd.getRules() != null) { - detectorbuilder.setDetectorRules(dd.getRules()); + detectorbuilder.setRules(dd.getRules()); } ac.getDetectors().set(dd.getDetectorIndex(), detectorbuilder.build()); } @@ -435,13 +435,11 @@ public class JobUpdate implements Writeable, ToXContentObject { new ConstructingObjectParser<>("detector_update", a -> new DetectorUpdate((int) a[0], (String) a[1], (List) a[2])); - public static final ParseField RULES = new ParseField("rules"); - static { PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), Detector.DETECTOR_INDEX); PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION); - PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), - (parser, parseFieldMatcher) -> DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), RULES); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (parser, parseFieldMatcher) -> + DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), Detector.RULES_FIELD); } private int detectorIndex; @@ -495,7 +493,7 @@ public class JobUpdate implements Writeable, ToXContentObject { builder.field(Job.DESCRIPTION.getPreferredName(), description); } if (rules != null) { - builder.field(RULES.getPreferredName(), rules); + builder.field(Detector.RULES_FIELD.getPreferredName(), rules); } builder.endObject(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java index 20efbf01ac6..2a96cc45c4a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java @@ -14,8 +14,7 @@ import java.util.Locale; public enum RuleAction implements Writeable { FILTER_RESULTS, - SKIP_SAMPLING, - SKIP_SAMPLING_AND_FILTER_RESULTS; + SKIP_SAMPLING; /** * Case-insensitive from string method. diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java index b23ed163066..a950189931f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java @@ -26,11 +26,10 @@ import java.util.Map; import java.util.Objects; public class RuleCondition implements ToXContentObject, Writeable { - public static final ParseField CONDITION_TYPE_FIELD = new ParseField("condition_type"); + public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition"); public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name"); public static final ParseField FIELD_VALUE_FIELD = new ParseField("field_value"); - public static final ParseField VALUE_FILTER_FIELD = new ParseField("value_filter"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ConstructingObjectParser METADATA_PARSER = @@ -53,60 +52,60 @@ public class RuleCondition implements ToXContentObject, Writeable { return RuleConditionType.fromString(p.text()); } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, CONDITION_TYPE_FIELD, ValueType.STRING); + }, TYPE_FIELD, ValueType.STRING); parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_NAME_FIELD); parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_VALUE_FIELD); parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), Condition.PARSER, Condition.CONDITION_FIELD); - parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), VALUE_FILTER_FIELD); + parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), MlFilter.ID); } } - private final RuleConditionType conditionType; + private final RuleConditionType type; private final String fieldName; private final String fieldValue; private final Condition condition; - private final String valueFilter; + private final String filterId; public RuleCondition(StreamInput in) throws IOException { - conditionType = RuleConditionType.readFromStream(in); + type = RuleConditionType.readFromStream(in); condition = in.readOptionalWriteable(Condition::new); fieldName = in.readOptionalString(); fieldValue = in.readOptionalString(); - valueFilter = in.readOptionalString(); + filterId = in.readOptionalString(); } @Override public void writeTo(StreamOutput out) throws IOException { - conditionType.writeTo(out); + type.writeTo(out); out.writeOptionalWriteable(condition); out.writeOptionalString(fieldName); out.writeOptionalString(fieldValue); - out.writeOptionalString(valueFilter); + out.writeOptionalString(filterId); } - RuleCondition(RuleConditionType conditionType, String fieldName, String fieldValue, Condition condition, String valueFilter) { - this.conditionType = conditionType; + RuleCondition(RuleConditionType type, String fieldName, String fieldValue, Condition condition, String filterId) { + this.type = type; this.fieldName = fieldName; this.fieldValue = fieldValue; this.condition = condition; - this.valueFilter = valueFilter; + this.filterId = filterId; verifyFieldsBoundToType(this); verifyFieldValueRequiresFieldName(this); } public RuleCondition(RuleCondition ruleCondition) { - this.conditionType = ruleCondition.conditionType; + this.type = ruleCondition.type; this.fieldName = ruleCondition.fieldName; this.fieldValue = ruleCondition.fieldValue; this.condition = ruleCondition.condition; - this.valueFilter = ruleCondition.valueFilter; + this.filterId = ruleCondition.filterId; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(CONDITION_TYPE_FIELD.getPreferredName(), conditionType); + builder.field(TYPE_FIELD.getPreferredName(), type); if (condition != null) { builder.field(Condition.CONDITION_FIELD.getPreferredName(), condition); } @@ -116,15 +115,15 @@ public class RuleCondition implements ToXContentObject, Writeable { if (fieldValue != null) { builder.field(FIELD_VALUE_FIELD.getPreferredName(), fieldValue); } - if (valueFilter != null) { - builder.field(VALUE_FILTER_FIELD.getPreferredName(), valueFilter); + if (filterId != null) { + builder.field(MlFilter.ID.getPreferredName(), filterId); } builder.endObject(); return builder; } - public RuleConditionType getConditionType() { - return conditionType; + public RuleConditionType getType() { + return type; } /** @@ -153,8 +152,8 @@ public class RuleCondition implements ToXContentObject, Writeable { * The unique identifier of a filter. Required when the rule type is * categorical. Should be null for all other types. */ - public String getValueFilter() { - return valueFilter; + public String getFilterId() { + return filterId; } @Override @@ -168,14 +167,14 @@ public class RuleCondition implements ToXContentObject, Writeable { } RuleCondition other = (RuleCondition) obj; - return Objects.equals(conditionType, other.conditionType) && Objects.equals(fieldName, other.fieldName) + return Objects.equals(type, other.type) && Objects.equals(fieldName, other.fieldName) && Objects.equals(fieldValue, other.fieldValue) && Objects.equals(condition, other.condition) - && Objects.equals(valueFilter, other.valueFilter); + && Objects.equals(filterId, other.filterId); } @Override public int hashCode() { - return Objects.hash(conditionType, fieldName, fieldValue, condition, valueFilter); + return Objects.hash(type, fieldName, fieldValue, condition, filterId); } public static RuleCondition createCategorical(String fieldName, String valueFilter) { @@ -195,7 +194,7 @@ public class RuleCondition implements ToXContentObject, Writeable { } private static void verifyFieldsBoundToType(RuleCondition ruleCondition) throws ElasticsearchParseException { - switch (ruleCondition.getConditionType()) { + switch (ruleCondition.getType()) { case CATEGORICAL: verifyCategorical(ruleCondition); break; @@ -215,7 +214,7 @@ public class RuleCondition implements ToXContentObject, Writeable { private static void verifyCategorical(RuleCondition ruleCondition) throws ElasticsearchParseException { checkCategoricalHasNoField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); checkCategoricalHasNoField(RuleCondition.FIELD_VALUE_FIELD.getPreferredName(), ruleCondition.getFieldValue()); - checkCategoricalHasField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter()); + checkCategoricalHasField(MlFilter.ID.getPreferredName(), ruleCondition.getFilterId()); } private static void checkCategoricalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { @@ -233,7 +232,7 @@ public class RuleCondition implements ToXContentObject, Writeable { } private static void verifyNumerical(RuleCondition ruleCondition) throws ElasticsearchParseException { - checkNumericalHasNoField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter()); + checkNumericalHasNoField(MlFilter.ID.getPreferredName(), ruleCondition.getFilterId()); checkNumericalHasField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); if (ruleCondition.getFieldName() != null && ruleCondition.getFieldValue() == null) { String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java index 3d107d39ff1..c25635478dc 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java @@ -107,8 +107,8 @@ public class FieldConfigWriter { StringBuilder contents) throws IOException { List rules = new ArrayList<>(); - if (detector.getDetectorRules() != null) { - rules.addAll(detector.getDetectorRules()); + if (detector.getRules() != null) { + rules.addAll(detector.getRules()); } rules.addAll(specialEvents); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java index 148b6200d78..6ecd8496ac8 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.EnumSet; import java.util.List; public class SpecialEventTests extends AbstractSerializingTestCase { @@ -60,14 +61,14 @@ public class SpecialEventTests extends AbstractSerializingTestCase DetectionRule rule = event.toDetectionRule(TimeValue.timeValueSeconds(bucketSpanSecs)); assertEquals(Connective.AND, rule.getConditionsConnective()); - assertEquals(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS, rule.getRuleAction()); + assertEquals(rule.getActions(), EnumSet.of(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING)); assertNull(rule.getTargetFieldName()); assertNull(rule.getTargetFieldValue()); - List conditions = rule.getRuleConditions(); + List conditions = rule.getConditions(); assertEquals(2, conditions.size()); - assertEquals(RuleConditionType.TIME, conditions.get(0).getConditionType()); - assertEquals(RuleConditionType.TIME, conditions.get(1).getConditionType()); + assertEquals(RuleConditionType.TIME, conditions.get(0).getType()); + assertEquals(RuleConditionType.TIME, conditions.get(1).getType()); assertEquals(Operator.GTE, conditions.get(0).getCondition().getOperator()); assertEquals(Operator.LT, conditions.get(1).getCondition().getOperator()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java index d22b34e5369..0933f8780f4 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java @@ -61,7 +61,7 @@ public class DetectionRulesIT extends MlNativeAutodetectIntegTestCase { DetectionRule rule = new DetectionRule.Builder(Arrays.asList(condition1, condition2, condition3)).build(); Detector.Builder detector = new Detector.Builder("max", "value"); - detector.setDetectorRules(Arrays.asList(rule)); + detector.setRules(Arrays.asList(rule)); detector.setByFieldName("by_field"); AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java index 724efa2ab46..850ea920857 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java @@ -431,10 +431,10 @@ public class JobProviderIT extends XPackSingleNodeTestCase { } DetectionRule.Builder rule = new DetectionRule.Builder(conditions) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setConditionsConnective(Connective.OR); - detector.setDetectorRules(Collections.singletonList(rule.build())); + detector.setRules(Collections.singletonList(rule.build())); } return new AnalysisConfig.Builder(Collections.singletonList(detector.build())); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java index 5ebdc4f86ce..16c9242f9d7 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java @@ -461,9 +461,9 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase ruleConditions = new ArrayList<>(size); for (int i = 0; i < size; i++) { // no need for random condition (it is already tested) ruleConditions.addAll(createRule(Double.toString(randomDouble()))); } - return new DetectionRule.Builder(ruleConditions) - .setRuleAction(ruleAction) - .setTargetFieldName(targetFieldName) - .setTargetFieldValue(targetFieldValue) - .setConditionsConnective(connective) - .build(); + DetectionRule.Builder builder = new DetectionRule.Builder(ruleConditions); + + if (randomBoolean()) { + EnumSet actions = EnumSet.noneOf(RuleAction.class); + int actionsCount = randomIntBetween(1, RuleAction.values().length); + for (int i = 0; i < actionsCount; ++i) { + actions.add(randomFrom(RuleAction.values())); + } + builder.setActions(actions); + } + + if (randomBoolean()) { + builder.setConditionsConnective(randomFrom(Connective.values())); + } + + if (randomBoolean()) { + builder.setTargetFieldName(randomAlphaOfLengthBetween(1, 20)); + builder.setTargetFieldValue(randomAlphaOfLengthBetween(1, 20)); + } + + return builder.build(); } @Override @@ -125,16 +133,16 @@ public class DetectionRuleTests extends AbstractSerializingTestCase ruleConditions = instance.getRuleConditions(); - RuleAction ruleAction = instance.getRuleAction(); + List conditions = instance.getConditions(); + EnumSet actions = instance.getActions(); String targetFieldName = instance.getTargetFieldName(); String targetFieldValue = instance.getTargetFieldValue(); Connective connective = instance.getConditionsConnective(); switch (between(0, 3)) { case 0: - ruleConditions = new ArrayList<>(ruleConditions); - ruleConditions.addAll(createRule(Double.toString(randomDouble()))); + conditions = new ArrayList<>(conditions); + conditions.addAll(createRule(Double.toString(randomDouble()))); break; case 1: targetFieldName = randomAlphaOfLengthBetween(5, 10); @@ -156,7 +164,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase { Condition condition = new Condition(Operator.GT, "5"); DetectionRule rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field") + .setActions(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field") .setTargetFieldValue("targetValue") .setConditionsConnective(Connective.AND) .build(); - builder.setDetectorRules(Collections.singletonList(rule)); + builder.setRules(Collections.singletonList(rule)); detector2 = builder.build(); assertFalse(detector1.equals(detector2)); } @@ -84,22 +84,22 @@ public class DetectorTests extends AbstractSerializingTestCase { Condition condition = new Condition(Operator.GT, "5"); DetectionRule rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setTargetFieldName("over_field") .setTargetFieldValue("targetValue") .setConditionsConnective(Connective.AND) .build(); - builder.setDetectorRules(Collections.singletonList(rule)); + builder.setRules(Collections.singletonList(rule)); builder.setByFieldName(null); detector = builder.build(); assertEquals(Collections.singletonList("over_field"), detector.extractAnalysisFields()); builder = new Detector.Builder(detector); rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setConditionsConnective(Connective.AND) .build(); - builder.setDetectorRules(Collections.singletonList(rule)); + builder.setRules(Collections.singletonList(rule)); builder.setOverFieldName(null); detector = builder.build(); assertTrue(detector.extractAnalysisFields().isEmpty()); @@ -107,7 +107,7 @@ public class DetectorTests extends AbstractSerializingTestCase { public void testExtractReferencedLists() { Detector.Builder builder = createDetector(); - builder.setDetectorRules(Arrays.asList( + builder.setRules(Arrays.asList( new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list1"))).build(), new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list2"))).build())); @@ -140,12 +140,12 @@ public class DetectorTests extends AbstractSerializingTestCase { Condition condition = new Condition(Operator.GT, "5"); DetectionRule rule = new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by_field", "val", condition, null))) - .setRuleAction(RuleAction.FILTER_RESULTS) + .setActions(RuleAction.FILTER_RESULTS) .setTargetFieldName("over_field") .setTargetFieldValue("targetValue") .setConditionsConnective(Connective.AND) .build(); - detector.setDetectorRules(Collections.singletonList(rule)); + detector.setRules(Collections.singletonList(rule)); return detector; } @@ -176,15 +176,15 @@ public class DetectorTests extends AbstractSerializingTestCase { } if (randomBoolean()) { int size = randomInt(10); - List detectorRules = new ArrayList<>(size); + List rules = new ArrayList<>(size); for (int i = 0; i < size; i++) { // no need for random DetectionRule (it is already tested) Condition condition = new Condition(Operator.GT, "5"); - detectorRules.add(new DetectionRule.Builder( + rules.add(new DetectionRule.Builder( Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) .setTargetFieldName(fieldName).build()); } - detector.setDetectorRules(detectorRules); + detector.setRules(rules); } if (randomBoolean()) { detector.setUseNull(randomBoolean()); @@ -508,14 +508,14 @@ public class DetectorTests extends AbstractSerializingTestCase { } } - public void testVerify_GivenInvalidDetectionRuleTargetFieldName() { + public void testVerify_GivenInvalidRuleTargetFieldName() { Detector.Builder detector = new Detector.Builder("mean", "metricVale"); detector.setByFieldName("metricName"); detector.setPartitionFieldName("instance"); RuleCondition ruleCondition = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null); DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instancE").build(); - detector.setDetectorRules(Collections.singletonList(rule)); + detector.setRules(Collections.singletonList(rule)); ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); @@ -524,14 +524,14 @@ public class DetectorTests extends AbstractSerializingTestCase { e.getMessage()); } - public void testVerify_GivenValidDetectionRule() { + public void testVerify_GivenValidRule() { Detector.Builder detector = new Detector.Builder("mean", "metricVale"); detector.setByFieldName("metricName"); detector.setPartitionFieldName("instance"); RuleCondition ruleCondition = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null); DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instance").build(); - detector.setDetectorRules(Collections.singletonList(rule)); + detector.setRules(Collections.singletonList(rule)); detector.build(); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java index fa03921f282..7ce8e74ef23 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java @@ -165,7 +165,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase { updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); assertEquals(detectorUpdate.getRules(), - updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorRules()); + updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getRules()); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java index f46bf5a2482..b371471227a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java @@ -11,14 +11,12 @@ import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; -public class -RuleActionTests extends ESTestCase { +public class RuleActionTests extends ESTestCase { public void testForString() { assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("filter_results")); assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("FILTER_RESULTS")); assertEquals(RuleAction.SKIP_SAMPLING, RuleAction.fromString("SKip_sampLing")); - assertEquals(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS, RuleAction.fromString("skip_sampling_and_filter_results")); } public void testToString() { @@ -34,9 +32,9 @@ RuleActionTests extends ESTestCase { } try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(2); + out.writeVInt(1); try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS)); + assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING)); } } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java index 0a38319e169..84e8e2face9 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java @@ -52,7 +52,7 @@ public class RuleConditionTests extends AbstractSerializingTestCase new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, null)); - assertEquals("Invalid detector rule: a categorical rule_condition requires value_filter to be set", e.getMessage()); + assertEquals("Invalid detector rule: a categorical rule_condition requires filter_id to be set", e.getMessage()); } - public void testVerify_GivenNumericalActualWithValueFilter() { + public void testVerify_GivenNumericalActualWithFilterId() { ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); } public void testVerify_GivenNumericalActualWithoutCondition() { @@ -146,10 +146,10 @@ public class RuleConditionTests extends AbstractSerializingTestCase new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); } public void testVerify_GivenNumericalTypicalWithoutCondition() { @@ -158,10 +158,10 @@ public class RuleConditionTests extends AbstractSerializingTestCase new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage()); + assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); } public void testVerify_GivenNumericalDiffAbsWithoutCondition() { @@ -220,12 +220,12 @@ public class RuleConditionTests extends AbstractSerializingTestCase