honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/sql_2 

Original commit: elastic/x-pack-elasticsearch@82985d6481
This commit is contained in:
Nik Everett 2017-12-19 13:43:49 -05:00
parent 2cb58f3dc4 08a35d44c6
commit 7e11a1b388
48 changed files with 1261 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
plugin/bwc-snapshot-dummy-projects/build.gradle
View File

@ -33,6 +33,7 @@ subprojects {
*/
Object esCheckoutPath = """${->
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
esCheckoutDir.mkdirs()
NodeInfo.getShortPathName(esCheckoutDir.toString())
} else {
esCheckoutDir.toString()
@ -41,6 +42,7 @@ subprojects {
File xpackCheckoutDir = file("${esCheckoutDir}-extra/x-pack-elasticsearch")
Object xpackCheckoutPath = """${->
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
xpackCheckoutDir.mkdirs()
NodeInfo.getShortPathName(xpackCheckoutDir.toString())
} else {
xpackCheckoutDir.toString()

11
plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -31,7 +31,6 @@ import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -53,6 +52,7 @@ import org.elasticsearch.xpack.XPackPlugin;
import org.elasticsearch.xpack.XPackSettings;
import org.elasticsearch.xpack.ml.action.CloseJobAction;
import org.elasticsearch.xpack.ml.action.DeleteCalendarAction;
import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction;
import org.elasticsearch.xpack.ml.action.DeleteDatafeedAction;
import org.elasticsearch.xpack.ml.action.DeleteExpiredDataAction;
import org.elasticsearch.xpack.ml.action.DeleteFilterAction;
@ -118,8 +118,10 @@ import org.elasticsearch.xpack.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.rest.RestDeleteExpiredDataAction;
import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarAction;
import org.elasticsearch.xpack.ml.rest.calendar.RestDeleteCalendarJobAction;
import org.elasticsearch.xpack.ml.rest.calendar.RestGetCalendarsAction;
import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarAction;
import org.elasticsearch.xpack.ml.rest.calendar.RestPutCalendarJobAction;
import org.elasticsearch.xpack.ml.rest.datafeeds.RestDeleteDatafeedAction;
import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedStatsAction;
import org.elasticsearch.xpack.ml.rest.datafeeds.RestGetDatafeedsAction;
@ -467,7 +469,9 @@ public class MachineLearning implements ActionPlugin {
new RestForecastJobAction(settings, restController),
new RestGetCalendarsAction(settings, restController),
new RestPutCalendarAction(settings, restController),
new RestDeleteCalendarAction(settings, restController)
new RestDeleteCalendarAction(settings, restController),
new RestDeleteCalendarJobAction(settings, restController),
new RestPutCalendarJobAction(settings, restController)
);
}
@ -516,7 +520,8 @@ public class MachineLearning implements ActionPlugin {
new ActionHandler<>(ForecastJobAction.INSTANCE, ForecastJobAction.TransportAction.class),
new ActionHandler<>(GetCalendarsAction.INSTANCE, GetCalendarsAction.TransportAction.class),
new ActionHandler<>(PutCalendarAction.INSTANCE, PutCalendarAction.TransportAction.class),
new ActionHandler<>(DeleteCalendarAction.INSTANCE, DeleteCalendarAction.TransportAction.class)
new ActionHandler<>(DeleteCalendarAction.INSTANCE, DeleteCalendarAction.TransportAction.class),
new ActionHandler<>(UpdateCalendarJobAction.INSTANCE, UpdateCalendarJobAction.TransportAction.class)
);
}

7
plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.ml;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.calendars.SpecialEvent;
import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings;
@ -32,6 +33,12 @@ public final class MlMetaIndex {
builder.startObject(TYPE);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Calendar.ID.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Calendar.JOB_IDS.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(SpecialEvent.START_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()

4
plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetadata.java
View File

@ -81,6 +81,10 @@ public class MlMetadata implements MetaData.Custom {
return jobs;
}
public boolean isGroupOrJob(String id) {
return groupOrJobLookup.isGroupOrJob(id);
}
public Set<String> expandJobIds(String expression, boolean allowNoJobs) {
return groupOrJobLookup.expandJobIds(expression, allowNoJobs);
}

87
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetCalendarsAction.java
View File

@ -42,6 +42,7 @@ import org.elasticsearch.xpack.ml.MlMetaIndex;
import org.elasticsearch.xpack.ml.action.util.PageParams;
import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import java.io.IOException;
@ -212,16 +213,16 @@ public class GetCalendarsAction extends Action<GetCalendarsAction.Request, GetCa
public static class TransportAction extends HandledTransportAction<Request, Response> {
private final Client client;
private final JobProvider jobProvider;
@Inject
public TransportAction(Settings settings, ThreadPool threadPool,
TransportService transportService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
Client client) {
JobProvider jobProvider) {
super(settings, NAME, threadPool, transportService, actionFilters,
indexNameExpressionResolver, Request::new);
this.client = client;
this.jobProvider = jobProvider;
}
@Override
@ -239,76 +240,24 @@ public class GetCalendarsAction extends Action<GetCalendarsAction.Request, GetCa
}
private void getCalendar(String calendarId, ActionListener<Response> listener) {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId));
executeAsyncWithOrigin(client, ML_ORIGIN, GetAction.INSTANCE, getRequest, new ActionListener<GetResponse>() {
@Override
public void onResponse(GetResponse getDocResponse) {
try {
QueryPage<Calendar> calendars;
if (getDocResponse.isExists()) {
BytesReference docSource = getDocResponse.getSourceAsBytesRef();
try (XContentParser parser =
XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource)) {
Calendar calendar = Calendar.PARSER.apply(parser, null).build();
calendars = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD);
Response response = new Response(calendars);
listener.onResponse(response);
}
} else {
this.onFailure(QueryPage.emptyQueryPage(Calendar.RESULTS_FIELD));
}
} catch (Exception e) {
this.onFailure(e);
}
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
});
jobProvider.calendar(calendarId, ActionListener.wrap(
calendar -> {
QueryPage<Calendar> page = new QueryPage<>(Collections.singletonList(calendar), 1, Calendar.RESULTS_FIELD);
listener.onResponse(new Response(page));
},
listener::onFailure
));
}
private void getCalendars(PageParams pageParams, ActionListener<Response> listener) {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder()
.from(pageParams.getFrom())
.size(pageParams.getSize())
.sort(Calendar.ID.getPreferredName())
.query(QueryBuilders.termQuery(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE));
SearchRequest searchRequest = new SearchRequest(MlMetaIndex.INDEX_NAME)
.indicesOptions(JobProvider.addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS))
.source(sourceBuilder);
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest, new ActionListener<SearchResponse>() {
@Override
public void onResponse(SearchResponse response) {
List<Calendar> docs = new ArrayList<>();
for (SearchHit hit : response.getHits().getHits()) {
BytesReference docSource = hit.getSourceRef();
try (XContentParser parser = XContentFactory.xContent(docSource).createParser(
NamedXContentRegistry.EMPTY, docSource)) {
docs.add(Calendar.PARSER.apply(parser, null).build());
} catch (IOException e) {
this.onFailure(e);
}
}
Response getResponse = new Response(
new QueryPage<>(docs, docs.size(), Calendar.RESULTS_FIELD));
listener.onResponse(getResponse);
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
},
client::search);
CalendarQueryBuilder query = new CalendarQueryBuilder().pageParams(pageParams).sort(true);
jobProvider.calendars(query, ActionListener.wrap(
calendars -> {
listener.onResponse(new Response(calendars));
},
listener::onFailure
));
}
}
}

49
plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutCalendarAction.java
View File

@ -20,7 +20,9 @@ import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
@ -34,13 +36,18 @@ import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.MlMetaIndex;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.watcher.support.Exceptions;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import static org.elasticsearch.action.ValidateActions.addValidationError;
import static org.elasticsearch.xpack.ClientHelper.ML_ORIGIN;
@ -162,36 +169,62 @@ public class PutCalendarAction extends Action<PutCalendarAction.Request, PutCale
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
readAcknowledged(in);
calendar = new Calendar(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
writeAcknowledged(out);
calendar.writeTo(out);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
return calendar.toXContent(builder, params);
}
@Override
public int hashCode() {
return Objects.hash(isAcknowledged(), calendar);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
Response other = (Response) obj;
return Objects.equals(isAcknowledged(), other.isAcknowledged()) && Objects.equals(calendar, other.calendar);
}
}
public static class TransportAction extends HandledTransportAction<Request, Response> {
private final Client client;
private final ClusterService clusterService;
@Inject
public TransportAction(Settings settings, ThreadPool threadPool,
TransportService transportService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, Client client) {
IndexNameExpressionResolver indexNameExpressionResolver,
Client client, ClusterService clusterService) {
super(settings, NAME, threadPool, transportService, actionFilters,
indexNameExpressionResolver, Request::new);
this.client = client;
this.clusterService = clusterService;
}
@Override
protected void doExecute(Request request, ActionListener<Response> listener) {
final Calendar calendar = request.getCalendar();
Calendar calendar = request.getCalendar();
checkJobsExist(calendar.getJobIds(), listener::onFailure);
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
indexRequest.source(calendar.toXContent(builder,
@ -218,5 +251,17 @@ public class PutCalendarAction extends Action<PutCalendarAction.Request, PutCale
}
});
}
private void checkJobsExist(List<String> jobIds, Consumer<Exception> errorHandler) {
ClusterState state = clusterService.state();
MlMetadata mlMetadata = state.getMetaData().custom(MlMetadata.TYPE);
for (String jobId: jobIds) {
Set<String> jobs = mlMetadata.expandJobIds(jobId, true);
if (jobs.isEmpty()) {
errorHandler.accept(ExceptionsHelper.missingJobException(jobId));
return;
}
}
}
}
}

168
plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobAction.java Normal file
View File

@ -0,0 +1,168 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.action;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Objects;
import java.util.Set;
public class UpdateCalendarJobAction extends Action<UpdateCalendarJobAction.Request, PutCalendarAction.Response,
UpdateCalendarJobAction.RequestBuilder> {
public static final UpdateCalendarJobAction INSTANCE = new UpdateCalendarJobAction();
public static final String NAME = "cluster:admin/xpack/ml/calendars/jobs/update";
private UpdateCalendarJobAction() {
super(NAME);
}
@Override
public RequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new RequestBuilder(client);
}
@Override
public PutCalendarAction.Response newResponse() {
return new PutCalendarAction.Response();
}
public static class Request extends ActionRequest {
private String calendarId;
private Set<String> jobIdsToAdd;
private Set<String> jobIdsToRemove;
Request() {
}
public Request(String calendarId, Set<String> jobIdsToAdd, Set<String> jobIdsToRemove) {
this.calendarId = ExceptionsHelper.requireNonNull(calendarId, Calendar.ID.getPreferredName());
this.jobIdsToAdd = ExceptionsHelper.requireNonNull(jobIdsToAdd, "job_ids_to_add");
this.jobIdsToRemove = ExceptionsHelper.requireNonNull(jobIdsToRemove, "job_ids_to_remove");
}
public String getCalendarId() {
return calendarId;
}
public Set<String> getJobIdsToAdd() {
return jobIdsToAdd;
}
public Set<String> getJobIdsToRemove() {
return jobIdsToRemove;
}
@Override
public ActionRequestValidationException validate() {
return null;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
calendarId = in.readString();
jobIdsToAdd = new HashSet<>(in.readList(StreamInput::readString));
jobIdsToRemove = new HashSet<>(in.readList(StreamInput::readString));
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeString(calendarId);
out.writeStringList(new ArrayList<>(jobIdsToAdd));
out.writeStringList(new ArrayList<>(jobIdsToRemove));
}
@Override
public int hashCode() {
return Objects.hash(calendarId, jobIdsToAdd, jobIdsToRemove);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
Request other = (Request) obj;
return Objects.equals(calendarId, other.calendarId) && Objects.equals(jobIdsToAdd, other.jobIdsToAdd)
&& Objects.equals(jobIdsToRemove, other.jobIdsToRemove);
}
}
public static class RequestBuilder extends ActionRequestBuilder<Request, PutCalendarAction.Response, RequestBuilder> {
public RequestBuilder(ElasticsearchClient client) {
super(client, INSTANCE, new Request());
}
}
public static class TransportAction extends HandledTransportAction<Request, PutCalendarAction.Response> {
private final ClusterService clusterService;
private final JobProvider jobProvider;
@Inject
public TransportAction(Settings settings, ThreadPool threadPool,
TransportService transportService, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver,
ClusterService clusterService, JobProvider jobProvider) {
super(settings, NAME, threadPool, transportService, actionFilters,
indexNameExpressionResolver, Request::new);
this.clusterService = clusterService;
this.jobProvider = jobProvider;
}
@Override
protected void doExecute(Request request, ActionListener<PutCalendarAction.Response> listener) {
ClusterState state = clusterService.state();
MlMetadata mlMetadata = state.getMetaData().custom(MlMetadata.TYPE);
for (String jobToAdd: request.getJobIdsToAdd()) {
if (mlMetadata.isGroupOrJob(jobToAdd) == false) {
listener.onFailure(ExceptionsHelper.missingJobException(jobToAdd));
return;
}
}
for (String jobToRemove: request.getJobIdsToRemove()) {
if (mlMetadata.isGroupOrJob(jobToRemove) == false) {
listener.onFailure(ExceptionsHelper.missingJobException(jobToRemove));
return;
}
}
jobProvider.updateCalendar(request.getCalendarId(), request.getJobIdsToAdd(), request.getJobIdsToRemove(),
c -> listener.onResponse(new PutCalendarAction.Response(c)), listener::onFailure);
}
}
}

4
plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/Calendar.java
View File

@ -13,10 +13,8 @@ import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.ml.MlMetaIndex;
import org.elasticsearch.xpack.ml.job.config.MlFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@ -70,7 +68,7 @@ public class Calendar implements ToXContentObject, Writeable {
}
public List<String> getJobIds() {
return new ArrayList<>(jobIds);
return Collections.unmodifiableList(jobIds);
}
@Override

2
plugin/src/main/java/org/elasticsearch/xpack/ml/calendars/SpecialEvent.java
View File

@ -150,7 +150,7 @@ public class SpecialEvent implements ToXContentObject, Writeable {
conditions.add(RuleCondition.createTime(Operator.LT, bucketEndTime));
DetectionRule.Builder builder = new DetectionRule.Builder(conditions);
builder.setRuleAction(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS);
builder.setActions(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING);
builder.setConditionsConnective(Connective.AND);
return builder.build();
}

15
plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
View File

@ -313,7 +313,7 @@ public class JobManager extends AbstractComponent {
String jobId = request.getJobId();
logger.debug("Deleting job '" + jobId + "'");
// Step 3. When the job has been removed from the cluster state, return a response
// Step 4. When the job has been removed from the cluster state, return a response
// -------
CheckedConsumer<Boolean, Exception> apiResponseHandler = jobDeleted -> {
if (jobDeleted) {
@ -325,7 +325,7 @@ public class JobManager extends AbstractComponent {
}
};
// Step 2. When the physical storage has been deleted, remove from Cluster State
// Step 3. When the physical storage has been deleted, remove from Cluster State
// -------
CheckedConsumer<Boolean, Exception> deleteJobStateHandler = response -> clusterService.submitStateUpdateTask("delete-job-" + jobId,
new AckedClusterStateUpdateTask<Boolean>(request, ActionListener.wrap(apiResponseHandler, actionListener::onFailure)) {
@ -351,11 +351,18 @@ public class JobManager extends AbstractComponent {
}
});
// Step 2. Remove the job from any calendars
CheckedConsumer<Boolean, Exception> removeFromCalendarsHandler = response -> {
jobProvider.removeJobFromCalendars(jobId, ActionListener.<Boolean>wrap(deleteJobStateHandler::accept,
actionListener::onFailure ));
};
// Step 1. Delete the physical storage
// This task manages the physical deletion of the job state and results
task.delete(jobId, client, clusterService.state(), deleteJobStateHandler::accept, actionListener::onFailure);
task.delete(jobId, client, clusterService.state(), removeFromCalendarsHandler, actionListener::onFailure);
}
public void revertSnapshot(RevertModelSnapshotAction.Request request, ActionListener<RevertModelSnapshotAction.Response> actionListener,

106
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/DetectionRule.java
View File

@ -21,8 +21,10 @@ import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@ -32,11 +34,11 @@ import java.util.stream.Collectors;
public class DetectionRule implements ToXContentObject, Writeable {
public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule");
public static final ParseField RULE_ACTION_FIELD = new ParseField("rule_action");
public static final ParseField ACTIONS_FIELD = new ParseField("actions");
public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name");
public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value");
public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective");
public static final ParseField RULE_CONDITIONS_FIELD = new ParseField("rule_conditions");
public static final ParseField CONDITIONS_FIELD = new ParseField("conditions");
// These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
public static final ObjectParser<Builder, Void> METADATA_PARSER =
@ -51,12 +53,7 @@ public class DetectionRule implements ToXContentObject, Writeable {
for (MlParserType parserType : MlParserType.values()) {
ObjectParser<Builder, Void> parser = PARSERS.get(parserType);
assert parser != null;
parser.declareField(Builder::setRuleAction, p -> {
if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
return RuleAction.fromString(p.text());
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, RULE_ACTION_FIELD, ValueType.STRING);
parser.declareStringArray(Builder::setActions, ACTIONS_FIELD);
parser.declareString(Builder::setTargetFieldName, TARGET_FIELD_NAME_FIELD);
parser.declareString(Builder::setTargetFieldValue, TARGET_FIELD_VALUE_FIELD);
parser.declareField(Builder::setConditionsConnective, p -> {
@ -65,33 +62,38 @@ public class DetectionRule implements ToXContentObject, Writeable {
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING);
parser.declareObjectArray(Builder::setRuleConditions, (p, c) ->
RuleCondition.PARSERS.get(parserType).apply(p, c), RULE_CONDITIONS_FIELD);
parser.declareObjectArray(Builder::setConditions, (p, c) ->
RuleCondition.PARSERS.get(parserType).apply(p, c), CONDITIONS_FIELD);
}
}
private final RuleAction ruleAction;
private final EnumSet<RuleAction> actions;
private final String targetFieldName;
private final String targetFieldValue;
private final Connective conditionsConnective;
private final List<RuleCondition> ruleConditions;
private final List<RuleCondition> conditions;
private DetectionRule(RuleAction ruleAction, @Nullable String targetFieldName, @Nullable String targetFieldValue,
Connective conditionsConnective, List<RuleCondition> ruleConditions) {
this.ruleAction = Objects.requireNonNull(ruleAction);
private DetectionRule(EnumSet<RuleAction> actions, @Nullable String targetFieldName, @Nullable String targetFieldValue,
Connective conditionsConnective, List<RuleCondition> conditions) {
this.actions = Objects.requireNonNull(actions);
this.targetFieldName = targetFieldName;
this.targetFieldValue = targetFieldValue;
this.conditionsConnective = Objects.requireNonNull(conditionsConnective);
this.ruleConditions = Collections.unmodifiableList(ruleConditions);
this.conditions = Collections.unmodifiableList(conditions);
}
public DetectionRule(StreamInput in) throws IOException {
ruleAction = RuleAction.readFromStream(in);
int actionsCount = in.readVInt();
actions = EnumSet.noneOf(RuleAction.class);
for (int i = 0; i < actionsCount; ++i) {
actions.add(RuleAction.readFromStream(in));
}
conditionsConnective = Connective.readFromStream(in);
int size = in.readVInt();
ruleConditions = new ArrayList<>(size);
conditions = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
ruleConditions.add(new RuleCondition(in));
conditions.add(new RuleCondition(in));
}
targetFieldName = in.readOptionalString();
targetFieldValue = in.readOptionalString();
@ -99,10 +101,14 @@ public class DetectionRule implements ToXContentObject, Writeable {
@Override
public void writeTo(StreamOutput out) throws IOException {
ruleAction.writeTo(out);
out.writeVInt(actions.size());
for (RuleAction action : actions) {
action.writeTo(out);
}
conditionsConnective.writeTo(out);
out.writeVInt(ruleConditions.size());
for (RuleCondition condition : ruleConditions) {
out.writeVInt(conditions.size());
for (RuleCondition condition : conditions) {
condition.writeTo(out);
}
out.writeOptionalString(targetFieldName);
@ -112,9 +118,9 @@ public class DetectionRule implements ToXContentObject, Writeable {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(RULE_ACTION_FIELD.getPreferredName(), ruleAction);
builder.field(ACTIONS_FIELD.getPreferredName(), actions);
builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective);
builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions);
builder.field(CONDITIONS_FIELD.getPreferredName(), conditions);
if (targetFieldName != null) {
builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName);
}
@ -125,8 +131,8 @@ public class DetectionRule implements ToXContentObject, Writeable {
return builder;
}
public RuleAction getRuleAction() {
return ruleAction;
public EnumSet<RuleAction> getActions() {
return actions;
}
@Nullable
@ -143,12 +149,12 @@ public class DetectionRule implements ToXContentObject, Writeable {
return conditionsConnective;
}
public List<RuleCondition> getRuleConditions() {
return ruleConditions;
public List<RuleCondition> getConditions() {
return conditions;
}
public Set<String> extractReferencedFilters() {
return ruleConditions.stream().map(RuleCondition::getValueFilter).filter(Objects::nonNull).collect(Collectors.toSet());
return conditions.stream().map(RuleCondition::getFilterId).filter(Objects::nonNull).collect(Collectors.toSet());
}
@Override
@ -162,34 +168,46 @@ public class DetectionRule implements ToXContentObject, Writeable {
}
DetectionRule other = (DetectionRule) obj;
return Objects.equals(ruleAction, other.ruleAction)
return Objects.equals(actions, other.actions)
&& Objects.equals(targetFieldName, other.targetFieldName)
&& Objects.equals(targetFieldValue, other.targetFieldValue)
&& Objects.equals(conditionsConnective, other.conditionsConnective)
&& Objects.equals(ruleConditions, other.ruleConditions);
&& Objects.equals(conditions, other.conditions);
}
@Override
public int hashCode() {
return Objects.hash(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions);
return Objects.hash(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions);
}
public static class Builder {
private RuleAction ruleAction = RuleAction.FILTER_RESULTS;
private EnumSet<RuleAction> actions = EnumSet.of(RuleAction.FILTER_RESULTS);
private String targetFieldName;
private String targetFieldValue;
private Connective conditionsConnective = Connective.OR;
private List<RuleCondition> ruleConditions = Collections.emptyList();
private List<RuleCondition> conditions = Collections.emptyList();
public Builder(List<RuleCondition> ruleConditions) {
this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_CONDITIONS_FIELD.getPreferredName());
public Builder(List<RuleCondition> conditions) {
this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName());
}
private Builder() {
}
public Builder setRuleAction(RuleAction ruleAction) {
this.ruleAction = ExceptionsHelper.requireNonNull(ruleAction, RULE_ACTION_FIELD.getPreferredName());
public Builder setActions(List<String> actions) {
this.actions.clear();
actions.stream().map(RuleAction::fromString).forEach(this.actions::add);
return this;
}
public Builder setActions(EnumSet<RuleAction> actions) {
this.actions = Objects.requireNonNull(actions, ACTIONS_FIELD.getPreferredName());
return this;
}
public Builder setActions(RuleAction... actions) {
this.actions.clear();
Arrays.stream(actions).forEach(this.actions::add);
return this;
}
@ -208,8 +226,8 @@ public class DetectionRule implements ToXContentObject, Writeable {
return this;
}
public Builder setRuleConditions(List<RuleCondition> ruleConditions) {
this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_ACTION_FIELD.getPreferredName());
public Builder setConditions(List<RuleCondition> conditions) {
this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName());
return this;
}
@ -218,18 +236,18 @@ public class DetectionRule implements ToXContentObject, Writeable {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue);
throw ExceptionsHelper.badRequestException(msg);
}
if (ruleConditions == null || ruleConditions.isEmpty()) {
if (conditions == null || conditions.isEmpty()) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION);
throw ExceptionsHelper.badRequestException(msg);
}
for (RuleCondition condition : ruleConditions) {
if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) {
for (RuleCondition condition : conditions) {
if (condition.getType() == RuleConditionType.CATEGORICAL && targetFieldName != null) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION,
DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName());
throw ExceptionsHelper.badRequestException(msg);
}
}
return new DetectionRule(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions);
return new DetectionRule(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions);
}
}
}

50
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/Detector.java
View File

@ -83,7 +83,7 @@ public class Detector implements ToXContentObject, Writeable {
public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name");
public static final ParseField USE_NULL_FIELD = new ParseField("use_null");
public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent");
public static final ParseField DETECTOR_RULES_FIELD = new ParseField("detector_rules");
public static final ParseField RULES_FIELD = new ParseField("rules");
public static final ParseField DETECTOR_INDEX = new ParseField("detector_index");
// These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
@ -110,8 +110,8 @@ public class Detector implements ToXContentObject, Writeable {
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING);
parser.declareObjectArray(Builder::setDetectorRules, (p, c) ->
DetectionRule.PARSERS.get(parserType).apply(p, c).build(), DETECTOR_RULES_FIELD);
parser.declareObjectArray(Builder::setRules, (p, c) ->
DetectionRule.PARSERS.get(parserType).apply(p, c).build(), RULES_FIELD);
parser.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX);
}
}
@ -329,7 +329,7 @@ public class Detector implements ToXContentObject, Writeable {
private final String partitionFieldName;
private final boolean useNull;
private final ExcludeFrequent excludeFrequent;
private final List<DetectionRule> detectorRules;
private final List<DetectionRule> rules;
private final int detectorIndex;
public Detector(StreamInput in) throws IOException {
@ -341,7 +341,7 @@ public class Detector implements ToXContentObject, Writeable {
partitionFieldName = in.readOptionalString();
useNull = in.readBoolean();
excludeFrequent = in.readBoolean() ? ExcludeFrequent.readFromStream(in) : null;
detectorRules = in.readList(DetectionRule::new);
rules = in.readList(DetectionRule::new);
if (in.getVersion().onOrAfter(Version.V_5_5_0)) {
detectorIndex = in.readInt();
} else {
@ -365,7 +365,7 @@ public class Detector implements ToXContentObject, Writeable {
} else {
out.writeBoolean(false);
}
out.writeList(detectorRules);
out.writeList(rules);
if (out.getVersion().onOrAfter(Version.V_5_5_0)) {
out.writeInt(detectorIndex);
}
@ -394,7 +394,7 @@ public class Detector implements ToXContentObject, Writeable {
if (excludeFrequent != null) {
builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent);
}
builder.field(DETECTOR_RULES_FIELD.getPreferredName(), detectorRules);
builder.field(RULES_FIELD.getPreferredName(), rules);
// negative means "unknown", which should only happen for a 5.4 job
if (detectorIndex >= 0
// no point writing this to cluster state, as the indexes will get reassigned on reload anyway
@ -406,7 +406,7 @@ public class Detector implements ToXContentObject, Writeable {
}
private Detector(String detectorDescription, String function, String fieldName, String byFieldName, String overFieldName,
String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List<DetectionRule> detectorRules,
String partitionFieldName, boolean useNull, ExcludeFrequent excludeFrequent, List<DetectionRule> rules,
int detectorIndex) {
this.function = function;
this.fieldName = fieldName;
@ -415,7 +415,7 @@ public class Detector implements ToXContentObject, Writeable {
this.partitionFieldName = partitionFieldName;
this.useNull = useNull;
this.excludeFrequent = excludeFrequent;
this.detectorRules = Collections.unmodifiableList(detectorRules);
this.rules = Collections.unmodifiableList(rules);
this.detectorDescription = detectorDescription != null ? detectorDescription : DefaultDetectorDescription.of(this);
this.detectorIndex = detectorIndex;
}
@ -491,8 +491,8 @@ public class Detector implements ToXContentObject, Writeable {
return excludeFrequent;
}
public List<DetectionRule> getDetectorRules() {
return detectorRules;
public List<DetectionRule> getRules() {
return rules;
}
/**
@ -514,8 +514,8 @@ public class Detector implements ToXContentObject, Writeable {
}
public Set<String> extractReferencedFilters() {
return detectorRules == null ? Collections.emptySet()
: detectorRules.stream().map(DetectionRule::extractReferencedFilters)
return rules == null ? Collections.emptySet()
: rules.stream().map(DetectionRule::extractReferencedFilters)
.flatMap(Set::stream).collect(Collectors.toSet());
}
@ -556,15 +556,14 @@ public class Detector implements ToXContentObject, Writeable {
Objects.equals(this.partitionFieldName, that.partitionFieldName) &&
Objects.equals(this.useNull, that.useNull) &&
Objects.equals(this.excludeFrequent, that.excludeFrequent) &&
Objects.equals(this.detectorRules, that.detectorRules) &&
Objects.equals(this.rules, that.rules) &&
this.detectorIndex == that.detectorIndex;
}
@Override
public int hashCode() {
return Objects.hash(detectorDescription, function, fieldName, byFieldName,
overFieldName, partitionFieldName, useNull, excludeFrequent,
detectorRules, detectorIndex);
return Objects.hash(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName, useNull,
excludeFrequent, rules, detectorIndex);
}
public static class Builder {
@ -587,7 +586,7 @@ public class Detector implements ToXContentObject, Writeable {
private String partitionFieldName;
private boolean useNull = false;
private ExcludeFrequent excludeFrequent;
private List<DetectionRule> detectorRules = Collections.emptyList();
private List<DetectionRule> rules = Collections.emptyList();
// negative means unknown, and is expected for v5.4 jobs
private int detectorIndex = -1;
@ -603,8 +602,7 @@ public class Detector implements ToXContentObject, Writeable {
partitionFieldName = detector.partitionFieldName;
useNull = detector.useNull;
excludeFrequent = detector.excludeFrequent;
detectorRules = new ArrayList<>(detector.detectorRules.size());
detectorRules.addAll(detector.getDetectorRules());
rules = new ArrayList<>(detector.getRules());
detectorIndex = detector.detectorIndex;
}
@ -645,8 +643,8 @@ public class Detector implements ToXContentObject, Writeable {
this.excludeFrequent = excludeFrequent;
}
public void setDetectorRules(List<DetectionRule> detectorRules) {
this.detectorRules = detectorRules;
public void setRules(List<DetectionRule> rules) {
this.rules = rules;
}
public void setDetectorIndex(int detectorIndex) {
@ -704,12 +702,12 @@ public class Detector implements ToXContentObject, Writeable {
}
String function = this.function == null ? Detector.METRIC : this.function;
if (detectorRules.isEmpty() == false) {
if (rules.isEmpty() == false) {
if (FUNCTIONS_WITHOUT_RULE_SUPPORT.contains(function)) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function);
throw ExceptionsHelper.badRequestException(msg);
}
for (DetectionRule rule : detectorRules) {
for (DetectionRule rule : rules) {
checkScoping(rule);
}
}
@ -764,7 +762,7 @@ public class Detector implements ToXContentObject, Writeable {
}
return new Detector(detectorDescription, function, fieldName, byFieldName, overFieldName, partitionFieldName,
useNull, excludeFrequent, detectorRules, detectorIndex);
useNull, excludeFrequent, rules, detectorIndex);
}
public List<String> extractAnalysisFields() {
@ -802,7 +800,7 @@ public class Detector implements ToXContentObject, Writeable {
String targetFieldName = rule.getTargetFieldName();
checkTargetFieldNameIsValid(extractAnalysisFields(), targetFieldName);
List<String> validOptions = getValidFieldNameOptions(rule);
for (RuleCondition condition : rule.getRuleConditions()) {
for (RuleCondition condition : rule.getConditions()) {
if (!validOptions.contains(condition.getFieldName())) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, validOptions,
condition.getFieldName());

10
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/JobUpdate.java
View File

@ -315,7 +315,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
detectorbuilder.setDetectorDescription(dd.getDescription());
}
if (dd.getRules() != null) {
detectorbuilder.setDetectorRules(dd.getRules());
detectorbuilder.setRules(dd.getRules());
}
ac.getDetectors().set(dd.getDetectorIndex(), detectorbuilder.build());
}
@ -435,13 +435,11 @@ public class JobUpdate implements Writeable, ToXContentObject {
new ConstructingObjectParser<>("detector_update", a -> new DetectorUpdate((int) a[0], (String) a[1],
(List<DetectionRule>) a[2]));
public static final ParseField RULES = new ParseField("rules");
static {
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), Detector.DETECTOR_INDEX);
PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(),
(parser, parseFieldMatcher) -> DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), RULES);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (parser, parseFieldMatcher) ->
DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), Detector.RULES_FIELD);
}
private int detectorIndex;
@ -495,7 +493,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
builder.field(Job.DESCRIPTION.getPreferredName(), description);
}
if (rules != null) {
builder.field(RULES.getPreferredName(), rules);
builder.field(Detector.RULES_FIELD.getPreferredName(), rules);
}
builder.endObject();

3
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleAction.java
View File

@ -14,8 +14,7 @@ import java.util.Locale;
public enum RuleAction implements Writeable {
FILTER_RESULTS,
SKIP_SAMPLING,
SKIP_SAMPLING_AND_FILTER_RESULTS;
SKIP_SAMPLING;
/**
* Case-insensitive from string method.

55
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/RuleCondition.java
View File

@ -26,11 +26,10 @@ import java.util.Map;
import java.util.Objects;
public class RuleCondition implements ToXContentObject, Writeable {
public static final ParseField CONDITION_TYPE_FIELD = new ParseField("condition_type");
public static final ParseField TYPE_FIELD = new ParseField("type");
public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition");
public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name");
public static final ParseField FIELD_VALUE_FIELD = new ParseField("field_value");
public static final ParseField VALUE_FILTER_FIELD = new ParseField("value_filter");
// These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
public static final ConstructingObjectParser<RuleCondition, Void> METADATA_PARSER =
@ -53,60 +52,60 @@ public class RuleCondition implements ToXContentObject, Writeable {
return RuleConditionType.fromString(p.text());
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, CONDITION_TYPE_FIELD, ValueType.STRING);
}, TYPE_FIELD, ValueType.STRING);
parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_NAME_FIELD);
parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_VALUE_FIELD);
parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), Condition.PARSER, Condition.CONDITION_FIELD);
parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), VALUE_FILTER_FIELD);
parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), MlFilter.ID);
}
}
private final RuleConditionType conditionType;
private final RuleConditionType type;
private final String fieldName;
private final String fieldValue;
private final Condition condition;
private final String valueFilter;
private final String filterId;
public RuleCondition(StreamInput in) throws IOException {
conditionType = RuleConditionType.readFromStream(in);
type = RuleConditionType.readFromStream(in);
condition = in.readOptionalWriteable(Condition::new);
fieldName = in.readOptionalString();
fieldValue = in.readOptionalString();
valueFilter = in.readOptionalString();
filterId = in.readOptionalString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
conditionType.writeTo(out);
type.writeTo(out);
out.writeOptionalWriteable(condition);
out.writeOptionalString(fieldName);
out.writeOptionalString(fieldValue);
out.writeOptionalString(valueFilter);
out.writeOptionalString(filterId);
}
RuleCondition(RuleConditionType conditionType, String fieldName, String fieldValue, Condition condition, String valueFilter) {
this.conditionType = conditionType;
RuleCondition(RuleConditionType type, String fieldName, String fieldValue, Condition condition, String filterId) {
this.type = type;
this.fieldName = fieldName;
this.fieldValue = fieldValue;
this.condition = condition;
this.valueFilter = valueFilter;
this.filterId = filterId;
verifyFieldsBoundToType(this);
verifyFieldValueRequiresFieldName(this);
}
public RuleCondition(RuleCondition ruleCondition) {
this.conditionType = ruleCondition.conditionType;
this.type = ruleCondition.type;
this.fieldName = ruleCondition.fieldName;
this.fieldValue = ruleCondition.fieldValue;
this.condition = ruleCondition.condition;
this.valueFilter = ruleCondition.valueFilter;
this.filterId = ruleCondition.filterId;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(CONDITION_TYPE_FIELD.getPreferredName(), conditionType);
builder.field(TYPE_FIELD.getPreferredName(), type);
if (condition != null) {
builder.field(Condition.CONDITION_FIELD.getPreferredName(), condition);
}
@ -116,15 +115,15 @@ public class RuleCondition implements ToXContentObject, Writeable {
if (fieldValue != null) {
builder.field(FIELD_VALUE_FIELD.getPreferredName(), fieldValue);
}
if (valueFilter != null) {
builder.field(VALUE_FILTER_FIELD.getPreferredName(), valueFilter);
if (filterId != null) {
builder.field(MlFilter.ID.getPreferredName(), filterId);
}
builder.endObject();
return builder;
}
public RuleConditionType getConditionType() {
return conditionType;
public RuleConditionType getType() {
return type;
}
/**
@ -153,8 +152,8 @@ public class RuleCondition implements ToXContentObject, Writeable {
* The unique identifier of a filter. Required when the rule type is
* categorical. Should be null for all other types.
*/
public String getValueFilter() {
return valueFilter;
public String getFilterId() {
return filterId;
}
@Override
@ -168,14 +167,14 @@ public class RuleCondition implements ToXContentObject, Writeable {
}
RuleCondition other = (RuleCondition) obj;
return Objects.equals(conditionType, other.conditionType) && Objects.equals(fieldName, other.fieldName)
return Objects.equals(type, other.type) && Objects.equals(fieldName, other.fieldName)
&& Objects.equals(fieldValue, other.fieldValue) && Objects.equals(condition, other.condition)
&& Objects.equals(valueFilter, other.valueFilter);
&& Objects.equals(filterId, other.filterId);
}
@Override
public int hashCode() {
return Objects.hash(conditionType, fieldName, fieldValue, condition, valueFilter);
return Objects.hash(type, fieldName, fieldValue, condition, filterId);
}
public static RuleCondition createCategorical(String fieldName, String valueFilter) {
@ -195,7 +194,7 @@ public class RuleCondition implements ToXContentObject, Writeable {
}
private static void verifyFieldsBoundToType(RuleCondition ruleCondition) throws ElasticsearchParseException {
switch (ruleCondition.getConditionType()) {
switch (ruleCondition.getType()) {
case CATEGORICAL:
verifyCategorical(ruleCondition);
break;
@ -215,7 +214,7 @@ public class RuleCondition implements ToXContentObject, Writeable {
private static void verifyCategorical(RuleCondition ruleCondition) throws ElasticsearchParseException {
checkCategoricalHasNoField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition());
checkCategoricalHasNoField(RuleCondition.FIELD_VALUE_FIELD.getPreferredName(), ruleCondition.getFieldValue());
checkCategoricalHasField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter());
checkCategoricalHasField(MlFilter.ID.getPreferredName(), ruleCondition.getFilterId());
}
private static void checkCategoricalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException {
@ -233,7 +232,7 @@ public class RuleCondition implements ToXContentObject, Writeable {
}
private static void verifyNumerical(RuleCondition ruleCondition) throws ElasticsearchParseException {
checkNumericalHasNoField(RuleCondition.VALUE_FILTER_FIELD.getPreferredName(), ruleCondition.getValueFilter());
checkNumericalHasNoField(MlFilter.ID.getPreferredName(), ruleCondition.getFilterId());
checkNumericalHasField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition());
if (ruleCondition.getFieldName() != null && ruleCondition.getFieldValue() == null) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE);

4
plugin/src/main/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookup.java
View File

@ -59,6 +59,10 @@ public class GroupOrJobLookup {
return new GroupOrJobResolver().expand(expression, allowNoJobs);
}
public boolean isGroupOrJob(String id) {
return groupOrJobLookup.containsKey(id);
}
private class GroupOrJobResolver extends NameResolver {
private GroupOrJobResolver() {

71
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/CalendarQueryBuilder.java Normal file
View File

@ -0,0 +1,71 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.job.persistence;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.TermsQueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.xpack.ml.action.util.PageParams;
import org.elasticsearch.xpack.ml.calendars.Calendar;
public class CalendarQueryBuilder {
private PageParams pageParams = new PageParams(0, 10000);
private String jobId;
private boolean sort = false;
/**
* Page the query result
* @param params The page parameters
* @return this
*/
public CalendarQueryBuilder pageParams(PageParams params) {
this.pageParams = params;
return this;
}
/**
* Query only calendars used by this job
* @param jobId The job Id
* @return this
*/
public CalendarQueryBuilder jobId(String jobId) {
this.jobId = jobId;
return this;
}
/**
* Sort results by calendar_id
* @param sort Sort if true
* @return this
*/
public CalendarQueryBuilder sort(boolean sort) {
this.sort = sort;
return this;
}
public SearchSourceBuilder build() {
QueryBuilder qb;
if (jobId != null) {
qb = new BoolQueryBuilder()
.filter(new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE))
.filter(new TermsQueryBuilder(Calendar.JOB_IDS.getPreferredName(), jobId));
} else {
qb = new TermsQueryBuilder(Calendar.TYPE.getPreferredName(), Calendar.CALENDAR_TYPE);
}
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(qb);
if (sort) {
sourceBuilder.sort(Calendar.ID.getPreferredName());
}
sourceBuilder.from(pageParams.getFrom()).size(pageParams.getSize());
return sourceBuilder;
}
}

143
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java
View File

@ -18,6 +18,11 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.bulk.BulkAction;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.MultiSearchRequestBuilder;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.SearchRequest;
@ -25,6 +30,9 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.block.ClusterBlock;
@ -38,6 +46,7 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
@ -62,6 +71,7 @@ import org.elasticsearch.xpack.ml.action.GetCategoriesAction;
import org.elasticsearch.xpack.ml.action.GetInfluencersAction;
import org.elasticsearch.xpack.ml.action.GetRecordsAction;
import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.calendars.SpecialEvent;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.MlFilter;
@ -87,13 +97,16 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.elasticsearch.xpack.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.ClientHelper.clientWithOrigin;
@ -998,6 +1011,136 @@ public class JobProvider {
, client::search);
}
public void updateCalendar(String calendarId, Set<String> jobIdsToAdd, Set<String> jobIdsToRemove,
Consumer<Calendar> handler, Consumer<Exception> errorHandler) {
ActionListener<Calendar> getCalendarListener = ActionListener.wrap(
calendar -> {
Set<String> currentJobs = new HashSet<>(calendar.getJobIds());
currentJobs.addAll(jobIdsToAdd);
currentJobs.removeAll(jobIdsToRemove);
Calendar updatedCalendar = new Calendar(calendar.getId(), new ArrayList<>(currentJobs));
UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, updatedCalendar.documentId());
updateRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
updateRequest.doc(updatedCalendar.toXContent(builder, ToXContent.EMPTY_PARAMS));
} catch (IOException e) {
throw new IllegalStateException("Failed to serialise calendar with id [" + updatedCalendar.getId() + "]", e);
}
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, updateRequest,
ActionListener.<UpdateResponse>wrap(
response -> {
handler.accept(updatedCalendar);
},
errorHandler)
, client::update);
},
errorHandler
);
calendar(calendarId, getCalendarListener);
}
public void calendars(CalendarQueryBuilder queryBuilder, ActionListener<QueryPage<Calendar>> listener) {
SearchRequest searchRequest = client.prepareSearch(MlMetaIndex.INDEX_NAME)
.setIndicesOptions(IndicesOptions.lenientExpandOpen())
.setSource(queryBuilder.build()).request();
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, searchRequest,
ActionListener.<SearchResponse>wrap(
response -> {
List<Calendar> calendars = new ArrayList<>();
SearchHit[] hits = response.getHits().getHits();
for (SearchHit hit : hits) {
calendars.add(parseSearchHit(hit, Calendar.PARSER, listener::onFailure).build());
}
listener.onResponse(new QueryPage<Calendar>(calendars, response.getHits().getTotalHits(),
Calendar.RESULTS_FIELD));
},
listener::onFailure)
, client::search);
}
public void removeJobFromCalendars(String jobId, ActionListener<Boolean> listener) {
ActionListener<BulkResponse> updateCalandarsListener = ActionListener.wrap(
r -> {
if (r.hasFailures()) {
listener.onResponse(false);
}
listener.onResponse(true);
},
listener::onFailure
);
ActionListener<QueryPage<Calendar>> getCalendarsListener = ActionListener.wrap(
r -> {
BulkRequestBuilder bulkUpdate = client.prepareBulk();
bulkUpdate.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
r.results().stream()
.map(c -> {
Set<String> ids = new HashSet<>(c.getJobIds());
ids.remove(jobId);
return new Calendar(c.getId(), new ArrayList<>(ids));
}).forEach(c -> {
UpdateRequest updateRequest = new UpdateRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE,
c.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
updateRequest.doc(c.toXContent(builder, ToXContent.EMPTY_PARAMS));
} catch (IOException e) {
throw new IllegalStateException("Failed to serialise calendar with id [" + c.getId() + "]", e);
}
bulkUpdate.add(updateRequest);
});
if (bulkUpdate.numberOfActions() > 0) {
executeAsyncWithOrigin(client, ML_ORIGIN, BulkAction.INSTANCE, bulkUpdate.request(), updateCalandarsListener);
} else {
listener.onResponse(true);
}
},
listener::onFailure
);
CalendarQueryBuilder query = new CalendarQueryBuilder().jobId(jobId);
calendars(query, getCalendarsListener);
}
public void calendar(String calendarId, ActionListener<Calendar> listener) {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, Calendar.documentId(calendarId));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getRequest, new ActionListener<GetResponse>() {
@Override
public void onResponse(GetResponse getDocResponse) {
try {
if (getDocResponse.isExists()) {
BytesReference docSource = getDocResponse.getSourceAsBytesRef();
try (XContentParser parser =
XContentFactory.xContent(docSource).createParser(NamedXContentRegistry.EMPTY, docSource)) {
Calendar calendar = Calendar.PARSER.apply(parser, null).build();
listener.onResponse(calendar);
}
} else {
this.onFailure(new ResourceNotFoundException("No calendar with id [" + calendarId + "]"));
}
} catch (Exception e) {
this.onFailure(e);
}
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
},
client::get);
}
private void handleLatestModelSizeStats(String jobId, ModelSizeStats latestModelSizeStats, Consumer<Long> handler,
Consumer<Exception> errorHandler) {
if (latestModelSizeStats != null) {

4
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriter.java
View File

@ -107,8 +107,8 @@ public class FieldConfigWriter {
StringBuilder contents) throws IOException {
List<DetectionRule> rules = new ArrayList<>();
if (detector.getDetectorRules() != null) {
rules.addAll(detector.getDetectorRules());
if (detector.getRules() != null) {
rules.addAll(detector.getRules());
}
rules.addAll(specialEvents);

1
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarAction.java
View File

@ -35,5 +35,4 @@ public class RestDeleteCalendarAction extends BaseRestHandler {
DeleteCalendarAction.Request request = new DeleteCalendarAction.Request(restRequest.param(Calendar.ID.getPreferredName()));
return channel -> client.execute(DeleteCalendarAction.INSTANCE, request, new AcknowledgedRestListener<>(channel));
}
}

44
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestDeleteCalendarJobAction.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.rest.calendar;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.action.AcknowledgedRestListener;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.job.config.Job;
import java.io.IOException;
import java.util.Collections;
public class RestDeleteCalendarJobAction extends BaseRestHandler {
public RestDeleteCalendarJobAction(Settings settings, RestController controller) {
super(settings);
controller.registerHandler(RestRequest.Method.DELETE,
MachineLearning.BASE_PATH + "calendars/{" + Calendar.ID.getPreferredName() + "}/jobs/{" +
Job.ID.getPreferredName() + "}", this);
}
@Override
public String getName() {
return "xpack_ml_delete_calendar_job_action";
}
@Override
protected BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
String calendarId = restRequest.param(Calendar.ID.getPreferredName());
String jobId = restRequest.param(Job.ID.getPreferredName());
UpdateCalendarJobAction.Request request =
new UpdateCalendarJobAction.Request(calendarId, Collections.emptySet(), Collections.singleton(jobId));
return channel -> client.execute(UpdateCalendarJobAction.INSTANCE, request, new AcknowledgedRestListener<>(channel));
}
}

44
plugin/src/main/java/org/elasticsearch/xpack/ml/rest/calendar/RestPutCalendarJobAction.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.rest.calendar;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.action.RestToXContentListener;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.action.UpdateCalendarJobAction;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.job.config.Job;
import java.io.IOException;
import java.util.Collections;
public class RestPutCalendarJobAction extends BaseRestHandler {
public RestPutCalendarJobAction(Settings settings, RestController controller) {
super(settings);
controller.registerHandler(RestRequest.Method.PUT,
MachineLearning.BASE_PATH + "calendars/{" + Calendar.ID.getPreferredName() + "}/jobs/{" +
Job.ID.getPreferredName() + "}", this);
}
@Override
public String getName() {
return "xpack_ml_put_calendar_job_action";
}
@Override
protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException {
String calendarId = restRequest.param(Calendar.ID.getPreferredName());
String jobId = restRequest.param(Job.ID.getPreferredName());
UpdateCalendarJobAction.Request putCalendarRequest =
new UpdateCalendarJobAction.Request(calendarId, Collections.singleton(jobId), Collections.emptySet());
return channel -> client.execute(UpdateCalendarJobAction.INSTANCE, putCalendarRequest, new RestToXContentListener<>(channel));
}
}

3
plugin/src/main/resources/monitoring-beats.json
View File

@ -19,6 +19,9 @@
"type": "date",
"format": "date_time"
},
"interval_ms": {
"type": "long"
},
"type": {
"type": "keyword"
},

3
plugin/src/main/resources/monitoring-kibana.json
View File

@ -19,6 +19,9 @@
"type": "date",
"format": "date_time"
},
"interval_ms": {
"type": "long"
},
"type": {
"type": "keyword"
},

3
plugin/src/main/resources/monitoring-logstash.json
View File

@ -19,6 +19,9 @@
"type": "date",
"format": "date_time"
},
"interval_ms": {
"type": "long"
},
"type": {
"type": "keyword"
},

10
plugin/src/test/java/org/elasticsearch/integration/ldap/AbstractAdLdapRealmTestCase.java
View File

@ -17,10 +17,12 @@ import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.settings.MockSecureSettings;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.SecurityIntegTestCase;
import org.elasticsearch.test.SecuritySettingsSource;
import org.elasticsearch.xpack.security.SecurityLifecycleService;
import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingRequestBuilder;
import org.elasticsearch.xpack.security.action.rolemapping.PutRoleMappingResponse;
import org.elasticsearch.xpack.security.authc.ldap.LdapRealm;
@ -42,6 +44,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -191,6 +194,13 @@ public abstract class AbstractAdLdapRealmTestCase extends SecurityIntegTestCase
super.deleteSecurityIndex();
}
@Override
public Set<String> excludeTemplates() {
Set<String> templates = Sets.newHashSet(super.excludeTemplates());
templates.add(SecurityLifecycleService.SECURITY_TEMPLATE_NAME); // don't remove the security index template
return templates;
}
private List<String> getRoleMappingContent(Function<RoleMappingEntry, String> contentFunction) {
return getRoleMappingContent(contentFunction, AbstractAdLdapRealmTestCase.roleMappings);
}

10
plugin/src/test/java/org/elasticsearch/test/NativeRealmIntegTestCase.java
View File

@ -15,6 +15,8 @@ import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.xpack.security.SecurityLifecycleService;
import org.elasticsearch.xpack.security.authc.support.UsernamePasswordToken;
import org.elasticsearch.xpack.security.client.SecurityClient;
import org.elasticsearch.xpack.security.user.ElasticUser;
@ -26,6 +28,7 @@ import org.junit.Before;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Set;
/**
* Test case with method to handle the starting and stopping the stores for native users and roles
@ -59,6 +62,13 @@ public abstract class NativeRealmIntegTestCase extends SecurityIntegTestCase {
.build();
}
@Override
public Set<String> excludeTemplates() {
Set<String> templates = Sets.newHashSet(super.excludeTemplates());
templates.add(SecurityLifecycleService.SECURITY_TEMPLATE_NAME); // don't remove the security index template
return templates;
}
private SecureString reservedPassword = SecuritySettingsSource.TEST_PASSWORD_SECURE_STRING;
protected SecureString getReservedPassword() {

22
plugin/src/test/java/org/elasticsearch/xpack/ml/action/PutCalendarActionResponseTests.java Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.action;
import org.elasticsearch.test.AbstractStreamableTestCase;
import org.elasticsearch.xpack.ml.calendars.CalendarTests;
public class PutCalendarActionResponseTests extends AbstractStreamableTestCase<PutCalendarAction.Response> {
@Override
protected PutCalendarAction.Response createTestInstance() {
return new PutCalendarAction.Response(CalendarTests.testInstance());
}
@Override
protected PutCalendarAction.Response createBlankInstance() {
return new PutCalendarAction.Response();
}
}

36
plugin/src/test/java/org/elasticsearch/xpack/ml/action/UpdateCalendarJobActionResquestTests.java Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.action;
import org.elasticsearch.test.AbstractStreamableTestCase;
import java.util.HashSet;
import java.util.Set;
public class UpdateCalendarJobActionResquestTests extends AbstractStreamableTestCase<UpdateCalendarJobAction.Request> {
@Override
protected UpdateCalendarJobAction.Request createTestInstance() {
int addSize = randomIntBetween(0, 2);
Set<String> toAdd = new HashSet<>();
for (int i=0; i<addSize; i++) {
toAdd.add(randomAlphaOfLength(10));
}
int removeSize = randomIntBetween(0, 2);
Set<String> toRemove = new HashSet<>();
for (int i=0; i<removeSize; i++) {
toRemove.add(randomAlphaOfLength(10));
}
return new UpdateCalendarJobAction.Request(randomAlphaOfLength(10), toAdd, toRemove);
}
@Override
protected UpdateCalendarJobAction.Request createBlankInstance() {
return new UpdateCalendarJobAction.Request();
}
}

8
plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/CalendarTests.java
View File

@ -18,8 +18,7 @@ import static org.hamcrest.Matchers.equalTo;
public class CalendarTests extends AbstractSerializingTestCase<Calendar> {
@Override
protected Calendar createTestInstance() {
public static Calendar testInstance() {
int size = randomInt(10);
List<String> items = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
@ -28,6 +27,11 @@ public class CalendarTests extends AbstractSerializingTestCase<Calendar> {
return new Calendar(randomAlphaOfLengthBetween(1, 20), items);
}
@Override
protected Calendar createTestInstance() {
return testInstance();
}
@Override
protected Writeable.Reader<Calendar> instanceReader() {
return Calendar::new;

9
plugin/src/test/java/org/elasticsearch/xpack/ml/calendars/SpecialEventTests.java
View File

@ -22,6 +22,7 @@ import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
public class SpecialEventTests extends AbstractSerializingTestCase<SpecialEvent> {
@ -60,14 +61,14 @@ public class SpecialEventTests extends AbstractSerializingTestCase<SpecialEvent>
DetectionRule rule = event.toDetectionRule(TimeValue.timeValueSeconds(bucketSpanSecs));
assertEquals(Connective.AND, rule.getConditionsConnective());
assertEquals(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS, rule.getRuleAction());
assertEquals(rule.getActions(), EnumSet.of(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING));
assertNull(rule.getTargetFieldName());
assertNull(rule.getTargetFieldValue());
List<RuleCondition> conditions = rule.getRuleConditions();
List<RuleCondition> conditions = rule.getConditions();
assertEquals(2, conditions.size());
assertEquals(RuleConditionType.TIME, conditions.get(0).getConditionType());
assertEquals(RuleConditionType.TIME, conditions.get(1).getConditionType());
assertEquals(RuleConditionType.TIME, conditions.get(0).getType());
assertEquals(RuleConditionType.TIME, conditions.get(1).getType());
assertEquals(Operator.GTE, conditions.get(0).getCondition().getOperator());
assertEquals(Operator.LT, conditions.get(1).getCondition().getOperator());

2
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java
View File

@ -61,7 +61,7 @@ public class DetectionRulesIT extends MlNativeAutodetectIntegTestCase {
DetectionRule rule = new DetectionRule.Builder(Arrays.asList(condition1, condition2, condition3)).build();
Detector.Builder detector = new Detector.Builder("max", "value");
detector.setDetectorRules(Arrays.asList(rule));
detector.setRules(Arrays.asList(rule));
detector.setByFieldName("by_field");
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(

190
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java
View File

@ -25,6 +25,8 @@ import org.elasticsearch.xpack.XPackSingleNodeTestCase;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.MlMetaIndex;
import org.elasticsearch.xpack.ml.action.PutJobAction;
import org.elasticsearch.xpack.ml.action.util.QueryPage;
import org.elasticsearch.xpack.ml.calendars.Calendar;
import org.elasticsearch.xpack.ml.calendars.SpecialEvent;
import org.elasticsearch.xpack.ml.job.config.AnalysisConfig;
import org.elasticsearch.xpack.ml.job.config.Connective;
@ -36,6 +38,7 @@ import org.elasticsearch.xpack.ml.job.config.MlFilter;
import org.elasticsearch.xpack.ml.job.config.RuleAction;
import org.elasticsearch.xpack.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder;
import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
@ -56,9 +59,21 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.collection.IsEmptyCollection.empty;
import static org.hamcrest.core.Is.is;
public class JobProviderIT extends XPackSingleNodeTestCase {
@ -97,6 +112,160 @@ public class JobProviderIT extends XPackSingleNodeTestCase {
});
}
public void testGetCalandarByJobId() throws Exception {
List<Calendar> calendars = new ArrayList<>();
calendars.add(new Calendar("empty calendar", Collections.emptyList()));
calendars.add(new Calendar("foo calendar", Collections.singletonList("foo")));
calendars.add(new Calendar("foo bar calendar", Arrays.asList("foo", "bar")));
calendars.add(new Calendar("cat calendar", Collections.singletonList("cat")));
calendars.add(new Calendar("cat foo calendar", Arrays.asList("cat", "foo")));
indexCalendars(calendars);
List<Calendar> queryResult = getCalendars("ted");
assertThat(queryResult, is(empty()));
queryResult = getCalendars("foo");
assertThat(queryResult, hasSize(3));
Long matchedCount = queryResult.stream().filter(
c -> c.getId().equals("foo calendar") || c.getId().equals("foo bar calendar") || c.getId().equals("cat foo calendar"))
.collect(Collectors.counting());
assertEquals(new Long(3), matchedCount);
queryResult = getCalendars("bar");
assertThat(queryResult, hasSize(1));
assertEquals("foo bar calendar", queryResult.get(0).getId());
}
public void testUpdateCalendar() throws Exception {
String calendarId = "empty calendar";
Calendar emptyCal = new Calendar(calendarId, Collections.emptyList());
indexCalendars(Collections.singletonList(emptyCal));
Set<String> addedIds = new HashSet<>();
addedIds.add("foo");
addedIds.add("bar");
updateCalendar(calendarId, addedIds, Collections.emptySet());
Calendar updated = getCalendar(calendarId);
assertEquals(calendarId, updated.getId());
assertEquals(addedIds, new HashSet<>(updated.getJobIds()));
Set<String> removedIds = new HashSet<>();
removedIds.add("foo");
updateCalendar(calendarId, Collections.emptySet(), removedIds);
updated = getCalendar(calendarId);
assertEquals(calendarId, updated.getId());
assertEquals(1, updated.getJobIds().size());
assertEquals("bar", updated.getJobIds().get(0));
}
public void testRemoveJobFromCalendar() throws Exception {
List<Calendar> calendars = new ArrayList<>();
calendars.add(new Calendar("empty calendar", Collections.emptyList()));
calendars.add(new Calendar("foo calendar", Collections.singletonList("foo")));
calendars.add(new Calendar("foo bar calendar", Arrays.asList("foo", "bar")));
calendars.add(new Calendar("cat calendar", Collections.singletonList("cat")));
calendars.add(new Calendar("cat foo calendar", Arrays.asList("cat", "foo")));
indexCalendars(calendars);
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
jobProvider.removeJobFromCalendars("bar", ActionListener.wrap(
r -> latch.countDown(),
exceptionHolder::set));
latch.await();
if (exceptionHolder.get() != null) {
throw exceptionHolder.get();
}
List<Calendar> updatedCalendars = getCalendars(null);
assertEquals(5, updatedCalendars.size());
for (Calendar cal: updatedCalendars) {
assertThat("bar", not(isIn(cal.getJobIds())));
}
Calendar catFoo = getCalendar("cat foo calendar");
assertThat(catFoo.getJobIds(), contains("cat", "foo"));
CountDownLatch latch2 = new CountDownLatch(1);
exceptionHolder = new AtomicReference<>();
jobProvider.removeJobFromCalendars("cat", ActionListener.wrap(
r -> latch2.countDown(),
exceptionHolder::set));
latch2.await();
if (exceptionHolder.get() != null) {
throw exceptionHolder.get();
}
updatedCalendars = getCalendars(null);
assertEquals(5, updatedCalendars.size());
for (Calendar cal: updatedCalendars) {
assertThat("bar", not(isIn(cal.getJobIds())));
assertThat("cat", not(isIn(cal.getJobIds())));
}
}
private List<Calendar> getCalendars(String jobId) throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
AtomicReference<QueryPage<Calendar>> result = new AtomicReference<>();
CalendarQueryBuilder query = new CalendarQueryBuilder();
if (jobId != null) {
query.jobId(jobId);
}
jobProvider.calendars(query, ActionListener.wrap(
r -> {
latch.countDown();
result.set(r);
},
exceptionHolder::set));
latch.await();
if (exceptionHolder.get() != null) {
throw exceptionHolder.get();
}
return result.get().results();
}
private void updateCalendar(String calendarId, Set<String> idsToAdd, Set<String> idsToRemove) throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
jobProvider.updateCalendar(calendarId, idsToAdd, idsToRemove,
r -> latch.countDown(),
exceptionHolder::set);
latch.await();
if (exceptionHolder.get() != null) {
throw exceptionHolder.get();
}
client().admin().indices().prepareRefresh(MlMetaIndex.INDEX_NAME).get();
}
private Calendar getCalendar(String calendarId) throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
AtomicReference<Calendar> calendarHolder = new AtomicReference<>();
jobProvider.calendar(calendarId, ActionListener.wrap(
c -> { latch.countDown(); calendarHolder.set(c); },
exceptionHolder::set)
);
latch.await();
if (exceptionHolder.get() != null) {
throw exceptionHolder.get();
}
return calendarHolder.get();
}
public void testSpecialEvents() throws Exception {
List<SpecialEvent> events = new ArrayList<>();
events.add(new SpecialEvent("A_and_B_downtime", "downtime", createZonedDateTime(1000L), createZonedDateTime(2000L),
@ -262,10 +431,10 @@ public class JobProviderIT extends XPackSingleNodeTestCase {
}
DetectionRule.Builder rule = new DetectionRule.Builder(conditions)
.setRuleAction(RuleAction.FILTER_RESULTS)
.setActions(RuleAction.FILTER_RESULTS)
.setConditionsConnective(Connective.OR);
detector.setDetectorRules(Collections.singletonList(rule.build()));
detector.setRules(Collections.singletonList(rule.build()));
}
return new AnalysisConfig.Builder(Collections.singletonList(detector.build()));
@ -321,7 +490,8 @@ public class JobProviderIT extends XPackSingleNodeTestCase {
for (MlFilter filter : filters) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
indexRequest.source(filter.toXContent(builder, ToXContent.EMPTY_PARAMS));
ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true"));
indexRequest.source(filter.toXContent(builder, params));
bulkRequest.add(indexRequest);
}
}
@ -341,7 +511,21 @@ public class JobProviderIT extends XPackSingleNodeTestCase {
private void indexQuantiles(Quantiles quantiles) {
JobResultsPersister persister = new JobResultsPersister(nodeSettings(), client());
persister.persistQuantiles(quantiles);
}
private void indexCalendars(List<Calendar> calendars) throws IOException {
BulkRequestBuilder bulkRequest = client().prepareBulk();
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for (Calendar calendar: calendars) {
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, calendar.documentId());
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlMetaIndex.INCLUDE_TYPE_KEY, "true"));
indexRequest.source(calendar.toXContent(builder, params));
bulkRequest.add(indexRequest);
}
}
bulkRequest.execute().actionGet();
}
private ZonedDateTime createZonedDateTime(long epochMs) {

4
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/AnalysisConfigTests.java
View File

@ -461,9 +461,9 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
"filter2"))).build();
Detector.Builder detector1 = new Detector.Builder("count", null);
detector1.setByFieldName("foo");
detector1.setDetectorRules(Collections.singletonList(rule1));
detector1.setRules(Collections.singletonList(rule1));
Detector.Builder detector2 = new Detector.Builder("count", null);
detector2.setDetectorRules(Collections.singletonList(rule2));
detector2.setRules(Collections.singletonList(rule2));
detector2.setByFieldName("foo");
AnalysisConfig config = new AnalysisConfig.Builder(
Arrays.asList(detector1.build(), detector2.build(), new Detector.Builder("count", null).build())).build();

50
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectionRuleTests.java
View File

@ -13,6 +13,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
@ -63,7 +64,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
public void testEqualsGivenRules() {
DetectionRule rule1 = createFullyPopulated().build();
DetectionRule rule2 = createFullyPopulated().setRuleConditions(createRule("10")).build();
DetectionRule rule2 = createFullyPopulated().setConditions(createRule("10")).build();
assertFalse(rule1.equals(rule2));
assertFalse(rule2.equals(rule1));
}
@ -78,7 +79,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
private static DetectionRule.Builder createFullyPopulated() {
return new DetectionRule.Builder(createRule("5"))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setActions(EnumSet.of(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING))
.setTargetFieldName("targetField")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND);
@ -91,26 +92,33 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
@Override
protected DetectionRule createTestInstance() {
RuleAction ruleAction = randomFrom(RuleAction.values());
String targetFieldName = null;
String targetFieldValue = null;
Connective connective = randomFrom(Connective.values());
if (randomBoolean()) {
targetFieldName = randomAlphaOfLengthBetween(1, 20);
targetFieldValue = randomAlphaOfLengthBetween(1, 20);
}
int size = 1 + randomInt(20);
List<RuleCondition> ruleConditions = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
// no need for random condition (it is already tested)
ruleConditions.addAll(createRule(Double.toString(randomDouble())));
}
return new DetectionRule.Builder(ruleConditions)
.setRuleAction(ruleAction)
.setTargetFieldName(targetFieldName)
.setTargetFieldValue(targetFieldValue)
.setConditionsConnective(connective)
.build();
DetectionRule.Builder builder = new DetectionRule.Builder(ruleConditions);
if (randomBoolean()) {
EnumSet<RuleAction> actions = EnumSet.noneOf(RuleAction.class);
int actionsCount = randomIntBetween(1, RuleAction.values().length);
for (int i = 0; i < actionsCount; ++i) {
actions.add(randomFrom(RuleAction.values()));
}
builder.setActions(actions);
}
if (randomBoolean()) {
builder.setConditionsConnective(randomFrom(Connective.values()));
}
if (randomBoolean()) {
builder.setTargetFieldName(randomAlphaOfLengthBetween(1, 20));
builder.setTargetFieldValue(randomAlphaOfLengthBetween(1, 20));
}
return builder.build();
}
@Override
@ -125,16 +133,16 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
@Override
protected DetectionRule mutateInstance(DetectionRule instance) throws IOException {
List<RuleCondition> ruleConditions = instance.getRuleConditions();
RuleAction ruleAction = instance.getRuleAction();
List<RuleCondition> conditions = instance.getConditions();
EnumSet<RuleAction> actions = instance.getActions();
String targetFieldName = instance.getTargetFieldName();
String targetFieldValue = instance.getTargetFieldValue();
Connective connective = instance.getConditionsConnective();
switch (between(0, 3)) {
case 0:
ruleConditions = new ArrayList<>(ruleConditions);
ruleConditions.addAll(createRule(Double.toString(randomDouble())));
conditions = new ArrayList<>(conditions);
conditions.addAll(createRule(Double.toString(randomDouble())));
break;
case 1:
targetFieldName = randomAlphaOfLengthBetween(5, 10);
@ -156,7 +164,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
throw new AssertionError("Illegal randomisation branch");
}
return new DetectionRule.Builder(ruleConditions).setRuleAction(ruleAction).setTargetFieldName(targetFieldName)
return new DetectionRule.Builder(conditions).setActions(actions).setTargetFieldName(targetFieldName)
.setTargetFieldValue(targetFieldValue).setConditionsConnective(connective).build();
}
}

32
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/DetectorTests.java
View File

@ -64,11 +64,11 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
Condition condition = new Condition(Operator.GT, "5");
DetectionRule rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field")
.setActions(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND)
.build();
builder.setDetectorRules(Collections.singletonList(rule));
builder.setRules(Collections.singletonList(rule));
detector2 = builder.build();
assertFalse(detector1.equals(detector2));
}
@ -84,22 +84,22 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
Condition condition = new Condition(Operator.GT, "5");
DetectionRule rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setActions(RuleAction.FILTER_RESULTS)
.setTargetFieldName("over_field")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND)
.build();
builder.setDetectorRules(Collections.singletonList(rule));
builder.setRules(Collections.singletonList(rule));
builder.setByFieldName(null);
detector = builder.build();
assertEquals(Collections.singletonList("over_field"), detector.extractAnalysisFields());
builder = new Detector.Builder(detector);
rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setActions(RuleAction.FILTER_RESULTS)
.setConditionsConnective(Connective.AND)
.build();
builder.setDetectorRules(Collections.singletonList(rule));
builder.setRules(Collections.singletonList(rule));
builder.setOverFieldName(null);
detector = builder.build();
assertTrue(detector.extractAnalysisFields().isEmpty());
@ -107,7 +107,7 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
public void testExtractReferencedLists() {
Detector.Builder builder = createDetector();
builder.setDetectorRules(Arrays.asList(
builder.setRules(Arrays.asList(
new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list1"))).build(),
new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list2"))).build()));
@ -140,12 +140,12 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
Condition condition = new Condition(Operator.GT, "5");
DetectionRule rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by_field", "val", condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setActions(RuleAction.FILTER_RESULTS)
.setTargetFieldName("over_field")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND)
.build();
detector.setDetectorRules(Collections.singletonList(rule));
detector.setRules(Collections.singletonList(rule));
return detector;
}
@ -176,15 +176,15 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
}
if (randomBoolean()) {
int size = randomInt(10);
List<DetectionRule> detectorRules = new ArrayList<>(size);
List<DetectionRule> rules = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
// no need for random DetectionRule (it is already tested)
Condition condition = new Condition(Operator.GT, "5");
detectorRules.add(new DetectionRule.Builder(
rules.add(new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setTargetFieldName(fieldName).build());
}
detector.setDetectorRules(detectorRules);
detector.setRules(rules);
}
if (randomBoolean()) {
detector.setUseNull(randomBoolean());
@ -508,14 +508,14 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
}
}
public void testVerify_GivenInvalidDetectionRuleTargetFieldName() {
public void testVerify_GivenInvalidRuleTargetFieldName() {
Detector.Builder detector = new Detector.Builder("mean", "metricVale");
detector.setByFieldName("metricName");
detector.setPartitionFieldName("instance");
RuleCondition ruleCondition =
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null);
DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instancE").build();
detector.setDetectorRules(Collections.singletonList(rule));
detector.setRules(Collections.singletonList(rule));
ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build);
@ -524,14 +524,14 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
e.getMessage());
}
public void testVerify_GivenValidDetectionRule() {
public void testVerify_GivenValidRule() {
Detector.Builder detector = new Detector.Builder("mean", "metricVale");
detector.setByFieldName("metricName");
detector.setPartitionFieldName("instance");
RuleCondition ruleCondition =
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null);
DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instance").build();
detector.setDetectorRules(Collections.singletonList(rule));
detector.setRules(Collections.singletonList(rule));
detector.build();
}

2
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/JobUpdateTests.java
View File

@ -165,7 +165,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription());
assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription());
assertEquals(detectorUpdate.getRules(),
updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorRules());
updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getRules());
}
}

8
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java
View File

@ -11,14 +11,12 @@ import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.equalTo;
public class
RuleActionTests extends ESTestCase {
public class RuleActionTests extends ESTestCase {
public void testForString() {
assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("filter_results"));
assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("FILTER_RESULTS"));
assertEquals(RuleAction.SKIP_SAMPLING, RuleAction.fromString("SKip_sampLing"));
assertEquals(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS, RuleAction.fromString("skip_sampling_and_filter_results"));
}
public void testToString() {
@ -34,9 +32,9 @@ RuleActionTests extends ESTestCase {
}
try (BytesStreamOutput out = new BytesStreamOutput()) {
out.writeVInt(2);
out.writeVInt(1);
try (StreamInput in = out.bytes().streamInput()) {
assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING_AND_FILTER_RESULTS));
assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING));
}
}
}

26
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTests.java
View File

@ -52,7 +52,7 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
public void testConstructor() {
RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter");
assertEquals(RuleConditionType.CATEGORICAL, condition.getConditionType());
assertEquals(RuleConditionType.CATEGORICAL, condition.getType());
assertNull(condition.getFieldName());
assertNull(condition.getFieldValue());
assertNull(condition.getCondition());
@ -122,16 +122,16 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
assertEquals("Invalid detector rule: a categorical rule_condition does not support field_value", e.getMessage());
}
public void testVerify_GivenCategoricalWithoutValueFilter() {
public void testVerify_GivenCategoricalWithoutFilterId() {
ElasticsearchException e = expectThrows(ElasticsearchException.class,
() -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, null));
assertEquals("Invalid detector rule: a categorical rule_condition requires value_filter to be set", e.getMessage());
assertEquals("Invalid detector rule: a categorical rule_condition requires filter_id to be set", e.getMessage());
}
public void testVerify_GivenNumericalActualWithValueFilter() {
public void testVerify_GivenNumericalActualWithFilterId() {
ElasticsearchException e = expectThrows(ElasticsearchException.class,
() -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter"));
assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage());
assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage());
}
public void testVerify_GivenNumericalActualWithoutCondition() {
@ -146,10 +146,10 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
assertEquals("Invalid detector rule: a numerical rule_condition with field_name requires that field_value is set", e.getMessage());
}
public void testVerify_GivenNumericalTypicalWithValueFilter() {
public void testVerify_GivenNumericalTypicalWithFilterId() {
ElasticsearchException e = expectThrows(ElasticsearchException.class,
() -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter"));
assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage());
assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage());
}
public void testVerify_GivenNumericalTypicalWithoutCondition() {
@ -158,10 +158,10 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage());
}
public void testVerify_GivenNumericalDiffAbsWithValueFilter() {
public void testVerify_GivenNumericalDiffAbsWithFilterId() {
ElasticsearchException e = expectThrows(ElasticsearchException.class,
() -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, "myFilter"));
assertEquals("Invalid detector rule: a numerical rule_condition does not support value_filter", e.getMessage());
assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage());
}
public void testVerify_GivenNumericalDiffAbsWithoutCondition() {
@ -220,12 +220,12 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
public void testCreateTimeBased() {
RuleCondition timeBased = RuleCondition.createTime(Operator.GTE, 100L);
assertEquals(RuleConditionType.TIME, timeBased.getConditionType());
assertEquals(RuleConditionType.TIME, timeBased.getType());
assertEquals(Operator.GTE, timeBased.getCondition().getOperator());
assertEquals("100", timeBased.getCondition().getValue());
assertNull(timeBased.getFieldName());
assertNull(timeBased.getFieldValue());
assertNull(timeBased.getValueFilter());
assertNull(timeBased.getFilterId());
}
public void testCreateTimeBased_GivenOperatorMatch() {
@ -237,11 +237,11 @@ public class RuleConditionTests extends AbstractSerializingTestCase<RuleConditio
public void testCreateNumerical() {
RuleCondition ruleCondition = RuleCondition.createNumerical(RuleConditionType.NUMERICAL_ACTUAL, "foo", "bar",
new Condition(Operator.GTE, "100"));
assertEquals(RuleConditionType.NUMERICAL_ACTUAL, ruleCondition.getConditionType());
assertEquals(RuleConditionType.NUMERICAL_ACTUAL, ruleCondition.getType());
assertEquals(Operator.GTE, ruleCondition.getCondition().getOperator());
assertEquals("100", ruleCondition.getCondition().getValue());
assertEquals("foo", ruleCondition.getFieldName());
assertEquals("bar", ruleCondition.getFieldValue());
assertNull(ruleCondition.getValueFilter());
assertNull(ruleCondition.getFilterId());
}
}

14
plugin/src/test/java/org/elasticsearch/xpack/ml/job/groups/GroupOrJobLookupTests.java
View File

@ -89,6 +89,20 @@ public class GroupOrJobLookupTests extends ESTestCase {
assertThat(groupOrJobLookup.expandJobIds("foo-group,*-2", false), contains("bar-2", "foo-1", "foo-2"));
}
public void testIsGroupOrJob() {
List<Job> jobs = new ArrayList<>();
jobs.add(mockJob("foo-1", Arrays.asList("foo-group", "ones")));
jobs.add(mockJob("foo-2", Arrays.asList("foo-group", "twos")));
jobs.add(mockJob("bar-1", Arrays.asList("bar-group", "ones")));
jobs.add(mockJob("nogroup", Collections.emptyList()));
GroupOrJobLookup groupOrJobLookup = new GroupOrJobLookup(jobs);
assertTrue(groupOrJobLookup.isGroupOrJob("foo-1"));
assertTrue(groupOrJobLookup.isGroupOrJob("twos"));
assertTrue(groupOrJobLookup.isGroupOrJob("nogroup"));
assertFalse(groupOrJobLookup.isGroupOrJob("missing"));
}
private static Job mockJob(String jobId, List<String> groups) {
Job job = mock(Job.class);
when(job.getId()).thenReturn(jobId);

14
plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.ml.job.process.autodetect.writer;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.calendars.SpecialEvent;
import org.elasticsearch.xpack.ml.job.config.Condition;
import org.elasticsearch.xpack.ml.job.config.Connective;
import org.elasticsearch.xpack.ml.job.config.DetectionRule;
@ -22,13 +21,8 @@ import org.mockito.InOrder;
import org.mockito.Mockito;
import java.io.IOException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.stream.IntStream;
@ -202,11 +196,11 @@ public class ControlMsgToProcessWriterTests extends ESTestCase {
inOrder.verify(lengthEncodedWriter).writeNumFields(4);
inOrder.verify(lengthEncodedWriter, times(3)).writeField("");
inOrder.verify(lengthEncodedWriter).writeField("u[detectorRules]\ndetectorIndex=2\n" +
"rulesJson=[{\"rule_action\":\"filter_results\",\"conditions_connective\":\"and\",\"rule_conditions\":" +
"[{\"condition_type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"rulesJson=[{\"actions\":[\"filter_results\"],\"conditions_connective\":\"and\",\"conditions\":" +
"[{\"type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"\"target_field_name\":\"targetField1\",\"target_field_value\":\"targetValue\"}," +
"{\"rule_action\":\"filter_results\",\"conditions_connective\":\"and\",\"rule_conditions\":[" +
"{\"condition_type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"{\"actions\":[\"filter_results\"],\"conditions_connective\":\"and\",\"conditions\":[" +
"{\"type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"\"target_field_name\":\"targetField2\",\"target_field_value\":\"targetValue\"}]");
verifyNoMoreInteractions(lengthEncodedWriter);
}

16
plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java
View File

@ -26,7 +26,6 @@ import org.mockito.ArgumentCaptor;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
@ -36,7 +35,6 @@ import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
@ -195,7 +193,7 @@ public class FieldConfigWriterTests extends ESTestCase {
RuleCondition ruleCondition = RuleCondition.createNumerical
(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricValue", new Condition(Operator.LT, "5"));
DetectionRule rule = new DetectionRule.Builder(Arrays.asList(ruleCondition)).setTargetFieldName("instance").build();
detector.setDetectorRules(Arrays.asList(rule));
detector.setRules(Arrays.asList(rule));
AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Arrays.asList(detector.build()));
analysisConfig = builder.build();
@ -251,12 +249,12 @@ public class FieldConfigWriterTests extends ESTestCase {
createFieldConfigWriter().write();
verify(writer).write("detector.0.clause = count\n" +
"detector.0.rules = [{\"rule_action\":\"skip_sampling_and_filter_results\",\"conditions_connective\":\"and\"," +
"\"rule_conditions\":[{\"condition_type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1511395200\"}}," +
"{\"condition_type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515369600\"}}]}," +
"{\"rule_action\":\"skip_sampling_and_filter_results\",\"conditions_connective\":\"and\"," +
"\"rule_conditions\":[{\"condition_type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1519603200\"}}," +
"{\"condition_type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1519862400\"}}]}]" +
"detector.0.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," +
"\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1511395200\"}}," +
"{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515369600\"}}]}," +
"{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," +
"\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1519603200\"}}," +
"{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1519862400\"}}]}]" +
"\n");
verifyNoMoreInteractions(writer);

22
plugin/src/test/resources/rest-api-spec/api/xpack.ml.delete_calendar_job.json Normal file
View File

@ -0,0 +1,22 @@
{
"xpack.ml.delete_calendar_job": {
"methods": [ "DELETE" ],
"url": {
"path": "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}",
"paths": [ "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}" ],
"parts": {
"calendar_id": {
"type" : "string",
"required" : true,
"description" : "The ID of the calendar to modify"
},
"job_id": {
"type": "string",
"required": true,
"description": "The ID of the job to remove from the calendar"
}
}
},
"body": null
}
}

22
plugin/src/test/resources/rest-api-spec/api/xpack.ml.put_calendar_job.json Normal file
View File

@ -0,0 +1,22 @@
{
"xpack.ml.put_calendar_job": {
"methods": [ "PUT" ],
"url": {
"path": "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}",
"paths": [ "/_xpack/ml/calendars/{calendar_id}/jobs/{job_id}" ],
"parts": {
"calendar_id": {
"type": "string",
"required": true,
"description": "The ID of the calendar to modify"
},
"job_id": {
"type": "string",
"required": true,
"description": "The ID of the job to add to the calendar"
}
}
},
"body": null
}
}

152
plugin/src/test/resources/rest-api-spec/test/ml/calendar_crud.yml
View File

@ -1,25 +1,52 @@
---
"Test calendar CRUD":
- do:
xpack.ml.put_job:
job_id: cal-job
body: >
{
"analysis_config" : {
"detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
},
"data_description" : {
}
}
- match: { job_id: "cal-job" }
- do:
xpack.ml.put_job:
job_id: cal-job2
body: >
{
"analysis_config" : {
"detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
},
"data_description" : {
}
}
- match: { job_id: "cal-job2" }
- do:
xpack.ml.put_calendar:
calendar_id: "advent"
body: >
{
"job_ids": ["abc", "xyz"]
"job_ids": ["cal-job", "cal-job2"]
}
- match: { calendar_id: advent }
- match: { job_ids.0: abc }
- match: { job_ids.1: xyz }
- match: { job_ids.0: cal-job }
- match: { job_ids.1: cal-job2 }
- do:
xpack.ml.get_calendars:
calendar_id: "advent"
- match: { count: 1 }
- match: { count: 1 }
- match:
calendars.0:
calendar_id: "advent"
job_ids: ["abc", "xyz"]
job_ids: ["cal-job", "cal-job2"]
- is_false: type
- do:
@ -27,7 +54,7 @@
calendar_id: "Dogs of the Year"
body: >
{
"job_ids": ["abc2"]
"job_ids": ["cal-job"]
}
- do:
@ -51,6 +78,15 @@
xpack.ml.get_calendars:
calendar_id: "Dogs of the Year"
- do:
catch: missing
xpack.ml.put_calendar:
calendar_id: "new cal with unknown job"
body: >
{
"job_ids": ["cal-job", "unknown-job"]
}
---
"Test PageParams":
- do:
@ -66,14 +102,16 @@
- do:
xpack.ml.get_calendars:
from: 2
- match: { count: 1 }
- match: { count: 3 }
- length: { calendars: 1}
- match: { calendars.0.calendar_id: Calendar3 }
- do:
xpack.ml.get_calendars:
from: 1
size: 1
- match: { count: 1 }
- match: { count: 3 }
- length: { calendars: 1}
- match: { calendars.0.calendar_id: Calendar2 }
---
@ -90,10 +128,6 @@
- do:
xpack.ml.put_calendar:
calendar_id: "Mayan"
body: >
{
"job_ids": ["apocalypse"]
}
- do:
catch: /version_conflict_engine_exception/
@ -106,3 +140,97 @@
catch: bad_request
xpack.ml.put_calendar:
calendar_id: "_all"
---
"Test deleted job is removed from calendar":
- do:
xpack.ml.put_job:
job_id: cal-crud-test-delete
body: >
{
"analysis_config" : {
"detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
},
"data_description" : {
}
}
- match: { job_id: "cal-crud-test-delete" }
- do:
xpack.ml.put_calendar:
calendar_id: "delete-test"
body: >
{
"job_ids": ["cal-crud-test-delete"]
}
- do:
xpack.ml.delete_job:
job_id: cal-crud-test-delete
- match: { acknowledged: true }
- do:
xpack.ml.get_calendars:
calendar_id: "delete-test"
- match: { count: 1 }
- match: { calendars.0.job_ids: [] }
---
"Test update calendar":
- do:
xpack.ml.put_calendar:
calendar_id: "Wildlife"
- do:
xpack.ml.put_job:
job_id: tiger
body: >
{
"analysis_config" : {
"detectors" :[{"function":"metric","field_name":"responsetime","by_field_name":"airline"}]
},
"data_description" : {
}
}
- match: { job_id: "tiger" }
- do:
xpack.ml.put_calendar_job:
calendar_id: "Wildlife"
job_id: "tiger"
- match: { calendar_id: "Wildlife" }
- match: { job_ids.0: "tiger" }
- do:
xpack.ml.get_calendars:
calendar_id: "Wildlife"
- match: { count: 1 }
- match: { calendars.0.calendar_id: "Wildlife" }
- length: { calendars.0.job_ids: 1 }
- match: { calendars.0.job_ids.0: "tiger" }
- do:
xpack.ml.delete_calendar_job:
calendar_id: "Wildlife"
job_id: "tiger"
- do:
xpack.ml.get_calendars:
calendar_id: "Wildlife"
- match: { count: 1 }
- match: { calendars.0.calendar_id: "Wildlife" }
- length: { calendars.0.job_ids: 0 }
- do:
catch: missing
xpack.ml.put_calendar_job:
calendar_id: "Wildlife"
job_id: "missing job"
- do:
catch: missing
xpack.ml.delete_calendar_job:
calendar_id: "Wildlife"
job_id: "missing job"

8
plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
View File

@ -152,12 +152,12 @@ setup:
"analysis_config" : {
"bucket_span": "3600s",
"detectors" :[{"function":"mean","field_name":"airline",
"detector_rules": [
"rules": [
{
"rule_conditions": [
"conditions": [
{
"condition_type": "categorical",
"value_filter": "filter-foo"
"type": "categorical",
"filter_id": "filter-foo"
}
]
}

6
plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml
View File

@ -301,8 +301,8 @@
"groups": ["group-1", "group-2"],
"description":"Post update description",
"detectors": [{"detector_index": 0, "rules": {"target_field_name": "airline",
"rule_conditions": [ { "condition_type": "numerical_actual",
"condition": {"operator": "gt", "value": "10" } } ] } },
"conditions": [ { "type": "numerical_actual",
"condition": {"operator": "gt", "value": "10" } } ] } },
{"detector_index": 1, "description": "updated description"}],
"model_plot_config": {
"enabled": false,
@ -327,7 +327,7 @@
- match: { model_plot_config.terms: "foobar" }
- match: { analysis_limits.model_memory_limit: "20mb" }
- match: { analysis_config.categorization_filters: ["cat3.*"] }
- match: { analysis_config.detectors.0.detector_rules.0.target_field_name: "airline" }
- match: { analysis_config.detectors.0.rules.0.target_field_name: "airline" }
- match: { analysis_config.detectors.0.detector_index: 0 }
- match: { analysis_config.detectors.1.detector_description: "updated description" }
- match: { analysis_config.detectors.1.detector_index: 1 }

3
qa/tribe-tests-with-security/src/test/java/org/elasticsearch/xpack/security/SecurityTribeTests.java
View File

@ -20,6 +20,7 @@ import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.IndexNotFoundException;
@ -141,7 +142,7 @@ public class SecurityTribeTests extends NativeRealmIntegTestCase {
public void tearDownTribeNodeAndWipeCluster() throws Exception {
if (cluster2 != null) {
try {
cluster2.wipe(Collections.emptySet());
cluster2.wipe(Collections.singleton(SecurityLifecycleService.SECURITY_TEMPLATE_NAME));
try {
// this is a hack to clean up the .security index since only the XPackSecurity user or superusers can delete it
final Client cluster2Client = cluster2.client().filterWithHeader(Collections.singletonMap("Authorization",