[ML] Fix detector rules and add integ test (elastic/x-pack-elasticsearch#1084)

Relates elastic/x-pack-elasticsearch#882

Original commit: elastic/x-pack-elasticsearch@fd1cc0d402
This commit is contained in:
Dimitris Athanasiou 2017-04-19 12:23:38 +01:00 committed by GitHub
parent 618341db6c
commit 9865d5b955
14 changed files with 431 additions and 157 deletions

View File

@ -6,15 +6,17 @@
package org.elasticsearch.xpack.ml.job.config;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser.ValueType;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException;
import java.util.ArrayList;
@ -25,41 +27,52 @@ import java.util.Set;
import java.util.stream.Collectors;
public class DetectionRule extends ToXContentToBytes implements Writeable {
public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule");
public static final ParseField RULE_ACTION_FIELD = new ParseField("rule_action");
public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name");
public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value");
public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective");
public static final ParseField RULE_CONDITIONS_FIELD = new ParseField("rule_conditions");
public static final ConstructingObjectParser<DetectionRule, Void> PARSER = new ConstructingObjectParser<>(
DETECTION_RULE_FIELD.getPreferredName(),
arr -> {
@SuppressWarnings("unchecked")
List<RuleCondition> rules = (List<RuleCondition>) arr[3];
return new DetectionRule((String) arr[0], (String) arr[1], (Connective) arr[2], rules);
}
);
public static final ObjectParser<Builder, Void> PARSER = new ObjectParser<>(DETECTION_RULE_FIELD.getPreferredName(), Builder::new);
static {
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_FIELD_NAME_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TARGET_FIELD_VALUE_FIELD);
PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> {
PARSER.declareField(Builder::setRuleAction, p -> {
if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
return RuleAction.fromString(p.text());
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, RULE_ACTION_FIELD, ValueType.STRING);
PARSER.declareString(Builder::setTargetFieldName, TARGET_FIELD_NAME_FIELD);
PARSER.declareString(Builder::setTargetFieldValue, TARGET_FIELD_VALUE_FIELD);
PARSER.declareField(Builder::setConditionsConnective, p -> {
if (p.currentToken() == XContentParser.Token.VALUE_STRING) {
return Connective.fromString(p.text());
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(),
PARSER.declareObjectArray(Builder::setRuleConditions,
(parser, parseFieldMatcher) -> RuleCondition.PARSER.apply(parser, parseFieldMatcher), RULE_CONDITIONS_FIELD);
}
private final RuleAction ruleAction = RuleAction.FILTER_RESULTS;
private final RuleAction ruleAction;
private final String targetFieldName;
private final String targetFieldValue;
private final Connective conditionsConnective;
private final List<RuleCondition> ruleConditions;
private DetectionRule(RuleAction ruleAction, @Nullable String targetFieldName, @Nullable String targetFieldValue,
Connective conditionsConnective, List<RuleCondition> ruleConditions) {
this.ruleAction = Objects.requireNonNull(ruleAction);
this.targetFieldName = targetFieldName;
this.targetFieldValue = targetFieldValue;
this.conditionsConnective = Objects.requireNonNull(conditionsConnective);
this.ruleConditions = Collections.unmodifiableList(ruleConditions);
}
public DetectionRule(StreamInput in) throws IOException {
ruleAction = RuleAction.readFromStream(in);
conditionsConnective = Connective.readFromStream(in);
int size = in.readVInt();
ruleConditions = new ArrayList<>(size);
@ -72,6 +85,7 @@ public class DetectionRule extends ToXContentToBytes implements Writeable {
@Override
public void writeTo(StreamOutput out) throws IOException {
ruleAction.writeTo(out);
conditionsConnective.writeTo(out);
out.writeVInt(ruleConditions.size());
for (RuleCondition condition : ruleConditions) {
@ -84,50 +98,29 @@ public class DetectionRule extends ToXContentToBytes implements Writeable {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(RULE_ACTION_FIELD.getPreferredName(), ruleAction);
builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective);
builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions);
if (targetFieldName != null) {
builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName);
}
if (targetFieldValue != null) {
builder.field(TARGET_FIELD_VALUE_FIELD.getPreferredName(), targetFieldValue);
}
builder.field(RULE_CONDITIONS_FIELD.getPreferredName(), ruleConditions);
builder.endObject();
return builder;
}
public DetectionRule(String targetFieldName, String targetFieldValue, Connective conditionsConnective,
List<RuleCondition> ruleConditions) {
if (targetFieldValue != null && targetFieldName == null) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue);
throw new IllegalArgumentException(msg);
}
if (ruleConditions == null || ruleConditions.isEmpty()) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION);
throw new IllegalArgumentException(msg);
}
for (RuleCondition condition : ruleConditions) {
if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION,
DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName());
throw new IllegalArgumentException(msg);
}
}
this.targetFieldName = targetFieldName;
this.targetFieldValue = targetFieldValue;
this.conditionsConnective = conditionsConnective != null ? conditionsConnective : Connective.OR;
this.ruleConditions = Collections.unmodifiableList(ruleConditions);
}
public RuleAction getRuleAction() {
return ruleAction;
}
@Nullable
public String getTargetFieldName() {
return targetFieldName;
}
@Nullable
public String getTargetFieldValue() {
return targetFieldValue;
}
@ -155,13 +148,74 @@ public class DetectionRule extends ToXContentToBytes implements Writeable {
}
DetectionRule other = (DetectionRule) obj;
return Objects.equals(ruleAction, other.ruleAction) && Objects.equals(targetFieldName, other.targetFieldName)
return Objects.equals(ruleAction, other.ruleAction)
&& Objects.equals(targetFieldName, other.targetFieldName)
&& Objects.equals(targetFieldValue, other.targetFieldValue)
&& Objects.equals(conditionsConnective, other.conditionsConnective) && Objects.equals(ruleConditions, other.ruleConditions);
&& Objects.equals(conditionsConnective, other.conditionsConnective)
&& Objects.equals(ruleConditions, other.ruleConditions);
}
@Override
public int hashCode() {
return Objects.hash(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions);
}
public static class Builder {
private RuleAction ruleAction = RuleAction.FILTER_RESULTS;
private String targetFieldName;
private String targetFieldValue;
private Connective conditionsConnective = Connective.OR;
private List<RuleCondition> ruleConditions = Collections.emptyList();
public Builder(List<RuleCondition> ruleConditions) {
this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_CONDITIONS_FIELD.getPreferredName());
}
private Builder() {
}
public Builder setRuleAction(RuleAction ruleAction) {
this.ruleAction = ExceptionsHelper.requireNonNull(ruleAction, RULE_ACTION_FIELD.getPreferredName());
return this;
}
public Builder setTargetFieldName(String targetFieldName) {
this.targetFieldName = targetFieldName;
return this;
}
public Builder setTargetFieldValue(String targetFieldValue) {
this.targetFieldValue = targetFieldValue;
return this;
}
public Builder setConditionsConnective(Connective connective) {
this.conditionsConnective = ExceptionsHelper.requireNonNull(connective, CONDITIONS_CONNECTIVE_FIELD.getPreferredName());
return this;
}
public Builder setRuleConditions(List<RuleCondition> ruleConditions) {
this.ruleConditions = ExceptionsHelper.requireNonNull(ruleConditions, RULE_ACTION_FIELD.getPreferredName());
return this;
}
public DetectionRule build() {
if (targetFieldValue != null && targetFieldName == null) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue);
throw new IllegalArgumentException(msg);
}
if (ruleConditions == null || ruleConditions.isEmpty()) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION);
throw new IllegalArgumentException(msg);
}
for (RuleCondition condition : ruleConditions) {
if (condition.getConditionType() == RuleConditionType.CATEGORICAL && targetFieldName != null) {
String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION,
DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName());
throw new IllegalArgumentException(msg);
}
}
return new DetectionRule(ruleAction, targetFieldName, targetFieldValue, conditionsConnective, ruleConditions);
}
}
}

View File

@ -98,7 +98,8 @@ public class Detector extends ToXContentToBytes implements Writeable {
}
throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
}, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING);
PARSER.declareObjectArray(Builder::setDetectorRules, DetectionRule.PARSER, DETECTOR_RULES_FIELD);
PARSER.declareObjectArray(Builder::setDetectorRules,
(parser, parseFieldMatcher) -> DetectionRule.PARSER.apply(parser, parseFieldMatcher).build(), DETECTOR_RULES_FIELD);
}
public static final String COUNT = "count";

View File

@ -323,7 +323,8 @@ public class JobUpdate implements Writeable, ToXContent {
static {
PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), INDEX);
PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), DetectionRule.PARSER, RULES);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(),
(parser, parseFieldMatcher) -> DetectionRule.PARSER.apply(parser, parseFieldMatcher).build(), RULES);
}
private int index;

View File

@ -5,9 +5,14 @@
*/
package org.elasticsearch.xpack.ml.job.config;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import java.io.IOException;
import java.util.Locale;
public enum RuleAction {
public enum RuleAction implements Writeable {
FILTER_RESULTS;
/**
@ -20,6 +25,19 @@ public enum RuleAction {
return RuleAction.valueOf(value.toUpperCase(Locale.ROOT));
}
public static RuleAction readFromStream(StreamInput in) throws IOException {
int ordinal = in.readVInt();
if (ordinal < 0 || ordinal >= values().length) {
throw new IOException("Unknown RuleAction ordinal [" + ordinal + "]");
}
return values()[ordinal];
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(ordinal());
}
@Override
public String toString() {
return name().toLowerCase(Locale.ROOT);

View File

@ -5,6 +5,14 @@
*/
package org.elasticsearch.xpack.ml.job.process.autodetect.writer;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.ml.job.config.DetectionRule;
import org.elasticsearch.xpack.ml.job.config.ModelPlotConfig;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams;
import java.io.IOException;
import java.io.OutputStream;
import java.io.StringWriter;
@ -13,20 +21,12 @@ import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicLong;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.ml.job.config.DetectionRule;
import org.elasticsearch.xpack.ml.job.config.ModelPlotConfig;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams;
/**
* A writer for sending control messages to the C++ autodetect process.
* The data written to outputIndex is length encoded.
*/
public class ControlMsgToProcessWriter {
/**
* This should be the same size as the buffer in the C++ autodetect process.
*/
@ -57,10 +57,6 @@ public class ControlMsgToProcessWriter {
*/
public static final String UPDATE_MESSAGE_CODE = "u";
private static final String EQUALS = " = ";
private static final char NEW_LINE = '\n';
/**
* An number to uniquely identify each flush so that subsequent code can
* wait for acknowledgement of the correct flush.

View File

@ -0,0 +1,165 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.integration;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.ml.action.GetRecordsAction;
import org.elasticsearch.xpack.ml.job.config.AnalysisConfig;
import org.elasticsearch.xpack.ml.job.config.Condition;
import org.elasticsearch.xpack.ml.job.config.DataDescription;
import org.elasticsearch.xpack.ml.job.config.DetectionRule;
import org.elasticsearch.xpack.ml.job.config.Detector;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.ml.job.config.Operator;
import org.elasticsearch.xpack.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.ml.job.config.RuleConditionType;
import org.elasticsearch.xpack.ml.job.results.AnomalyRecord;
import org.junit.After;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
/**
* An integration test for detection rules
*/
public class DetectionRulesIT extends MlNativeAutodetectIntegTestCase {
@After
public void cleanUpTest() throws Exception {
cleanUp();
}
public void test() throws Exception {
RuleCondition condition1 = new RuleCondition(
RuleConditionType.NUMERICAL_ACTUAL,
"by_field",
"by_field_value_1",
new Condition(Operator.LT, "1000"),
null);
RuleCondition condition2 = new RuleCondition(
RuleConditionType.NUMERICAL_ACTUAL,
"by_field",
"by_field_value_2",
new Condition(Operator.LT, "500"),
null);
RuleCondition condition3 = new RuleCondition(
RuleConditionType.NUMERICAL_ACTUAL,
"by_field",
"by_field_value_3",
new Condition(Operator.LT, "100"),
null);
DetectionRule rule = new DetectionRule.Builder(Arrays.asList(condition1, condition2, condition3)).build();
Detector.Builder detector = new Detector.Builder("max", "value");
detector.setDetectorRules(Arrays.asList(rule));
detector.setByFieldName("by_field");
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(
Arrays.asList(detector.build()));
analysisConfig.setBucketSpan(TimeValue.timeValueHours(1));
DataDescription.Builder dataDescription = new DataDescription.Builder();
Job.Builder job = new Job.Builder("detectrion-rule-numeric-test");
job.setAnalysisConfig(analysisConfig);
job.setDataDescription(dataDescription);
registerJob(job);
putJob(job);
openJob(job.getId());
long timestamp = 1491004800000L;
int totalBuckets = 2 * 24;
// each half of the buckets contains one anomaly for each by field value
Set<Integer> anomalousBuckets = new HashSet<>(Arrays.asList(20, 44));
List<String> byFieldValues = Arrays.asList("by_field_value_1", "by_field_value_2", "by_field_value_3");
Map<String, Integer> anomalousValues = new HashMap<>();
anomalousValues.put("by_field_value_1", 800);
anomalousValues.put("by_field_value_2", 400);
anomalousValues.put("by_field_value_3", 400);
int normalValue = 1;
List<String> data = new ArrayList<>();
for (int bucket = 0; bucket < totalBuckets; bucket++) {
for (String byFieldValue : byFieldValues) {
Map<String, Object> record = new HashMap<>();
record.put("time", timestamp);
record.put("value", anomalousBuckets.contains(bucket) ? anomalousValues.get(byFieldValue) : normalValue);
record.put("by_field", byFieldValue);
data.add(createJsonRecord(record));
}
timestamp += TimeValue.timeValueHours(1).getMillis();
}
// push the data for the first half buckets
postData(job.getId(), joinBetween(0, data.size() / 2, data));
closeJob(job.getId());
List<AnomalyRecord> records = getRecords(job.getId());
assertThat(records.size(), equalTo(1));
assertThat(records.get(0).getByFieldValue(), equalTo("by_field_value_3"));
long firstRecordTimestamp = records.get(0).getTimestamp().getTime();
{
// Update rules so that the anomalies suppression is inverted
RuleCondition newCondition1 = new RuleCondition(
RuleConditionType.NUMERICAL_ACTUAL,
"by_field",
"by_field_value_1",
new Condition(Operator.GT, "1000"),
null);
RuleCondition newCondition2 = new RuleCondition(
RuleConditionType.NUMERICAL_ACTUAL,
"by_field",
"by_field_value_2",
new Condition(Operator.GT, "500"),
null);
RuleCondition newCondition3 = new RuleCondition(
RuleConditionType.NUMERICAL_ACTUAL,
"by_field",
"by_field_value_3",
new Condition(Operator.GT, "0"),
null);
DetectionRule newRule = new DetectionRule.Builder(Arrays.asList(newCondition1, newCondition2, newCondition3)).build();
JobUpdate.Builder update = new JobUpdate.Builder(job.getId());
update.setDetectorUpdates(Arrays.asList(new JobUpdate.DetectorUpdate(0, null, Arrays.asList(newRule))));
updateJob(job.getId(), update.build());
}
// push second half
openJob(job.getId());
postData(job.getId(), joinBetween(data.size() / 2, data.size(), data));
closeJob(job.getId());
GetRecordsAction.Request recordsAfterFirstHalf = new GetRecordsAction.Request(job.getId());
recordsAfterFirstHalf.setStart(String.valueOf(firstRecordTimestamp + 1));
records = getRecords(recordsAfterFirstHalf);
assertThat(records.size(), equalTo(2));
Set<String> secondHaldRecordByFieldValues = records.stream().map(AnomalyRecord::getByFieldValue).collect(Collectors.toSet());
assertThat(secondHaldRecordByFieldValues, contains("by_field_value_1", "by_field_value_2"));
}
private static String createJsonRecord(Map<String, Object> keyValueMap) throws IOException {
return JsonXContent.contentBuilder().map(keyValueMap).string() + "\n";
}
private String joinBetween(int start, int end, List<String> input) {
StringBuilder result = new StringBuilder();
for (int i = start; i < end; i++) {
result.append(input.get(i));
}
return result.toString();
}
}

View File

@ -26,10 +26,12 @@ import org.elasticsearch.xpack.ml.action.PutDatafeedAction;
import org.elasticsearch.xpack.ml.action.PutJobAction;
import org.elasticsearch.xpack.ml.action.StartDatafeedAction;
import org.elasticsearch.xpack.ml.action.StopDatafeedAction;
import org.elasticsearch.xpack.ml.action.UpdateJobAction;
import org.elasticsearch.xpack.ml.action.util.PageParams;
import org.elasticsearch.xpack.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.JobState;
import org.elasticsearch.xpack.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.results.AnomalyRecord;
@ -124,6 +126,11 @@ abstract class MlNativeAutodetectIntegTestCase extends SecurityIntegTestCase {
client().execute(FlushJobAction.INSTANCE, request).get();
}
protected void updateJob(String jobId, JobUpdate update) throws Exception {
UpdateJobAction.Request request = new UpdateJobAction.Request(jobId, update);
client().execute(UpdateJobAction.INSTANCE, request);
}
protected void deleteJob(String jobId) throws Exception {
DeleteJobAction.Request request = new DeleteJobAction.Request(jobId);
client().execute(DeleteJobAction.INSTANCE, request).get();
@ -170,6 +177,10 @@ abstract class MlNativeAutodetectIntegTestCase extends SecurityIntegTestCase {
protected List<AnomalyRecord> getRecords(String jobId) throws Exception {
GetRecordsAction.Request request = new GetRecordsAction.Request(jobId);
return getRecords(request);
}
protected List<AnomalyRecord> getRecords(GetRecordsAction.Request request) throws Exception {
GetRecordsAction.Response response = client().execute(GetRecordsAction.INSTANCE, request).get();
return response.getRecords().results();
}

View File

@ -410,10 +410,8 @@ public class AnalysisConfigTests extends AbstractSerializingTestCase<AnalysisCon
}
public void testExtractReferencedLists() {
DetectionRule rule1 =
new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("foo", "filter1")));
DetectionRule rule2 =
new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("foo", "filter2")));
DetectionRule rule1 = new DetectionRule.Builder(Arrays.asList(RuleCondition.createCategorical("foo", "filter1"))).build();
DetectionRule rule2 = new DetectionRule.Builder(Arrays.asList(RuleCondition.createCategorical("foo", "filter2"))).build();
Detector.Builder detector1 = new Detector.Builder("count", null);
detector1.setByFieldName("foo");
detector1.setDetectorRules(Arrays.asList(rule1));

View File

@ -17,65 +17,70 @@ import java.util.List;
public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRule> {
public void testExtractReferoencedLists() {
public void testExtractReferencedLists() {
RuleCondition numericalCondition =
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "field", "value", new Condition(Operator.GT, "5"), null);
List<RuleCondition> conditions = Arrays.asList(
numericalCondition,
RuleCondition.createCategorical("foo", "filter1"),
RuleCondition.createCategorical("bar", "filter2"));
DetectionRule rule = new DetectionRule(null, null, Connective.OR, conditions);
DetectionRule rule = new DetectionRule.Builder(conditions).build();
assertEquals(new HashSet<>(Arrays.asList("filter1", "filter2")), rule.extractReferencedFilters());
}
public void testEqualsGivenSameObject() {
DetectionRule rule = createFullyPopulated();
DetectionRule rule = createFullyPopulated().build();
assertTrue(rule.equals(rule));
}
public void testEqualsGivenString() {
assertFalse(createFullyPopulated().equals("a string"));
assertFalse(createFullyPopulated().build().equals("a string"));
}
public void testEqualsGivenDifferentTargetFieldName() {
DetectionRule rule1 = createFullyPopulated();
DetectionRule rule2 = new DetectionRule("targetField2", "targetValue", Connective.AND, createRule("5"));
DetectionRule rule1 = createFullyPopulated().build();
DetectionRule rule2 = createFullyPopulated().setTargetFieldName("targetField2").build();
assertFalse(rule1.equals(rule2));
assertFalse(rule2.equals(rule1));
}
public void testEqualsGivenDifferentTargetFieldValue() {
DetectionRule rule1 = createFullyPopulated();
DetectionRule rule2 = new DetectionRule("targetField", "targetValue2", Connective.AND, createRule("5"));
DetectionRule rule1 = createFullyPopulated().build();
DetectionRule rule2 = createFullyPopulated().setTargetFieldValue("targetValue2").build();
assertFalse(rule1.equals(rule2));
assertFalse(rule2.equals(rule1));
}
public void testEqualsGivenDifferentConjunction() {
DetectionRule rule1 = createFullyPopulated();
DetectionRule rule2 = new DetectionRule("targetField", "targetValue", Connective.OR, createRule("5"));
public void testEqualsGivenDifferentConnective() {
DetectionRule rule1 = createFullyPopulated().build();
DetectionRule rule2 = createFullyPopulated().setConditionsConnective(Connective.OR).build();
assertFalse(rule1.equals(rule2));
assertFalse(rule2.equals(rule1));
}
public void testEqualsGivenRules() {
DetectionRule rule1 = createFullyPopulated();
DetectionRule rule2 = new DetectionRule("targetField", "targetValue", Connective.AND, createRule("10"));
DetectionRule rule1 = createFullyPopulated().build();
DetectionRule rule2 = createFullyPopulated().setRuleConditions(createRule("10")).build();
assertFalse(rule1.equals(rule2));
assertFalse(rule2.equals(rule1));
}
public void testEqualsGivenEqual() {
DetectionRule rule1 = createFullyPopulated();
DetectionRule rule2 = createFullyPopulated();
DetectionRule rule1 = createFullyPopulated().build();
DetectionRule rule2 = createFullyPopulated().build();
assertTrue(rule1.equals(rule2));
assertTrue(rule2.equals(rule1));
assertEquals(rule1.hashCode(), rule2.hashCode());
}
private static DetectionRule createFullyPopulated() {
return new DetectionRule("targetField", "targetValue", Connective.AND, createRule("5"));
private static DetectionRule.Builder createFullyPopulated() {
return new DetectionRule.Builder(createRule("5"))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setTargetFieldName("targetField")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND);
}
private static List<RuleCondition> createRule(String value) {
@ -85,6 +90,7 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
@Override
protected DetectionRule createTestInstance() {
RuleAction ruleAction = randomFrom(RuleAction.values());
String targetFieldName = null;
String targetFieldValue = null;
Connective connective = randomFrom(Connective.values());
@ -98,7 +104,12 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
// no need for random condition (it is already tested)
ruleConditions.addAll(createRule(Double.toString(randomDouble())));
}
return new DetectionRule(targetFieldName, targetFieldValue, connective, ruleConditions);
return new DetectionRule.Builder(ruleConditions)
.setRuleAction(ruleAction)
.setTargetFieldName(targetFieldName)
.setTargetFieldValue(targetFieldValue)
.setConditionsConnective(connective)
.build();
}
@Override
@ -108,6 +119,6 @@ public class DetectionRuleTests extends AbstractSerializingTestCase<DetectionRul
@Override
protected DetectionRule parseInstance(XContentParser parser) {
return DetectionRule.PARSER.apply(parser, null);
return DetectionRule.PARSER.apply(parser, null).build();
}
}

View File

@ -60,26 +60,17 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
Detector.Builder builder = new Detector.Builder(detector2);
builder.setByFieldName("by2");
Condition condition = new Condition(Operator.GT, "5");
DetectionRule rule = new DetectionRule("over_field", "targetValue", Connective.AND,
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null)));
DetectionRule rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND)
.build();
builder.setDetectorRules(Collections.singletonList(rule));
detector2 = builder.build();
assertFalse(detector1.equals(detector2));
}
public void testEquals_GivenDifferentRules() {
Detector detector1 = createDetector().build();
Detector.Builder builder = new Detector.Builder(detector1);
DetectionRule rule = new DetectionRule(builder.getDetectorRules().get(0).getTargetFieldName(),
builder.getDetectorRules().get(0).getTargetFieldValue(), Connective.OR,
builder.getDetectorRules().get(0).getRuleConditions());
builder.getDetectorRules().set(0, rule);
Detector detector2 = builder.build();
assertFalse(detector1.equals(detector2));
assertFalse(detector2.equals(detector1));
}
public void testExtractAnalysisFields() {
Detector detector = createDetector().build();
assertEquals(Arrays.asList("by_field", "over_field", "partition"), detector.extractAnalysisFields());
@ -89,15 +80,23 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
assertEquals(Arrays.asList("by_field", "over_field"), detector.extractAnalysisFields());
builder = new Detector.Builder(detector);
Condition condition = new Condition(Operator.GT, "5");
DetectionRule rule = new DetectionRule("over_field", "targetValue", Connective.AND,
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)));
DetectionRule rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setTargetFieldName("over_field")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND)
.build();
builder.setDetectorRules(Collections.singletonList(rule));
builder.setByFieldName(null);
detector = builder.build();
assertEquals(Arrays.asList("over_field"), detector.extractAnalysisFields());
builder = new Detector.Builder(detector);
rule = new DetectionRule(null, null, Connective.AND,
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)));
rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setConditionsConnective(Connective.AND)
.build();
builder.setDetectorRules(Collections.singletonList(rule));
builder.setOverFieldName(null);
detector = builder.build();
@ -107,8 +106,8 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
public void testExtractReferencedLists() {
Detector.Builder builder = createDetector();
builder.setDetectorRules(Arrays.asList(
new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("by_field", "list1"))),
new DetectionRule(null, null, Connective.OR, Arrays.asList(RuleCondition.createCategorical("by_field", "list2")))));
new DetectionRule.Builder(Arrays.asList(RuleCondition.createCategorical("by_field", "list1"))).build(),
new DetectionRule.Builder(Arrays.asList(RuleCondition.createCategorical("by_field", "list2"))).build()));
Detector detector = builder.build();
assertEquals(new HashSet<>(Arrays.asList("list1", "list2")), detector.extractReferencedFilters());
@ -121,8 +120,13 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
detector.setPartitionFieldName("partition");
detector.setUseNull(true);
Condition condition = new Condition(Operator.GT, "5");
DetectionRule rule = new DetectionRule("over_field", "targetValue", Connective.AND,
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by_field", "val", condition, null)));
DetectionRule rule = new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by_field", "val", condition, null)))
.setRuleAction(RuleAction.FILTER_RESULTS)
.setTargetFieldName("over_field")
.setTargetFieldValue("targetValue")
.setConditionsConnective(Connective.AND)
.build();
detector.setDetectorRules(Arrays.asList(rule));
return detector;
}
@ -158,9 +162,9 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
for (int i = 0; i < size; i++) {
// no need for random DetectionRule (it is already tested)
Condition condition = new Condition(Operator.GT, "5");
detectorRules.add(new DetectionRule(fieldName, null, Connective.OR, Collections.singletonList(
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))
));
detectorRules.add(new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setTargetFieldName(fieldName).build());
}
detector.setDetectorRules(detectorRules);
}
@ -602,7 +606,7 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
detector.setPartitionFieldName("instance");
RuleCondition ruleCondition =
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null);
DetectionRule rule = new DetectionRule("instancE", null, Connective.OR, Arrays.asList(ruleCondition));
DetectionRule rule = new DetectionRule.Builder(Arrays.asList(ruleCondition)).setTargetFieldName("instancE").build();
detector.setDetectorRules(Arrays.asList(rule));
IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, detector::build);
@ -618,7 +622,7 @@ public class DetectorTests extends AbstractSerializingTestCase<Detector> {
detector.setPartitionFieldName("instance");
RuleCondition ruleCondition =
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null);
DetectionRule rule = new DetectionRule("instance", null, Connective.OR, Arrays.asList(ruleCondition));
DetectionRule rule = new DetectionRule.Builder(Arrays.asList(ruleCondition)).setTargetFieldName("instance").build();
detector.setDetectorRules(Arrays.asList(rule));
detector.build();
}

View File

@ -40,8 +40,9 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
if (randomBoolean()) {
detectionRules = new ArrayList<>();
Condition condition = new Condition(Operator.GT, "5");
detectionRules.add(new DetectionRule("foo", null, Connective.OR, Collections.singletonList(
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))));
detectionRules.add(new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)))
.setTargetFieldName("foo").build());
}
detectorUpdates.add(new JobUpdate.DetectorUpdate(i, detectorDescription, detectionRules));
}
@ -90,13 +91,14 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
public void testMergeWithJob() {
List<JobUpdate.DetectorUpdate> detectorUpdates = new ArrayList<>();
List<DetectionRule> detectionRules1 = Collections.singletonList(new DetectionRule("mlcategory", null, Connective.OR,
Collections.singletonList(
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5"), null))));
List<DetectionRule> detectionRules1 = Collections.singletonList(new DetectionRule.Builder(
Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5")
, null)))
.setTargetFieldName("mlcategory").build());
detectorUpdates.add(new JobUpdate.DetectorUpdate(0, "description-1", detectionRules1));
List<DetectionRule> detectionRules2 = Collections.singletonList(new DetectionRule("host", null, Connective.OR,
Collections.singletonList(
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5"), null))));
List<DetectionRule> detectionRules2 = Collections.singletonList(new DetectionRule.Builder(Collections.singletonList(
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5"), null)))
.setTargetFieldName("host").build());
detectorUpdates.add(new JobUpdate.DetectorUpdate(1, "description-2", detectionRules2));
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10));

View File

@ -5,8 +5,12 @@
*/
package org.elasticsearch.xpack.ml.job.config;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.test.ESTestCase;
import static org.hamcrest.Matchers.equalTo;
public class RuleActionTests extends ESTestCase {
public void testForString() {
@ -18,4 +22,13 @@ public class RuleActionTests extends ESTestCase {
public void testToString() {
assertEquals("filter_results", RuleAction.FILTER_RESULTS.toString());
}
public void testReadFrom() throws Exception {
try (BytesStreamOutput out = new BytesStreamOutput()) {
out.writeVInt(0);
try (StreamInput in = out.bytes().streamInput()) {
assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.FILTER_RESULTS));
}
}
}
}

View File

@ -5,17 +5,6 @@
*/
package org.elasticsearch.xpack.ml.job.process.autodetect.writer;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.IntStream;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.config.Condition;
import org.elasticsearch.xpack.ml.job.config.Connective;
@ -24,12 +13,23 @@ import org.elasticsearch.xpack.ml.job.config.ModelPlotConfig;
import org.elasticsearch.xpack.ml.job.config.Operator;
import org.elasticsearch.xpack.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.ml.job.config.RuleConditionType;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange;
import org.junit.Before;
import org.mockito.InOrder;
import org.mockito.Mockito;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams;
import org.elasticsearch.xpack.ml.job.process.autodetect.params.InterimResultsParams;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.stream.IntStream;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verifyNoMoreInteractions;
public class ControlMsgToProcessWriterTests extends ESTestCase {
private LengthEncodedWriter lengthEncodedWriter;
@ -163,19 +163,21 @@ public class ControlMsgToProcessWriterTests extends ESTestCase {
public void testWriteUpdateDetectorRulesMessage() throws IOException {
ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(lengthEncodedWriter, 2);
DetectionRule rule1 = new DetectionRule("targetField1", "targetValue", Connective.AND, createRule("5"));
DetectionRule rule2 = new DetectionRule("targetField2", "targetValue", Connective.AND, createRule("5"));
DetectionRule rule1 = new DetectionRule.Builder(createRule("5")).setTargetFieldName("targetField1")
.setTargetFieldValue("targetValue").setConditionsConnective(Connective.AND).build();
DetectionRule rule2 = new DetectionRule.Builder(createRule("5")).setTargetFieldName("targetField2")
.setTargetFieldValue("targetValue").setConditionsConnective(Connective.AND).build();
writer.writeUpdateDetectorRulesMessage(2, Arrays.asList(rule1, rule2));
InOrder inOrder = inOrder(lengthEncodedWriter);
inOrder.verify(lengthEncodedWriter).writeNumFields(4);
inOrder.verify(lengthEncodedWriter, times(3)).writeField("");
inOrder.verify(lengthEncodedWriter).writeField("u[detectorRules]\ndetectorIndex=2\n" +
"rulesJson=[{\"conditions_connective\":\"and\",\"rule_conditions\":" +
"rulesJson=[{\"rule_action\":\"filter_results\",\"conditions_connective\":\"and\",\"rule_conditions\":" +
"[{\"condition_type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"\"target_field_name\":\"targetField1\",\"target_field_value\":\"targetValue\"}," +
"{\"conditions_connective\":\"and\",\"rule_conditions\":[{\"condition_type\":\"numerical_actual\"," +
"\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"{\"rule_action\":\"filter_results\",\"conditions_connective\":\"and\",\"rule_conditions\":[" +
"{\"condition_type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," +
"\"target_field_name\":\"targetField2\",\"target_field_value\":\"targetValue\"}]");
verifyNoMoreInteractions(lengthEncodedWriter);
}

View File

@ -5,9 +5,23 @@
*/
package org.elasticsearch.xpack.ml.job.process.autodetect.writer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.config.AnalysisConfig;
import org.elasticsearch.xpack.ml.job.config.Condition;
import org.elasticsearch.xpack.ml.job.config.DetectionRule;
import org.elasticsearch.xpack.ml.job.config.Detector;
import org.elasticsearch.xpack.ml.job.config.MlFilter;
import org.elasticsearch.xpack.ml.job.config.Operator;
import org.elasticsearch.xpack.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.ml.job.config.RuleConditionType;
import org.ini4j.Config;
import org.ini4j.Ini;
import org.ini4j.Profile.Section;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@ -21,25 +35,9 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.config.Connective;
import org.ini4j.Config;
import org.ini4j.Ini;
import org.ini4j.Profile.Section;
import org.junit.Before;
import org.mockito.ArgumentCaptor;
import org.elasticsearch.xpack.ml.job.config.AnalysisConfig;
import org.elasticsearch.xpack.ml.job.config.Detector;
import org.elasticsearch.xpack.ml.job.config.Condition;
import org.elasticsearch.xpack.ml.job.config.Operator;
import org.elasticsearch.xpack.ml.job.config.DetectionRule;
import org.elasticsearch.xpack.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.ml.job.config.RuleConditionType;
import org.elasticsearch.xpack.ml.job.config.MlFilter;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
public class FieldConfigWriterTests extends ESTestCase {
@ -180,7 +178,7 @@ public class FieldConfigWriterTests extends ESTestCase {
detector.setPartitionFieldName("instance");
RuleCondition ruleCondition =
new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricValue", new Condition(Operator.LT, "5"), null);
DetectionRule rule = new DetectionRule("instance", null, Connective.OR, Arrays.asList(ruleCondition));
DetectionRule rule = new DetectionRule.Builder(Arrays.asList(ruleCondition)).setTargetFieldName("instance").build();
detector.setDetectorRules(Arrays.asList(rule));
AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Arrays.asList(detector.build()));