[7.x] Introduce ModelPlotConfig. annotations_enabled setting (#57539) (#57641)

This commit is contained in:
Przemysław Witek 2020-06-04 15:15:35 +02:00 committed by GitHub
parent 20aa4eec55
commit 6b5f49d097
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 118 additions and 35 deletions

View File

@ -30,22 +30,27 @@ public class ModelPlotConfig implements ToXContentObject {
private static final ParseField TYPE_FIELD = new ParseField("model_plot_config");
private static final ParseField ENABLED_FIELD = new ParseField("enabled");
public static final ParseField TERMS_FIELD = new ParseField("terms");
private static final ParseField TERMS_FIELD = new ParseField("terms");
private static final ParseField ANNOTATIONS_ENABLED_FIELD = new ParseField("annotations_enabled");
public static final ConstructingObjectParser<ModelPlotConfig, Void> PARSER =
new ConstructingObjectParser<>(TYPE_FIELD.getPreferredName(), true, a -> new ModelPlotConfig((boolean) a[0], (String) a[1]));
new ConstructingObjectParser<>(
TYPE_FIELD.getPreferredName(), true, a -> new ModelPlotConfig((boolean) a[0], (String) a[1], (Boolean) a[2]));
static {
PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD);
PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ANNOTATIONS_ENABLED_FIELD);
}
private final boolean enabled;
private final String terms;
private final boolean annotationsEnabled;
public ModelPlotConfig(boolean enabled, String terms) {
public ModelPlotConfig(boolean enabled, String terms, Boolean annotationsEnabled) {
this.enabled = enabled;
this.terms = terms;
this.annotationsEnabled = Boolean.TRUE.equals(annotationsEnabled);
}
@Override
@ -55,6 +60,7 @@ public class ModelPlotConfig implements ToXContentObject {
if (terms != null) {
builder.field(TERMS_FIELD.getPreferredName(), terms);
}
builder.field(ANNOTATIONS_ENABLED_FIELD.getPreferredName(), annotationsEnabled);
builder.endObject();
return builder;
}
@ -67,6 +73,10 @@ public class ModelPlotConfig implements ToXContentObject {
return this.terms;
}
public boolean annotationsEnabled() {
return annotationsEnabled;
}
@Override
public boolean equals(Object other) {
if (this == other) {
@ -78,11 +88,11 @@ public class ModelPlotConfig implements ToXContentObject {
}
ModelPlotConfig that = (ModelPlotConfig) other;
return this.enabled == that.enabled && Objects.equals(this.terms, that.terms);
return this.enabled == that.enabled && Objects.equals(this.terms, that.terms) && this.annotationsEnabled == that.annotationsEnabled;
}
@Override
public int hashCode() {
return Objects.hash(enabled, terms);
return Objects.hash(enabled, terms, annotationsEnabled);
}
}

View File

@ -590,7 +590,7 @@ public class MlClientDocumentationIT extends ESRestHighLevelClientTestCase {
.setDetectorUpdates(Arrays.asList(detectorUpdate)) // <6>
.setGroups(Arrays.asList("job-group-1")) // <7>
.setResultsRetentionDays(10L) // <8>
.setModelPlotConfig(new ModelPlotConfig(true, null)) // <9>
.setModelPlotConfig(new ModelPlotConfig(true, null, true)) // <9>
.setModelSnapshotRetentionDays(7L) // <10>
.setCustomSettings(customSettings) // <11>
.setRenormalizationWindowDays(3L) // <12>

View File

@ -133,7 +133,7 @@ public class JobTests extends AbstractXContentTestCase<Job> {
builder.setDataDescription(dataDescription);
if (randomBoolean()) {
builder.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
builder.setModelPlotConfig(ModelPlotConfigTests.createRandomized());
}
if (randomBoolean()) {
builder.setRenormalizationWindowDays(randomNonNegativeLong());

View File

@ -56,7 +56,7 @@ public class JobUpdateTests extends AbstractXContentTestCase<JobUpdate> {
update.setDetectorUpdates(createRandomDetectorUpdates());
}
if (randomBoolean()) {
update.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
update.setModelPlotConfig(ModelPlotConfigTests.createRandomized());
}
if (randomBoolean()) {
update.setAnalysisLimits(AnalysisLimitsTests.createRandomized());

View File

@ -25,7 +25,11 @@ public class ModelPlotConfigTests extends AbstractXContentTestCase<ModelPlotConf
@Override
protected ModelPlotConfig createTestInstance() {
return new ModelPlotConfig(randomBoolean(), randomAlphaOfLengthBetween(1, 30));
return createRandomized();
}
public static ModelPlotConfig createRandomized() {
return new ModelPlotConfig(randomBoolean(), randomAlphaOfLengthBetween(1, 30), randomBoolean() ? randomBoolean() : null);
}
@Override

View File

@ -243,6 +243,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config]
.Properties of `model_plot_config`
[%collapsible%open]
====
`annotations_enabled`:::
(boolean)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-annotations-enabled]
`enabled`:::
(boolean)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-enabled]

View File

@ -180,10 +180,17 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config]
.Properties of `model_plot_config`
[%collapsible%open]
====
`annotations_enabled`:::
(boolean)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-annotations-enabled]
`enabled`:::
(boolean)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-enabled]
`terms`:::
experimental[] (string)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-plot-config-terms]
====
//End model_plot_config

View File

@ -1000,6 +1000,11 @@ applied. For example, "CPU,NetworkIn,DiskWrites". Wildcards are not supported.
Only the specified `terms` can be viewed when using the Single Metric Viewer.
end::model-plot-config-terms[]
tag::model-plot-config-annotations-enabled[]
If true, enables calculation and storage of the model change annotations
for each entity that is being analyzed. By default, this is not enabled.
end::model-plot-config-annotations-enabled[]
tag::model-snapshot-retention-days[]
Advanced configuration option, which affects the automatic removal of old model
snapshots for this job. It specifies the maximum period of time (in days) that

View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.core.ml.job.config;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -21,6 +22,7 @@ public class ModelPlotConfig implements ToXContentObject, Writeable {
public static final ParseField TYPE_FIELD = new ParseField("model_plot_config");
public static final ParseField ENABLED_FIELD = new ParseField("enabled");
public static final ParseField TERMS_FIELD = new ParseField("terms");
public static final ParseField ANNOTATIONS_ENABLED_FIELD = new ParseField("annotations_enabled");
// These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly
public static final ConstructingObjectParser<ModelPlotConfig, Void> LENIENT_PARSER = createParser(true);
@ -28,35 +30,46 @@ public class ModelPlotConfig implements ToXContentObject, Writeable {
private static ConstructingObjectParser<ModelPlotConfig, Void> createParser(boolean ignoreUnknownFields) {
ConstructingObjectParser<ModelPlotConfig, Void> parser = new ConstructingObjectParser<>(TYPE_FIELD.getPreferredName(),
ignoreUnknownFields, a -> new ModelPlotConfig((boolean) a[0], (String) a[1]));
ignoreUnknownFields, a -> new ModelPlotConfig((boolean) a[0], (String) a[1], (Boolean) a[2]));
parser.declareBoolean(ConstructingObjectParser.constructorArg(), ENABLED_FIELD);
parser.declareString(ConstructingObjectParser.optionalConstructorArg(), TERMS_FIELD);
parser.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ANNOTATIONS_ENABLED_FIELD);
return parser;
}
private final boolean enabled;
private final String terms;
private final boolean annotationsEnabled;
public ModelPlotConfig() {
this(true, null);
this(true, null, true);
}
public ModelPlotConfig(boolean enabled, String terms) {
public ModelPlotConfig(boolean enabled, String terms, Boolean annotationsEnabled) {
this.enabled = enabled;
this.terms = terms;
this.annotationsEnabled = Boolean.TRUE.equals(annotationsEnabled);
}
public ModelPlotConfig(StreamInput in) throws IOException {
enabled = in.readBoolean();
terms = in.readOptionalString();
if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
annotationsEnabled = in.readBoolean();
} else {
annotationsEnabled = enabled;
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(enabled);
out.writeOptionalString(terms);
if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
out.writeBoolean(annotationsEnabled);
}
}
@Override
@ -66,6 +79,7 @@ public class ModelPlotConfig implements ToXContentObject, Writeable {
if (terms != null) {
builder.field(TERMS_FIELD.getPreferredName(), terms);
}
builder.field(ANNOTATIONS_ENABLED_FIELD.getPreferredName(), annotationsEnabled);
builder.endObject();
return builder;
}
@ -75,7 +89,11 @@ public class ModelPlotConfig implements ToXContentObject, Writeable {
}
public String getTerms() {
return this.terms;
return terms;
}
public boolean annotationsEnabled() {
return annotationsEnabled;
}
@Override
@ -89,11 +107,11 @@ public class ModelPlotConfig implements ToXContentObject, Writeable {
}
ModelPlotConfig that = (ModelPlotConfig) other;
return this.enabled == that.enabled && Objects.equals(this.terms, that.terms);
return this.enabled == that.enabled && Objects.equals(this.terms, that.terms) && this.annotationsEnabled == that.annotationsEnabled;
}
@Override
public int hashCode() {
return Objects.hash(enabled, terms);
return Objects.hash(enabled, terms, annotationsEnabled);
}
}

View File

@ -278,6 +278,7 @@ public final class ReservedFieldNames {
ModelPlotConfig.ENABLED_FIELD.getPreferredName(),
ModelPlotConfig.TERMS_FIELD.getPreferredName(),
ModelPlotConfig.ANNOTATIONS_ENABLED_FIELD.getPreferredName(),
DatafeedConfig.ID.getPreferredName(),
DatafeedConfig.QUERY_DELAY.getPreferredName(),

View File

@ -322,6 +322,9 @@
},
"terms" : {
"type" : "keyword"
},
"annotations_enabled" : {
"type" : "boolean"
}
}
},

View File

@ -11,6 +11,7 @@ import org.elasticsearch.xpack.core.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.core.ml.job.config.MlFilter;
import org.elasticsearch.xpack.core.ml.job.config.MlFilterTests;
import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfigTests;
import java.util.ArrayList;
import java.util.List;
@ -21,7 +22,7 @@ public class UpdateProcessActionRequestTests extends AbstractWireSerializingTest
protected UpdateProcessAction.Request createTestInstance() {
ModelPlotConfig config = null;
if (randomBoolean()) {
config = new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10));
config = ModelPlotConfigTests.createRandomized();
}
List<JobUpdate.DetectorUpdate> updates = null;
if (randomBoolean()) {

View File

@ -640,7 +640,7 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
builder.setDataDescription(dataDescription);
if (randomBoolean()) {
builder.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
builder.setModelPlotConfig(ModelPlotConfigTests.createRandomized());
}
if (randomBoolean()) {
builder.setRenormalizationWindowDays(randomNonNegativeLong());

View File

@ -60,7 +60,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
update.setDetectorUpdates(detectorUpdates);
}
if (randomBoolean()) {
update.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)));
update.setModelPlotConfig(ModelPlotConfigTests.createRandomized());
}
if (randomBoolean()) {
update.setAnalysisLimits(AnalysisLimits.validateAndSetDefaults(AnalysisLimitsTests.createRandomized(), null,
@ -221,7 +221,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))).build());
detectorUpdates.add(new JobUpdate.DetectorUpdate(1, "description-2", detectionRules2));
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10));
ModelPlotConfig modelPlotConfig = ModelPlotConfigTests.createRandomized();
AnalysisLimits analysisLimits = new AnalysisLimits(randomNonNegativeLong(), randomNonNegativeLong());
List<String> categorizationFilters = Arrays.asList(generateRandomStringArray(10, 10, false));
Map<String, Object> customSettings = Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10));
@ -299,7 +299,7 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
public void testIsAutodetectProcessUpdate() {
JobUpdate update = new JobUpdate.Builder("foo").build();
assertFalse(update.isAutodetectProcessUpdate());
update = new JobUpdate.Builder("foo").setModelPlotConfig(new ModelPlotConfig(true, "ff")).build();
update = new JobUpdate.Builder("foo").setModelPlotConfig(new ModelPlotConfig(true, "ff", false)).build();
assertTrue(update.isAutodetectProcessUpdate());
update = new JobUpdate.Builder("foo").setDetectorUpdates(Collections.singletonList(mock(JobUpdate.DetectorUpdate.class))).build();
assertTrue(update.isAutodetectProcessUpdate());

View File

@ -15,13 +15,19 @@ import static org.hamcrest.Matchers.nullValue;
public class ModelPlotConfigTests extends AbstractSerializingTestCase<ModelPlotConfig> {
public void testConstructorDefaults() {
assertThat(new ModelPlotConfig().isEnabled(), is(true));
assertThat(new ModelPlotConfig().getTerms(), is(nullValue()));
ModelPlotConfig modelPlotConfig = new ModelPlotConfig();
assertThat(modelPlotConfig.isEnabled(), is(true));
assertThat(modelPlotConfig.getTerms(), is(nullValue()));
assertThat(modelPlotConfig.annotationsEnabled(), is(true));
}
@Override
protected ModelPlotConfig createTestInstance() {
return new ModelPlotConfig(randomBoolean(), randomAlphaOfLengthBetween(1, 30));
return createRandomized();
}
public static ModelPlotConfig createRandomized() {
return new ModelPlotConfig(randomBoolean(), randomAlphaOfLengthBetween(1, 30), randomBoolean() ? randomBoolean() : null);
}
@Override

View File

@ -95,7 +95,7 @@ public class ModelPlotsIT extends MlNativeAutodetectIntegTestCase {
public void testPartitionFieldWithTerms() throws Exception {
Job.Builder job = jobWithPartitionUser("model-plots-it-test-partition-field-with-terms");
job.setModelPlotConfig(new ModelPlotConfig(true, "user_2,user_3"));
job.setModelPlotConfig(new ModelPlotConfig(true, "user_2,user_3", false));
registerJob(job);
putJob(job);
String datafeedId = job.getId() + "-feed";
@ -117,7 +117,7 @@ public class ModelPlotsIT extends MlNativeAutodetectIntegTestCase {
public void testByFieldWithTerms() throws Exception {
Job.Builder job = jobWithByUser("model-plots-it-test-by-field-with-terms");
job.setModelPlotConfig(new ModelPlotConfig(true, "user_2,user_3"));
job.setModelPlotConfig(new ModelPlotConfig(true, "user_2,user_3", false));
registerJob(job);
putJob(job);
String datafeedId = job.getId() + "-feed";

View File

@ -42,6 +42,11 @@ public class ModelPlotConfigWriter {
.append(terms == null ? "" : terms)
.append(NEW_LINE);
contents.append(ModelPlotConfig.ANNOTATIONS_ENABLED_FIELD.getPreferredName())
.append(EQUALS)
.append(modelPlotConfig.annotationsEnabled())
.append(NEW_LINE);
writer.write(contents.toString());
}
}

View File

@ -13,7 +13,7 @@ import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfigTests;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimitsTests;
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig;
import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfigTests;
import java.util.Collections;
import java.util.Date;
@ -48,8 +48,7 @@ public class JobBuilderTests extends AbstractSerializingTestCase<Job.Builder> {
builder.setDataDescription(dataDescription);
}
if (randomBoolean()) {
builder.setModelPlotConfig(new ModelPlotConfig(randomBoolean(),
randomAlphaOfLength(10)));
builder.setModelPlotConfig(ModelPlotConfigTests.createRandomized());
}
if (randomBoolean()) {
builder.setRenormalizationWindowDays(randomNonNegativeLong());

View File

@ -178,12 +178,13 @@ public class AutodetectControlMsgWriterTests extends ESTestCase {
public void testWriteUpdateModelPlotMessage() throws IOException {
AutodetectControlMsgWriter writer = new AutodetectControlMsgWriter(lengthEncodedWriter, 4);
writer.writeUpdateModelPlotMessage(new ModelPlotConfig(true, "foo,bar"));
writer.writeUpdateModelPlotMessage(new ModelPlotConfig(true, "foo,bar", false));
InOrder inOrder = inOrder(lengthEncodedWriter);
inOrder.verify(lengthEncodedWriter).writeNumFields(4);
inOrder.verify(lengthEncodedWriter, times(3)).writeField("");
inOrder.verify(lengthEncodedWriter).writeField("u[modelPlotConfig]\nboundspercentile = 95.0\nterms = foo,bar\n");
inOrder.verify(lengthEncodedWriter)
.writeField("u[modelPlotConfig]\nboundspercentile = 95.0\nterms = foo,bar\nannotations_enabled = false\n");
verifyNoMoreInteractions(lengthEncodedWriter);
}

View File

@ -36,24 +36,42 @@ public class ModelPlotConfigWriterTests extends ESTestCase {
writer.write();
verify(this.writer).write("boundspercentile = 95.0\nterms = \n");
verify(this.writer).write("boundspercentile = 95.0\nterms = \nannotations_enabled = true\n");
}
public void testWrite_GivenEnabledConfigWithTerms() throws IOException {
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(true, "foo,bar");
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(true, "foo,bar", false);
ModelPlotConfigWriter writer = new ModelPlotConfigWriter(modelPlotConfig, this.writer);
writer.write();
verify(this.writer).write("boundspercentile = 95.0\nterms = foo,bar\n");
verify(this.writer).write("boundspercentile = 95.0\nterms = foo,bar\nannotations_enabled = false\n");
}
public void testWrite_GivenDisabledConfigWithTerms() throws IOException {
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(false, "foo,bar");
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(false, "foo,bar", false);
ModelPlotConfigWriter writer = new ModelPlotConfigWriter(modelPlotConfig, this.writer);
writer.write();
verify(this.writer).write("boundspercentile = -1.0\nterms = foo,bar\n");
verify(this.writer).write("boundspercentile = -1.0\nterms = foo,bar\nannotations_enabled = false\n");
}
public void testWrite_GivenEnabledConfigWithEnabledAnnotations() throws IOException {
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(true, null, true);
ModelPlotConfigWriter writer = new ModelPlotConfigWriter(modelPlotConfig, this.writer);
writer.write();
verify(this.writer).write("boundspercentile = 95.0\nterms = \nannotations_enabled = true\n");
}
public void testWrite_GivenDisabledConfigWithEnabledAnnotations() throws IOException {
ModelPlotConfig modelPlotConfig = new ModelPlotConfig(false, null, true);
ModelPlotConfigWriter writer = new ModelPlotConfigWriter(modelPlotConfig, this.writer);
writer.write();
verify(this.writer).write("boundspercentile = -1.0\nterms = \nannotations_enabled = true\n");
}
}

View File

@ -393,6 +393,7 @@
- match: { description: "Post update description" }
- match: { model_plot_config.enabled: false }
- match: { model_plot_config.terms: "foobar" }
- match: { model_plot_config.annotations_enabled: false }
- match: { analysis_config.categorization_filters: ["cat3.*"] }
- match: { analysis_config.detectors.0.custom_rules.0.actions: ["skip_result"] }
- length: { analysis_config.detectors.0.custom_rules.0.conditions: 1 }