honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Add daily_model_snapshot_retention_after_days to job config (#55891) 

This change adds a new setting, daily_model_snapshot_retention_after_days,
to the anomaly detection job config.

Initially this has no effect, the effect will be added in a followup PR.
This PR gets the complexities of making changes that interact with BWC
over well before feature freeze.

Backport of #55878
This commit is contained in:
David Roberts 2020-04-29 09:12:53 +01:00 committed by GitHub
parent e982cf4381
commit 61ac09ae21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 161 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/Job.java
View File

@ -63,6 +63,8 @@ public class Job implements ToXContentObject {
public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days");
public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval");
public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days");
public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS =
new ParseField("daily_model_snapshot_retention_after_days");
public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
public static final ParseField RESULTS_INDEX_NAME = new ParseField("results_index_name");
@ -93,6 +95,7 @@ public class Job implements ToXContentObject {
TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
PARSER.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT);
PARSER.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
PARSER.declareString(Builder::setResultsIndexName, RESULTS_INDEX_NAME);
@ -114,6 +117,7 @@ public class Job implements ToXContentObject {
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long dailyModelSnapshotRetentionAfterDays;
private final Long resultsRetentionDays;
private final Map<String, Object> customSettings;
private final String modelSnapshotId;
@ -125,8 +129,9 @@ public class Job implements ToXContentObject {
Date createTime, Date finishedTime,
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
String modelSnapshotId, String resultsIndexName, Boolean deleting, Boolean allowLazyOpen) {
Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays,
Map<String, Object> customSettings, String modelSnapshotId, String resultsIndexName, Boolean deleting,
Boolean allowLazyOpen) {
this.jobId = jobId;
this.jobType = jobType;
@ -141,6 +146,7 @@ public class Job implements ToXContentObject {
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
this.resultsRetentionDays = resultsRetentionDays;
this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
this.modelSnapshotId = modelSnapshotId;
@ -259,6 +265,10 @@ public class Job implements ToXContentObject {
return modelSnapshotRetentionDays;
}
public Long getDailyModelSnapshotRetentionAfterDays() {
return dailyModelSnapshotRetentionAfterDays;
}
public Long getResultsRetentionDays() {
return resultsRetentionDays;
}
@ -319,6 +329,9 @@ public class Job implements ToXContentObject {
if (modelSnapshotRetentionDays != null) {
builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
@ -365,6 +378,7 @@ public class Job implements ToXContentObject {
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.customSettings, that.customSettings)
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
@ -377,8 +391,8 @@ public class Job implements ToXContentObject {
public int hashCode() {
return Objects.hash(jobId, jobType, groups, description, createTime, finishedTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
}
@Override
@ -405,6 +419,7 @@ public class Job implements ToXContentObject {
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays;
private Long dailyModelSnapshotRetentionAfterDays;
private Long resultsRetentionDays;
private Map<String, Object> customSettings;
private String modelSnapshotId;
@ -433,6 +448,7 @@ public class Job implements ToXContentObject {
this.renormalizationWindowDays = job.getRenormalizationWindowDays();
this.backgroundPersistInterval = job.getBackgroundPersistInterval();
this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays();
this.resultsRetentionDays = job.getResultsRetentionDays();
this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings());
this.modelSnapshotId = job.getModelSnapshotId();
@ -515,6 +531,11 @@ public class Job implements ToXContentObject {
return this;
}
public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) {
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
return this;
}
public Builder setResultsRetentionDays(Long resultsRetentionDays) {
this.resultsRetentionDays = resultsRetentionDays;
return this;
@ -551,8 +572,8 @@ public class Job implements ToXContentObject {
return new Job(
id, jobType, groups, description, createTime, finishedTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, resultsIndexName, deleting, allowLazyOpen);
}
}
}

33
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/JobUpdate.java
View File

@ -52,6 +52,7 @@ public class JobUpdate implements ToXContentObject {
PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT);
PARSER.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN);
@ -66,6 +67,7 @@ public class JobUpdate implements ToXContentObject {
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long dailyModelSnapshotRetentionAfterDays;
private final Long resultsRetentionDays;
private final List<String> categorizationFilters;
private final Map<String, Object> customSettings;
@ -75,7 +77,8 @@ public class JobUpdate implements ToXContentObject {
@Nullable List<DetectorUpdate> detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig,
@Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval,
@Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays,
@Nullable Long modelSnapshotRetentionDays, @Nullable List<String> categorisationFilters,
@Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays,
@Nullable List<String> categorizationFilters,
@Nullable Map<String, Object> customSettings, @Nullable Boolean allowLazyOpen) {
this.jobId = jobId;
this.groups = groups;
@ -86,8 +89,9 @@ public class JobUpdate implements ToXContentObject {
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
this.resultsRetentionDays = resultsRetentionDays;
this.categorizationFilters = categorisationFilters;
this.categorizationFilters = categorizationFilters;
this.customSettings = customSettings;
this.allowLazyOpen = allowLazyOpen;
}
@ -172,6 +176,9 @@ public class JobUpdate implements ToXContentObject {
if (modelSnapshotRetentionDays != null) {
builder.field(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
@ -209,6 +216,7 @@ public class JobUpdate implements ToXContentObject {
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.categorizationFilters, that.categorizationFilters)
&& Objects.equals(this.customSettings, that.customSettings)
@ -218,8 +226,8 @@ public class JobUpdate implements ToXContentObject {
@Override
public int hashCode() {
return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings,
allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
categorizationFilters, customSettings, allowLazyOpen);
}
public static class DetectorUpdate implements ToXContentObject {
@ -312,6 +320,7 @@ public class JobUpdate implements ToXContentObject {
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays;
private Long dailyModelSnapshotRetentionAfterDays;
private Long resultsRetentionDays;
private List<String> categorizationFilters;
private Map<String, Object> customSettings;
@ -422,6 +431,18 @@ public class JobUpdate implements ToXContentObject {
return this;
}
/**
* The time in days after which only one model snapshot per day is retained for the job.
*
* Updates the {@link Job#dailyModelSnapshotRetentionAfterDays} setting
*
* @param dailyModelSnapshotRetentionAfterDays number of days to keep a model snapshot
*/
public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) {
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
return this;
}
/**
* Advanced configuration option. The number of days for which job results are retained
*
@ -466,8 +487,8 @@ public class JobUpdate implements ToXContentObject {
public JobUpdate build() {
return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings,
allowLazyOpen);
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays,
categorizationFilters, customSettings, allowLazyOpen);
}
}
}

3
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobTests.java
View File

@ -144,6 +144,9 @@ public class JobTests extends AbstractXContentTestCase<Job> {
if (randomBoolean()) {
builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setResultsRetentionDays(randomNonNegativeLong());
}

3
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/config/JobUpdateTests.java
View File

@ -70,6 +70,9 @@ public class JobUpdateTests extends AbstractXContentTestCase<JobUpdate> {
if (randomBoolean()) {
update.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
update.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
update.setResultsRetentionDays(randomNonNegativeLong());
}

57
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java
View File

@ -72,6 +72,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
public static final ParseField RENORMALIZATION_WINDOW_DAYS = new ParseField("renormalization_window_days");
public static final ParseField BACKGROUND_PERSIST_INTERVAL = new ParseField("background_persist_interval");
public static final ParseField MODEL_SNAPSHOT_RETENTION_DAYS = new ParseField("model_snapshot_retention_days");
public static final ParseField DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS =
new ParseField("daily_model_snapshot_retention_after_days");
public static final ParseField RESULTS_RETENTION_DAYS = new ParseField("results_retention_days");
public static final ParseField MODEL_SNAPSHOT_ID = new ParseField("model_snapshot_id");
public static final ParseField MODEL_SNAPSHOT_MIN_VERSION = new ParseField("model_snapshot_min_version");
@ -123,6 +125,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
TimeValue.parseTimeValue(val, BACKGROUND_PERSIST_INTERVAL.getPreferredName())), BACKGROUND_PERSIST_INTERVAL);
parser.declareLong(Builder::setResultsRetentionDays, RESULTS_RETENTION_DAYS);
parser.declareLong(Builder::setModelSnapshotRetentionDays, MODEL_SNAPSHOT_RETENTION_DAYS);
parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
parser.declareField(Builder::setCustomSettings, (p, c) -> p.mapOrdered(), CUSTOM_SETTINGS, ValueType.OBJECT);
parser.declareStringOrNull(Builder::setModelSnapshotId, MODEL_SNAPSHOT_ID);
parser.declareStringOrNull(Builder::setModelSnapshotMinVersion, MODEL_SNAPSHOT_MIN_VERSION);
@ -155,6 +158,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long dailyModelSnapshotRetentionAfterDays;
private final Long resultsRetentionDays;
private final Map<String, Object> customSettings;
private final String modelSnapshotId;
@ -167,8 +171,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
Date createTime, Date finishedTime,
AnalysisConfig analysisConfig, AnalysisLimits analysisLimits, DataDescription dataDescription,
ModelPlotConfig modelPlotConfig, Long renormalizationWindowDays, TimeValue backgroundPersistInterval,
Long modelSnapshotRetentionDays, Long resultsRetentionDays, Map<String, Object> customSettings,
String modelSnapshotId, Version modelSnapshotMinVersion, String resultsIndexName, boolean deleting, boolean allowLazyOpen) {
Long modelSnapshotRetentionDays, Long dailyModelSnapshotRetentionAfterDays, Long resultsRetentionDays,
Map<String, Object> customSettings, String modelSnapshotId, Version modelSnapshotMinVersion, String resultsIndexName,
boolean deleting, boolean allowLazyOpen) {
this.jobId = jobId;
this.jobType = jobType;
@ -184,6 +189,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
this.resultsRetentionDays = resultsRetentionDays;
this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings);
this.modelSnapshotId = modelSnapshotId;
@ -222,6 +228,11 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
renormalizationWindowDays = in.readOptionalLong();
backgroundPersistInterval = in.readOptionalTimeValue();
modelSnapshotRetentionDays = in.readOptionalLong();
if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
dailyModelSnapshotRetentionAfterDays = in.readOptionalLong();
} else {
dailyModelSnapshotRetentionAfterDays = null;
}
resultsRetentionDays = in.readOptionalLong();
Map<String, Object> readCustomSettings = in.readMap();
customSettings = readCustomSettings == null ? null : Collections.unmodifiableMap(readCustomSettings);
@ -387,6 +398,10 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
return modelSnapshotRetentionDays;
}
public Long getDailyModelSnapshotRetentionAfterDays() {
return dailyModelSnapshotRetentionAfterDays;
}
public Long getResultsRetentionDays() {
return resultsRetentionDays;
}
@ -495,6 +510,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
out.writeOptionalLong(renormalizationWindowDays);
out.writeOptionalTimeValue(backgroundPersistInterval);
out.writeOptionalLong(modelSnapshotRetentionDays);
if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays);
}
out.writeOptionalLong(resultsRetentionDays);
out.writeMap(customSettings);
out.writeOptionalString(modelSnapshotId);
@ -559,6 +577,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
if (modelSnapshotRetentionDays != null) {
builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
@ -604,6 +625,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.customSettings, that.customSettings)
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
@ -617,8 +639,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
public int hashCode() {
return Objects.hash(jobId, jobType, jobVersion, groups, description, createTime, finishedTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen);
}
// Class already extends from AbstractDiffable, so copied from ToXContentToBytes#toString()
@ -660,6 +682,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays = DEFAULT_MODEL_SNAPSHOT_RETENTION_DAYS;
private Long dailyModelSnapshotRetentionAfterDays;
private Long resultsRetentionDays;
private Map<String, Object> customSettings;
private String modelSnapshotId;
@ -690,6 +713,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
this.renormalizationWindowDays = job.getRenormalizationWindowDays();
this.backgroundPersistInterval = job.getBackgroundPersistInterval();
this.modelSnapshotRetentionDays = job.getModelSnapshotRetentionDays();
this.dailyModelSnapshotRetentionAfterDays = job.getDailyModelSnapshotRetentionAfterDays();
this.resultsRetentionDays = job.getResultsRetentionDays();
this.customSettings = job.getCustomSettings() == null ? null : new LinkedHashMap<>(job.getCustomSettings());
this.modelSnapshotId = job.getModelSnapshotId();
@ -728,6 +752,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
renormalizationWindowDays = in.readOptionalLong();
backgroundPersistInterval = in.readOptionalTimeValue();
modelSnapshotRetentionDays = in.readOptionalLong();
if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
dailyModelSnapshotRetentionAfterDays = in.readOptionalLong();
}
resultsRetentionDays = in.readOptionalLong();
customSettings = in.readMap();
modelSnapshotId = in.readOptionalString();
@ -831,6 +858,11 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
return this;
}
public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) {
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
return this;
}
public Builder setResultsRetentionDays(Long resultsRetentionDays) {
this.resultsRetentionDays = resultsRetentionDays;
return this;
@ -927,6 +959,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
out.writeOptionalLong(renormalizationWindowDays);
out.writeOptionalTimeValue(backgroundPersistInterval);
out.writeOptionalLong(modelSnapshotRetentionDays);
if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays);
}
out.writeOptionalLong(resultsRetentionDays);
out.writeMap(customSettings);
out.writeOptionalString(modelSnapshotId);
@ -985,6 +1020,9 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
if (modelSnapshotRetentionDays != null) {
builder.field(MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
@ -1031,6 +1069,7 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.customSettings, that.customSettings)
&& Objects.equals(this.modelSnapshotId, that.modelSnapshotId)
@ -1044,8 +1083,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
public int hashCode() {
return Objects.hash(id, jobType, jobVersion, groups, description, analysisConfig, analysisLimits, dataDescription,
createTime, finishedTime, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId,
modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen);
}
/**
@ -1067,6 +1106,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
checkValidBackgroundPersistInterval();
checkValueNotLessThan(0, RENORMALIZATION_WINDOW_DAYS.getPreferredName(), renormalizationWindowDays);
checkValueNotLessThan(0, MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
checkValueNotLessThan(0, DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(),
dailyModelSnapshotRetentionAfterDays);
checkValueNotLessThan(0, RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
if (!MlStrings.isValidId(id)) {
@ -1171,8 +1212,8 @@ public class Job extends AbstractDiffable<Job> implements Writeable, ToXContentO
return new Job(
id, jobType, jobVersion, groups, description, createTime, finishedTime,
analysisConfig, analysisLimits, dataDescription, modelPlotConfig, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings,
modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
customSettings, modelSnapshotId, modelSnapshotMinVersion, resultsIndexName, deleting, allowLazyOpen);
}
private void checkValidBackgroundPersistInterval() {

48
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java
View File

@ -52,6 +52,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT);
parser.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN);
@ -72,6 +73,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
private final Long renormalizationWindowDays;
private final TimeValue backgroundPersistInterval;
private final Long modelSnapshotRetentionDays;
private final Long dailyModelSnapshotRetentionAfterDays;
private final Long resultsRetentionDays;
private final List<String> categorizationFilters;
private final Map<String, Object> customSettings;
@ -85,7 +87,8 @@ public class JobUpdate implements Writeable, ToXContentObject {
@Nullable List<DetectorUpdate> detectorUpdates, @Nullable ModelPlotConfig modelPlotConfig,
@Nullable AnalysisLimits analysisLimits, @Nullable TimeValue backgroundPersistInterval,
@Nullable Long renormalizationWindowDays, @Nullable Long resultsRetentionDays,
@Nullable Long modelSnapshotRetentionDays, @Nullable List<String> categorisationFilters,
@Nullable Long modelSnapshotRetentionDays, @Nullable Long dailyModelSnapshotRetentionAfterDays,
@Nullable List<String> categorizationFilters,
@Nullable Map<String, Object> customSettings, @Nullable String modelSnapshotId,
@Nullable Version modelSnapshotMinVersion, @Nullable Version jobVersion, @Nullable Boolean clearJobFinishTime,
@Nullable Boolean allowLazyOpen) {
@ -98,8 +101,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
this.renormalizationWindowDays = renormalizationWindowDays;
this.backgroundPersistInterval = backgroundPersistInterval;
this.modelSnapshotRetentionDays = modelSnapshotRetentionDays;
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
this.resultsRetentionDays = resultsRetentionDays;
this.categorizationFilters = categorisationFilters;
this.categorizationFilters = categorizationFilters;
this.customSettings = customSettings;
this.modelSnapshotId = modelSnapshotId;
this.modelSnapshotMinVersion = modelSnapshotMinVersion;
@ -127,6 +131,11 @@ public class JobUpdate implements Writeable, ToXContentObject {
renormalizationWindowDays = in.readOptionalLong();
backgroundPersistInterval = in.readOptionalTimeValue();
modelSnapshotRetentionDays = in.readOptionalLong();
if (in.getVersion().onOrAfter(Version.V_7_8_0)) {
dailyModelSnapshotRetentionAfterDays = in.readOptionalLong();
} else {
dailyModelSnapshotRetentionAfterDays = null;
}
resultsRetentionDays = in.readOptionalLong();
if (in.readBoolean()) {
categorizationFilters = in.readStringList();
@ -178,6 +187,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
out.writeOptionalLong(renormalizationWindowDays);
out.writeOptionalTimeValue(backgroundPersistInterval);
out.writeOptionalLong(modelSnapshotRetentionDays);
if (out.getVersion().onOrAfter(Version.V_7_8_0)) {
out.writeOptionalLong(dailyModelSnapshotRetentionAfterDays);
}
out.writeOptionalLong(resultsRetentionDays);
out.writeBoolean(categorizationFilters != null);
if (categorizationFilters != null) {
@ -249,6 +261,10 @@ public class JobUpdate implements Writeable, ToXContentObject {
return modelSnapshotRetentionDays;
}
public Long getDailyModelSnapshotRetentionAfterDays() {
return dailyModelSnapshotRetentionAfterDays;
}
public Long getResultsRetentionDays() {
return resultsRetentionDays;
}
@ -313,6 +329,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
if (modelSnapshotRetentionDays != null) {
builder.field(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName(), modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.field(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName(), dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.field(Job.RESULTS_RETENTION_DAYS.getPreferredName(), resultsRetentionDays);
}
@ -367,6 +386,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
if (modelSnapshotRetentionDays != null) {
updateFields.add(Job.MODEL_SNAPSHOT_RETENTION_DAYS.getPreferredName());
}
if (dailyModelSnapshotRetentionAfterDays != null) {
updateFields.add(Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS.getPreferredName());
}
if (resultsRetentionDays != null) {
updateFields.add(Job.RESULTS_RETENTION_DAYS.getPreferredName());
}
@ -445,6 +467,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
if (modelSnapshotRetentionDays != null) {
builder.setModelSnapshotRetentionDays(modelSnapshotRetentionDays);
}
if (dailyModelSnapshotRetentionAfterDays != null) {
builder.setDailyModelSnapshotRetentionAfterDays(dailyModelSnapshotRetentionAfterDays);
}
if (resultsRetentionDays != null) {
builder.setResultsRetentionDays(resultsRetentionDays);
}
@ -483,6 +508,8 @@ public class JobUpdate implements Writeable, ToXContentObject {
&& (renormalizationWindowDays == null || Objects.equals(renormalizationWindowDays, job.getRenormalizationWindowDays()))
&& (backgroundPersistInterval == null || Objects.equals(backgroundPersistInterval, job.getBackgroundPersistInterval()))
&& (modelSnapshotRetentionDays == null || Objects.equals(modelSnapshotRetentionDays, job.getModelSnapshotRetentionDays()))
&& (dailyModelSnapshotRetentionAfterDays == null
|| Objects.equals(dailyModelSnapshotRetentionAfterDays, job.getDailyModelSnapshotRetentionAfterDays()))
&& (resultsRetentionDays == null || Objects.equals(resultsRetentionDays, job.getResultsRetentionDays()))
&& (categorizationFilters == null
|| Objects.equals(categorizationFilters, job.getAnalysisConfig().getCategorizationFilters()))
@ -533,6 +560,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
&& Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays)
&& Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval)
&& Objects.equals(this.modelSnapshotRetentionDays, that.modelSnapshotRetentionDays)
&& Objects.equals(this.dailyModelSnapshotRetentionAfterDays, that.dailyModelSnapshotRetentionAfterDays)
&& Objects.equals(this.resultsRetentionDays, that.resultsRetentionDays)
&& Objects.equals(this.categorizationFilters, that.categorizationFilters)
&& Objects.equals(this.customSettings, that.customSettings)
@ -546,8 +574,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
@Override
public int hashCode() {
return Objects.hash(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, renormalizationWindowDays,
backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, categorizationFilters, customSettings,
modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen);
backgroundPersistInterval, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays, resultsRetentionDays,
categorizationFilters, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime,
allowLazyOpen);
}
public static class DetectorUpdate implements Writeable, ToXContentObject {
@ -652,6 +681,7 @@ public class JobUpdate implements Writeable, ToXContentObject {
private Long renormalizationWindowDays;
private TimeValue backgroundPersistInterval;
private Long modelSnapshotRetentionDays;
private Long dailyModelSnapshotRetentionAfterDays;
private Long resultsRetentionDays;
private List<String> categorizationFilters;
private Map<String, Object> customSettings;
@ -710,6 +740,11 @@ public class JobUpdate implements Writeable, ToXContentObject {
return this;
}
public Builder setDailyModelSnapshotRetentionAfterDays(Long dailyModelSnapshotRetentionAfterDays) {
this.dailyModelSnapshotRetentionAfterDays = dailyModelSnapshotRetentionAfterDays;
return this;
}
public Builder setResultsRetentionDays(Long resultsRetentionDays) {
this.resultsRetentionDays = resultsRetentionDays;
return this;
@ -762,8 +797,9 @@ public class JobUpdate implements Writeable, ToXContentObject {
public JobUpdate build() {
return new JobUpdate(jobId, groups, description, detectorUpdates, modelPlotConfig, analysisLimits, backgroundPersistInterval,
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, categorizationFilters, customSettings,
modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime, allowLazyOpen);
renormalizationWindowDays, resultsRetentionDays, modelSnapshotRetentionDays, dailyModelSnapshotRetentionAfterDays,
categorizationFilters, customSettings, modelSnapshotId, modelSnapshotMinVersion, jobVersion, clearJobFinishTime,
allowLazyOpen);
}
}
}

4
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobTests.java
View File

@ -106,6 +106,7 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
assertNull(job.getRenormalizationWindowDays());
assertNull(job.getBackgroundPersistInterval());
assertThat(job.getModelSnapshotRetentionDays(), equalTo(1L));
assertNull(job.getDailyModelSnapshotRetentionAfterDays());
assertNull(job.getResultsRetentionDays());
assertNotNull(job.allInputFields());
assertFalse(job.allInputFields().isEmpty());
@ -626,6 +627,9 @@ public class JobTests extends AbstractSerializingTestCase<Job> {
if (randomBoolean()) {
builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setResultsRetentionDays(randomNonNegativeLong());
}

3
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java
View File

@ -75,6 +75,9 @@ public class JobUpdateTests extends AbstractSerializingTestCase<JobUpdate> {
if (randomBoolean()) {
update.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
update.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
update.setResultsRetentionDays(randomNonNegativeLong());
}

3
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/JobBuilderTests.java
View File

@ -60,6 +60,9 @@ public class JobBuilderTests extends AbstractSerializingTestCase<Job.Builder> {
if (randomBoolean()) {
builder.setModelSnapshotRetentionDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setDailyModelSnapshotRetentionAfterDays(randomNonNegativeLong());
}
if (randomBoolean()) {
builder.setResultsRetentionDays(randomNonNegativeLong());
}