From f35ad0d4e118b429b1536d060de451798aef6e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Przemys=C5=82aw=20Witek?= Date: Mon, 6 Jul 2020 12:55:12 +0200 Subject: [PATCH] Report peak model memory in ModelSizeStats (#59017) (#59055) --- .../client/ml/job/process/ModelSizeStats.java | 36 ++++++++++--- .../client/MachineLearningGetResultsIT.java | 33 ++++++++---- .../ml/job/process/ModelSizeStatsTests.java | 4 ++ .../apis/get-job-stats.asciidoc | 4 ++ docs/reference/ml/ml-shared.asciidoc | 4 ++ .../autodetect/state/ModelSizeStats.java | 53 ++++++++++++++----- .../ml/job/results/ReservedFieldNames.java | 1 + .../results_index_mappings.json | 3 ++ .../autodetect/state/ModelSizeStatsTests.java | 4 ++ 9 files changed, 113 insertions(+), 29 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/ModelSizeStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/ModelSizeStats.java index 7d1538f529e..05099ca1ceb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/ModelSizeStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/ModelSizeStats.java @@ -47,6 +47,7 @@ public class ModelSizeStats implements ToXContentObject { * Field Names */ public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes"); + public static final ParseField PEAK_MODEL_BYTES_FIELD = new ParseField("peak_model_bytes"); public static final ParseField MODEL_BYTES_EXCEEDED_FIELD = new ParseField("model_bytes_exceeded"); public static final ParseField MODEL_BYTES_MEMORY_LIMIT_FIELD = new ParseField("model_bytes_memory_limit"); public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count"); @@ -70,6 +71,7 @@ public class ModelSizeStats implements ToXContentObject { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); + PARSER.declareLong(Builder::setPeakModelBytes, PEAK_MODEL_BYTES_FIELD); PARSER.declareLong(Builder::setModelBytesExceeded, MODEL_BYTES_EXCEEDED_FIELD); PARSER.declareLong(Builder::setModelBytesMemoryLimit, MODEL_BYTES_MEMORY_LIMIT_FIELD); PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); @@ -133,6 +135,7 @@ public class ModelSizeStats implements ToXContentObject { private final String jobId; private final long modelBytes; + private final Long peakModelBytes; private final Long modelBytesExceeded; private final Long modelBytesMemoryLimit; private final long totalByFieldCount; @@ -150,13 +153,14 @@ public class ModelSizeStats implements ToXContentObject { private final Date timestamp; private final Date logTime; - private ModelSizeStats(String jobId, long modelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, long totalByFieldCount, - long totalOverFieldCount, long totalPartitionFieldCount, long bucketAllocationFailuresCount, - MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount, - long rareCategoryCount, long deadCategoryCount, long failedCategoryCount, + private ModelSizeStats(String jobId, long modelBytes, Long peakModelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, + long totalByFieldCount, long totalOverFieldCount, long totalPartitionFieldCount, + long bucketAllocationFailuresCount, MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, + long frequentCategoryCount, long rareCategoryCount, long deadCategoryCount, long failedCategoryCount, CategorizationStatus categorizationStatus, Date timestamp, Date logTime) { this.jobId = jobId; this.modelBytes = modelBytes; + this.peakModelBytes = peakModelBytes; this.modelBytesExceeded = modelBytesExceeded; this.modelBytesMemoryLimit = modelBytesMemoryLimit; this.totalByFieldCount = totalByFieldCount; @@ -182,6 +186,9 @@ public class ModelSizeStats implements ToXContentObject { builder.field(Job.ID.getPreferredName(), jobId); builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes); + if (peakModelBytes != null) { + builder.field(PEAK_MODEL_BYTES_FIELD.getPreferredName(), peakModelBytes); + } if (modelBytesExceeded != null) { builder.field(MODEL_BYTES_EXCEEDED_FIELD.getPreferredName(), modelBytesExceeded); } @@ -217,6 +224,10 @@ public class ModelSizeStats implements ToXContentObject { return modelBytes; } + public Long getPeakModelBytes() { + return peakModelBytes; + } + public Long getModelBytesExceeded() { return modelBytesExceeded; } @@ -293,7 +304,8 @@ public class ModelSizeStats implements ToXContentObject { @Override public int hashCode() { - return Objects.hash(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, + return Objects.hash( + jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, this.bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount, frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime); } @@ -313,7 +325,9 @@ public class ModelSizeStats implements ToXContentObject { ModelSizeStats that = (ModelSizeStats) other; - return this.modelBytes == that.modelBytes && Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded) + return this.modelBytes == that.modelBytes + && Objects.equals(this.peakModelBytes, that.peakModelBytes) + && Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded) && Objects.equals(this.modelBytesMemoryLimit, that.modelBytesMemoryLimit) && this.totalByFieldCount == that.totalByFieldCount && this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount && this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount @@ -334,6 +348,7 @@ public class ModelSizeStats implements ToXContentObject { private final String jobId; private long modelBytes; + private Long peakModelBytes; private Long modelBytesExceeded; private Long modelBytesMemoryLimit; private long totalByFieldCount; @@ -361,6 +376,7 @@ public class ModelSizeStats implements ToXContentObject { public Builder(ModelSizeStats modelSizeStats) { this.jobId = modelSizeStats.jobId; this.modelBytes = modelSizeStats.modelBytes; + this.peakModelBytes = modelSizeStats.peakModelBytes; this.modelBytesExceeded = modelSizeStats.modelBytesExceeded; this.modelBytesMemoryLimit = modelSizeStats.modelBytesMemoryLimit; this.totalByFieldCount = modelSizeStats.totalByFieldCount; @@ -384,6 +400,11 @@ public class ModelSizeStats implements ToXContentObject { return this; } + public Builder setPeakModelBytes(long peakModelBytes) { + this.peakModelBytes = peakModelBytes; + return this; + } + public Builder setModelBytesExceeded(long modelBytesExceeded) { this.modelBytesExceeded = modelBytesExceeded; return this; @@ -467,7 +488,8 @@ public class ModelSizeStats implements ToXContentObject { } public ModelSizeStats build() { - return new ModelSizeStats(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, + return new ModelSizeStats( + jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount, frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java index 34ca5cd2aa4..98eaff54bbb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java @@ -157,8 +157,8 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541587919000, " + "\"description\":\"State persisted due to job close at 2018-11-07T10:51:59+0000\", \"snapshot_id\":\"1541587919\"," + "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + - "\"model_bytes\":51722, \"model_bytes_exceeded\":10762, \"model_bytes_memory_limit\":40960, \"total_by_field_count\":3, " + - "\"total_over_field_count\":0, \"total_partition_field_count\":2," + + "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"model_bytes_exceeded\":10762, \"model_bytes_memory_limit\":40960," + + "\"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," + "\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541587919000," + " \"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000," + " \"retain\":false }", XContentType.JSON); @@ -169,10 +169,10 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541588919000, " + "\"description\":\"State persisted due to job close at 2018-11-07T11:08:39+0000\", \"snapshot_id\":\"1541588919\"," + "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + - "\"model_bytes\":51722, \"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," + - "\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541588919000," + - "\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000, " + - "\"retain\":false }", XContentType.JSON); + "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"total_by_field_count\":3, \"total_over_field_count\":0," + + "\"total_partition_field_count\":2,\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\"," + + "\"log_time\":1541588919000,\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," + + "\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON); bulkRequest.add(indexRequest); } { @@ -180,10 +180,10 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541589919000, " + "\"description\":\"State persisted due to job close at 2018-11-07T11:25:19+0000\", \"snapshot_id\":\"1541589919\"," + "\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," + - "\"model_bytes\":51722, \"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," + - "\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541589919000," + - "\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000," + - "\"retain\":false }", XContentType.JSON); + "\"model_bytes\":51722, \"peak_model_bytes\":61322, \"total_by_field_count\":3, \"total_over_field_count\":0," + + "\"total_partition_field_count\":2,\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\"," + + "\"log_time\":1541589919000,\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," + + "\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON); bulkRequest.add(indexRequest); } } @@ -227,6 +227,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -247,6 +248,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -267,6 +269,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(2).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(2).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(2).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(2).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -298,6 +301,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(2).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(2).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(2).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L)); assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L)); assertThat(response.snapshots().get(2).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -318,6 +322,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -338,6 +343,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -369,6 +375,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -401,6 +408,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -422,6 +430,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -452,6 +461,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -494,6 +504,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -514,6 +525,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); @@ -545,6 +557,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase { assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L))); assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L)); + assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null)); assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/ModelSizeStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/ModelSizeStatsTests.java index fc692cd43b2..4404fb1cf4e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/ModelSizeStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/ModelSizeStatsTests.java @@ -32,6 +32,7 @@ public class ModelSizeStatsTests extends AbstractXContentTestCase {}, Result.RESULT_TYPE); parser.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); + parser.declareLong(Builder::setPeakModelBytes, PEAK_MODEL_BYTES_FIELD); parser.declareLong(Builder::setModelBytesExceeded, MODEL_BYTES_EXCEEDED_FIELD); parser.declareLong(Builder::setModelBytesMemoryLimit, MODEL_BYTES_MEMORY_LIMIT_FIELD); parser.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); @@ -118,6 +120,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable { private final String jobId; private final long modelBytes; + private final Long peakModelBytes; private final Long modelBytesExceeded; private final Long modelBytesMemoryLimit; private final long totalByFieldCount; @@ -135,13 +138,14 @@ public class ModelSizeStats implements ToXContentObject, Writeable { private final Date timestamp; private final Date logTime; - private ModelSizeStats(String jobId, long modelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, long totalByFieldCount, - long totalOverFieldCount, long totalPartitionFieldCount, long bucketAllocationFailuresCount, - MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount, - long rareCategoryCount, long deadCategoryCount, long failedCategoryCount, + private ModelSizeStats(String jobId, long modelBytes, Long peakModelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, + long totalByFieldCount, long totalOverFieldCount, long totalPartitionFieldCount, + long bucketAllocationFailuresCount, MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, + long frequentCategoryCount, long rareCategoryCount, long deadCategoryCount, long failedCategoryCount, CategorizationStatus categorizationStatus, Date timestamp, Date logTime) { this.jobId = jobId; this.modelBytes = modelBytes; + this.peakModelBytes = peakModelBytes; this.modelBytesExceeded = modelBytesExceeded; this.modelBytesMemoryLimit = modelBytesMemoryLimit; this.totalByFieldCount = totalByFieldCount; @@ -163,6 +167,11 @@ public class ModelSizeStats implements ToXContentObject, Writeable { public ModelSizeStats(StreamInput in) throws IOException { jobId = in.readString(); modelBytes = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_7_9_0)) { + peakModelBytes = in.readOptionalLong(); + } else { + peakModelBytes = null; + } if (in.getVersion().onOrAfter(Version.V_7_2_0)) { modelBytesExceeded = in.readOptionalLong(); } else { @@ -215,6 +224,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable { public void writeTo(StreamOutput out) throws IOException { out.writeString(jobId); out.writeVLong(modelBytes); + if (out.getVersion().onOrAfter(Version.V_7_9_0)) { + out.writeOptionalLong(peakModelBytes); + } if (out.getVersion().onOrAfter(Version.V_7_2_0)) { out.writeOptionalLong(modelBytesExceeded); } @@ -257,6 +269,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable { builder.field(Job.ID.getPreferredName(), jobId); builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes); + if (peakModelBytes != null) { + builder.field(PEAK_MODEL_BYTES_FIELD.getPreferredName(), peakModelBytes); + } if (modelBytesExceeded != null) { builder.field(MODEL_BYTES_EXCEEDED_FIELD.getPreferredName(), modelBytesExceeded); } @@ -291,6 +306,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable { return modelBytes; } + public Long getPeakModelBytes() { + return peakModelBytes; + } + public Long getModelBytesExceeded() { return modelBytesExceeded; } @@ -366,9 +385,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable { @Override public int hashCode() { // this.id excluded here as it is generated by the datastore - return Objects.hash(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, - totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount, - frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime); + return Objects.hash( + jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, + totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount, + frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime); } /** @@ -386,7 +406,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable { ModelSizeStats that = (ModelSizeStats) other; - return this.modelBytes == that.modelBytes && Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded) + return this.modelBytes == that.modelBytes + && Objects.equals(this.peakModelBytes, that.peakModelBytes) + && Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded) && Objects.equals(this.modelBytesMemoryLimit, that.modelBytesMemoryLimit) && this.totalByFieldCount == that.totalByFieldCount && this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount @@ -408,6 +430,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable { private final String jobId; private long modelBytes; + private Long peakModelBytes; private Long modelBytesExceeded; private Long modelBytesMemoryLimit; private long totalByFieldCount; @@ -435,6 +458,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable { public Builder(ModelSizeStats modelSizeStats) { this.jobId = modelSizeStats.jobId; this.modelBytes = modelSizeStats.modelBytes; + this.peakModelBytes = modelSizeStats.peakModelBytes; this.modelBytesExceeded = modelSizeStats.modelBytesExceeded; this.modelBytesMemoryLimit = modelSizeStats.modelBytesMemoryLimit; this.totalByFieldCount = modelSizeStats.totalByFieldCount; @@ -458,6 +482,11 @@ public class ModelSizeStats implements ToXContentObject, Writeable { return this; } + public Builder setPeakModelBytes(long peakModelBytes) { + this.peakModelBytes = peakModelBytes; + return this; + } + public Builder setModelBytesExceeded(long modelBytesExceeded) { this.modelBytesExceeded = modelBytesExceeded; return this; @@ -541,10 +570,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable { } public ModelSizeStats build() { - return new ModelSizeStats(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, - totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount, - frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, - logTime); + return new ModelSizeStats( + jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount, + totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount, + frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index c26bd2c6ce9..1fa00b65a83 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -173,6 +173,7 @@ public final class ReservedFieldNames { ForecastRequestStats.MEMORY_USAGE.getPreferredName(), ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName(), + ModelSizeStats.PEAK_MODEL_BYTES_FIELD.getPreferredName(), ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(), ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(), ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(), diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json index 20026c56d62..f0b98cd4fbf 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json @@ -362,6 +362,9 @@ "model_bytes" : { "type" : "long" }, + "peak_model_bytes" : { + "type" : "long" + }, "result_type" : { "type" : "keyword" }, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java index 362ef634558..ebac45b0f84 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java @@ -22,6 +22,7 @@ public class ModelSizeStatsTests extends AbstractSerializingTestCase