honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Report peak model memory in ModelSizeStats (#59017) (#59055)

This commit is contained in:
Przemysław Witek 2020-07-06 12:55:12 +02:00 committed by GitHub
parent c651135562
commit f35ad0d4e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 113 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/process/ModelSizeStats.java
View File

@ -47,6 +47,7 @@ public class ModelSizeStats implements ToXContentObject {
* Field Names
*/
public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes");
public static final ParseField PEAK_MODEL_BYTES_FIELD = new ParseField("peak_model_bytes");
public static final ParseField MODEL_BYTES_EXCEEDED_FIELD = new ParseField("model_bytes_exceeded");
public static final ParseField MODEL_BYTES_MEMORY_LIMIT_FIELD = new ParseField("model_bytes_memory_limit");
public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count");
@ -70,6 +71,7 @@ public class ModelSizeStats implements ToXContentObject {
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD);
PARSER.declareLong(Builder::setPeakModelBytes, PEAK_MODEL_BYTES_FIELD);
PARSER.declareLong(Builder::setModelBytesExceeded, MODEL_BYTES_EXCEEDED_FIELD);
PARSER.declareLong(Builder::setModelBytesMemoryLimit, MODEL_BYTES_MEMORY_LIMIT_FIELD);
PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD);
@ -133,6 +135,7 @@ public class ModelSizeStats implements ToXContentObject {
private final String jobId;
private final long modelBytes;
private final Long peakModelBytes;
private final Long modelBytesExceeded;
private final Long modelBytesMemoryLimit;
private final long totalByFieldCount;
@ -150,13 +153,14 @@ public class ModelSizeStats implements ToXContentObject {
private final Date timestamp;
private final Date logTime;
private ModelSizeStats(String jobId, long modelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, long totalByFieldCount,
long totalOverFieldCount, long totalPartitionFieldCount, long bucketAllocationFailuresCount,
MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount,
long rareCategoryCount, long deadCategoryCount, long failedCategoryCount,
private ModelSizeStats(String jobId, long modelBytes, Long peakModelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit,
long totalByFieldCount, long totalOverFieldCount, long totalPartitionFieldCount,
long bucketAllocationFailuresCount, MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount,
long frequentCategoryCount, long rareCategoryCount, long deadCategoryCount, long failedCategoryCount,
CategorizationStatus categorizationStatus, Date timestamp, Date logTime) {
this.jobId = jobId;
this.modelBytes = modelBytes;
this.peakModelBytes = peakModelBytes;
this.modelBytesExceeded = modelBytesExceeded;
this.modelBytesMemoryLimit = modelBytesMemoryLimit;
this.totalByFieldCount = totalByFieldCount;
@ -182,6 +186,9 @@ public class ModelSizeStats implements ToXContentObject {
builder.field(Job.ID.getPreferredName(), jobId);
builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes);
if (peakModelBytes != null) {
builder.field(PEAK_MODEL_BYTES_FIELD.getPreferredName(), peakModelBytes);
}
if (modelBytesExceeded != null) {
builder.field(MODEL_BYTES_EXCEEDED_FIELD.getPreferredName(), modelBytesExceeded);
}
@ -217,6 +224,10 @@ public class ModelSizeStats implements ToXContentObject {
return modelBytes;
}
public Long getPeakModelBytes() {
return peakModelBytes;
}
public Long getModelBytesExceeded() {
return modelBytesExceeded;
}
@ -293,7 +304,8 @@ public class ModelSizeStats implements ToXContentObject {
@Override
public int hashCode() {
return Objects.hash(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
return Objects.hash(
jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
totalPartitionFieldCount, this.bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
}
@ -313,7 +325,9 @@ public class ModelSizeStats implements ToXContentObject {
ModelSizeStats that = (ModelSizeStats) other;
return this.modelBytes == that.modelBytes && Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded)
return this.modelBytes == that.modelBytes
&& Objects.equals(this.peakModelBytes, that.peakModelBytes)
&& Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded)
&& Objects.equals(this.modelBytesMemoryLimit, that.modelBytesMemoryLimit) && this.totalByFieldCount == that.totalByFieldCount
&& this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount
&& this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount
@ -334,6 +348,7 @@ public class ModelSizeStats implements ToXContentObject {
private final String jobId;
private long modelBytes;
private Long peakModelBytes;
private Long modelBytesExceeded;
private Long modelBytesMemoryLimit;
private long totalByFieldCount;
@ -361,6 +376,7 @@ public class ModelSizeStats implements ToXContentObject {
public Builder(ModelSizeStats modelSizeStats) {
this.jobId = modelSizeStats.jobId;
this.modelBytes = modelSizeStats.modelBytes;
this.peakModelBytes = modelSizeStats.peakModelBytes;
this.modelBytesExceeded = modelSizeStats.modelBytesExceeded;
this.modelBytesMemoryLimit = modelSizeStats.modelBytesMemoryLimit;
this.totalByFieldCount = modelSizeStats.totalByFieldCount;
@ -384,6 +400,11 @@ public class ModelSizeStats implements ToXContentObject {
return this;
}
public Builder setPeakModelBytes(long peakModelBytes) {
this.peakModelBytes = peakModelBytes;
return this;
}
public Builder setModelBytesExceeded(long modelBytesExceeded) {
this.modelBytesExceeded = modelBytesExceeded;
return this;
@ -467,7 +488,8 @@ public class ModelSizeStats implements ToXContentObject {
}
public ModelSizeStats build() {
return new ModelSizeStats(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
return new ModelSizeStats(
jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
}

33
client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java
View File

@ -157,8 +157,8 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541587919000, " +
"\"description\":\"State persisted due to job close at 2018-11-07T10:51:59+0000\", \"snapshot_id\":\"1541587919\"," +
"\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," +
"\"model_bytes\":51722, \"model_bytes_exceeded\":10762, \"model_bytes_memory_limit\":40960, \"total_by_field_count\":3, " +
"\"total_over_field_count\":0, \"total_partition_field_count\":2," +
"\"model_bytes\":51722, \"peak_model_bytes\":61322, \"model_bytes_exceeded\":10762, \"model_bytes_memory_limit\":40960," +
"\"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," +
"\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541587919000," +
" \"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000," +
" \"retain\":false }", XContentType.JSON);
@ -169,10 +169,10 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541588919000, " +
"\"description\":\"State persisted due to job close at 2018-11-07T11:08:39+0000\", \"snapshot_id\":\"1541588919\"," +
"\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," +
"\"model_bytes\":51722, \"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," +
"\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541588919000," +
"\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000, " +
"\"retain\":false }", XContentType.JSON);
"\"model_bytes\":51722, \"peak_model_bytes\":61322, \"total_by_field_count\":3, \"total_over_field_count\":0," +
"\"total_partition_field_count\":2,\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\"," +
"\"log_time\":1541588919000,\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," +
"\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON);
bulkRequest.add(indexRequest);
}
{
@ -180,10 +180,10 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"timestamp\":1541589919000, " +
"\"description\":\"State persisted due to job close at 2018-11-07T11:25:19+0000\", \"snapshot_id\":\"1541589919\"," +
"\"snapshot_doc_count\":1, \"model_size_stats\":{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"model_size_stats\"," +
"\"model_bytes\":51722, \"total_by_field_count\":3, \"total_over_field_count\":0, \"total_partition_field_count\":2," +
"\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\", \"log_time\":1541589919000," +
"\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000, \"latest_result_time_stamp\":1519930800000," +
"\"retain\":false }", XContentType.JSON);
"\"model_bytes\":51722, \"peak_model_bytes\":61322, \"total_by_field_count\":3, \"total_over_field_count\":0," +
"\"total_partition_field_count\":2,\"bucket_allocation_failures_count\":0, \"memory_status\":\"ok\"," +
"\"log_time\":1541589919000,\"timestamp\":1519930800000},\"latest_record_time_stamp\":1519931700000," +
"\"latest_result_time_stamp\":1519930800000, \"retain\":false }", XContentType.JSON);
bulkRequest.add(indexRequest);
}
}
@ -227,6 +227,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -247,6 +248,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -267,6 +269,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(2).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(2).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(2).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(2).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -298,6 +301,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(2).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(2).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(2).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L));
assertThat(response.snapshots().get(2).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L));
assertThat(response.snapshots().get(2).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -318,6 +322,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -338,6 +343,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -369,6 +375,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -401,6 +408,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -422,6 +430,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -452,6 +461,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -494,6 +504,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(10762L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(40960L));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -514,6 +525,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(1).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(1).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(1).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(1).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));
@ -545,6 +557,7 @@ public class MachineLearningGetResultsIT extends ESRestHighLevelClientTestCase {
assertThat(response.snapshots().get(0).getLatestResultTimeStamp(), equalTo(new Date(1519930800000L)));
assertThat(response.snapshots().get(0).getModelSizeStats().getJobId(), equalTo(JOB_ID));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytes(), equalTo(51722L));
assertThat(response.snapshots().get(0).getModelSizeStats().getPeakModelBytes(), equalTo(61322L));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesExceeded(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getModelBytesMemoryLimit(), equalTo(null));
assertThat(response.snapshots().get(0).getModelSizeStats().getTotalByFieldCount(), equalTo(3L));

4
client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/process/ModelSizeStatsTests.java
View File

@ -32,6 +32,7 @@ public class ModelSizeStatsTests extends AbstractXContentTestCase<ModelSizeStats
public void testDefaultConstructor() {
ModelSizeStats stats = new ModelSizeStats.Builder("foo").build();
assertEquals(0, stats.getModelBytes());
assertNull(stats.getPeakModelBytes());
assertNull(stats.getModelBytesExceeded());
assertNull(stats.getModelBytesMemoryLimit());
assertEquals(0, stats.getTotalByFieldCount());
@ -77,6 +78,9 @@ public class ModelSizeStatsTests extends AbstractXContentTestCase<ModelSizeStats
if (randomBoolean()) {
stats.setModelBytes(randomNonNegativeLong());
}
if (randomBoolean()) {
stats.setPeakModelBytes(randomNonNegativeLong());
}
if (randomBoolean()) {
stats.setModelBytesExceeded(randomNonNegativeLong());
}

4
docs/reference/ml/anomaly-detection/apis/get-job-stats.asciidoc
View File

@ -247,6 +247,10 @@ include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-bytes-exceeded]
(long)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-anomaly-jobs]
`peak_model_bytes`:::
(long)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=peak-model-bytes]
`rare_category_count`:::
(long)
include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=rare-category-count]

4
docs/reference/ml/ml-shared.asciidoc
View File

@ -1114,6 +1114,10 @@ The field used to segment the analysis. When you use this property, you have
completely independent baselines for each value of this field.
end::partition-field-name[]
tag::peak-model-bytes[]
The peak number of bytes of memory ever used by the models.
end::peak-model-bytes[]
tag::per-partition-categorization[]
Settings related to how categorization interacts with partition fields.
end::per-partition-categorization[]

53
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java
View File

@ -38,6 +38,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
* Field Names
*/
public static final ParseField MODEL_BYTES_FIELD = new ParseField("model_bytes");
public static final ParseField PEAK_MODEL_BYTES_FIELD = new ParseField("peak_model_bytes");
public static final ParseField MODEL_BYTES_EXCEEDED_FIELD = new ParseField("model_bytes_exceeded");
public static final ParseField MODEL_BYTES_MEMORY_LIMIT_FIELD = new ParseField("model_bytes_memory_limit");
public static final ParseField TOTAL_BY_FIELD_COUNT_FIELD = new ParseField("total_by_field_count");
@ -65,6 +66,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID);
parser.declareString((modelSizeStat, s) -> {}, Result.RESULT_TYPE);
parser.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD);
parser.declareLong(Builder::setPeakModelBytes, PEAK_MODEL_BYTES_FIELD);
parser.declareLong(Builder::setModelBytesExceeded, MODEL_BYTES_EXCEEDED_FIELD);
parser.declareLong(Builder::setModelBytesMemoryLimit, MODEL_BYTES_MEMORY_LIMIT_FIELD);
parser.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD);
@ -118,6 +120,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
private final String jobId;
private final long modelBytes;
private final Long peakModelBytes;
private final Long modelBytesExceeded;
private final Long modelBytesMemoryLimit;
private final long totalByFieldCount;
@ -135,13 +138,14 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
private final Date timestamp;
private final Date logTime;
private ModelSizeStats(String jobId, long modelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit, long totalByFieldCount,
long totalOverFieldCount, long totalPartitionFieldCount, long bucketAllocationFailuresCount,
MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount,
long rareCategoryCount, long deadCategoryCount, long failedCategoryCount,
private ModelSizeStats(String jobId, long modelBytes, Long peakModelBytes, Long modelBytesExceeded, Long modelBytesMemoryLimit,
long totalByFieldCount, long totalOverFieldCount, long totalPartitionFieldCount,
long bucketAllocationFailuresCount, MemoryStatus memoryStatus, long categorizedDocCount, long totalCategoryCount,
long frequentCategoryCount, long rareCategoryCount, long deadCategoryCount, long failedCategoryCount,
CategorizationStatus categorizationStatus, Date timestamp, Date logTime) {
this.jobId = jobId;
this.modelBytes = modelBytes;
this.peakModelBytes = peakModelBytes;
this.modelBytesExceeded = modelBytesExceeded;
this.modelBytesMemoryLimit = modelBytesMemoryLimit;
this.totalByFieldCount = totalByFieldCount;
@ -163,6 +167,11 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
public ModelSizeStats(StreamInput in) throws IOException {
jobId = in.readString();
modelBytes = in.readVLong();
if (in.getVersion().onOrAfter(Version.V_7_9_0)) {
peakModelBytes = in.readOptionalLong();
} else {
peakModelBytes = null;
}
if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
modelBytesExceeded = in.readOptionalLong();
} else {
@ -215,6 +224,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
public void writeTo(StreamOutput out) throws IOException {
out.writeString(jobId);
out.writeVLong(modelBytes);
if (out.getVersion().onOrAfter(Version.V_7_9_0)) {
out.writeOptionalLong(peakModelBytes);
}
if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
out.writeOptionalLong(modelBytesExceeded);
}
@ -257,6 +269,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
builder.field(Job.ID.getPreferredName(), jobId);
builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE);
builder.field(MODEL_BYTES_FIELD.getPreferredName(), modelBytes);
if (peakModelBytes != null) {
builder.field(PEAK_MODEL_BYTES_FIELD.getPreferredName(), peakModelBytes);
}
if (modelBytesExceeded != null) {
builder.field(MODEL_BYTES_EXCEEDED_FIELD.getPreferredName(), modelBytesExceeded);
}
@ -291,6 +306,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
return modelBytes;
}
public Long getPeakModelBytes() {
return peakModelBytes;
}
public Long getModelBytesExceeded() {
return modelBytesExceeded;
}
@ -366,9 +385,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
@Override
public int hashCode() {
// this.id excluded here as it is generated by the datastore
return Objects.hash(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
return Objects.hash(
jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
}
/**
@ -386,7 +406,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
ModelSizeStats that = (ModelSizeStats) other;
return this.modelBytes == that.modelBytes && Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded)
return this.modelBytes == that.modelBytes
&& Objects.equals(this.peakModelBytes, that.peakModelBytes)
&& Objects.equals(this.modelBytesExceeded, that.modelBytesExceeded)
&& Objects.equals(this.modelBytesMemoryLimit, that.modelBytesMemoryLimit)
&& this.totalByFieldCount == that.totalByFieldCount
&& this.totalOverFieldCount == that.totalOverFieldCount && this.totalPartitionFieldCount == that.totalPartitionFieldCount
@ -408,6 +430,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
private final String jobId;
private long modelBytes;
private Long peakModelBytes;
private Long modelBytesExceeded;
private Long modelBytesMemoryLimit;
private long totalByFieldCount;
@ -435,6 +458,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
public Builder(ModelSizeStats modelSizeStats) {
this.jobId = modelSizeStats.jobId;
this.modelBytes = modelSizeStats.modelBytes;
this.peakModelBytes = modelSizeStats.peakModelBytes;
this.modelBytesExceeded = modelSizeStats.modelBytesExceeded;
this.modelBytesMemoryLimit = modelSizeStats.modelBytesMemoryLimit;
this.totalByFieldCount = modelSizeStats.totalByFieldCount;
@ -458,6 +482,11 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
return this;
}
public Builder setPeakModelBytes(long peakModelBytes) {
this.peakModelBytes = peakModelBytes;
return this;
}
public Builder setModelBytesExceeded(long modelBytesExceeded) {
this.modelBytesExceeded = modelBytesExceeded;
return this;
@ -541,10 +570,10 @@ public class ModelSizeStats implements ToXContentObject, Writeable {
}
public ModelSizeStats build() {
return new ModelSizeStats(jobId, modelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp,
logTime);
return new ModelSizeStats(
jobId, modelBytes, peakModelBytes, modelBytesExceeded, modelBytesMemoryLimit, totalByFieldCount, totalOverFieldCount,
totalPartitionFieldCount, bucketAllocationFailuresCount, memoryStatus, categorizedDocCount, totalCategoryCount,
frequentCategoryCount, rareCategoryCount, deadCategoryCount, failedCategoryCount, categorizationStatus, timestamp, logTime);
}
}
}

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
View File

@ -173,6 +173,7 @@ public final class ReservedFieldNames {
ForecastRequestStats.MEMORY_USAGE.getPreferredName(),
ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName(),
ModelSizeStats.PEAK_MODEL_BYTES_FIELD.getPreferredName(),
ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName(),
ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName(),
ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName(),

3
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json
View File

@ -362,6 +362,9 @@
"model_bytes" : {
"type" : "long"
},
"peak_model_bytes" : {
"type" : "long"
},
"result_type" : {
"type" : "keyword"
},

4
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java
View File

@ -22,6 +22,7 @@ public class ModelSizeStatsTests extends AbstractSerializingTestCase<ModelSizeSt
public void testDefaultConstructor() {
ModelSizeStats stats = new ModelSizeStats.Builder("foo").build();
assertEquals(0, stats.getModelBytes());
assertNull(stats.getPeakModelBytes());
assertNull(stats.getModelBytesExceeded());
assertNull(stats.getModelBytesMemoryLimit());
assertEquals(0, stats.getTotalByFieldCount());
@ -67,6 +68,9 @@ public class ModelSizeStatsTests extends AbstractSerializingTestCase<ModelSizeSt
if (randomBoolean()) {
stats.setModelBytes(randomNonNegativeLong());
}
if (randomBoolean()) {
stats.setPeakModelBytes(randomNonNegativeLong());
}
if (randomBoolean()) {
stats.setModelBytesExceeded(randomNonNegativeLong());
}