honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-17 10:25:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ML] Refactor method to process model size stats (elastic/x-pack-elasticsearch#4175)

Browse Source 
Original commit: elastic/x-pack-elasticsearch@6262ff33a9
This commit is contained in:
Dimitris Athanasiou 2018-03-21 14:54:55 +00:00 committed by GitHub
parent 506694c180
commit 97703bf0da
1 changed files with 20 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutoDetectResultProcessor.java
View File

@ -241,22 +241,7 @@ public class AutoDetectResultProcessor {
}
ModelSizeStats modelSizeStats = result.getModelSizeStats();
if (modelSizeStats != null) {
LOGGER.trace("[{}] Parsed ModelSizeStats: {} / {} / {} / {} / {} / {}",
context.jobId, modelSizeStats.getModelBytes(), modelSizeStats.getTotalByFieldCount(),
modelSizeStats.getTotalOverFieldCount(), modelSizeStats.getTotalPartitionFieldCount(),
modelSizeStats.getBucketAllocationFailuresCount(), modelSizeStats.getMemoryStatus());
latestModelSizeStats = modelSizeStats;
haveNewLatestModelSizeStats = true;
persister.persistModelSizeStats(modelSizeStats);
// This is a crude way to NOT refresh the index and NOT attempt to update established model memory during the first 20 buckets
// because this is when the model size stats are likely to be least stable and lots of updates will be coming through, and
// we'll NEVER consider memory usage to be established during this period
if (restoredSnapshot || bucketCount >= JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) {
// We need to make all results written up to and including these stats available for the established memory calculation
persister.commitResultWrites(context.jobId);
updateEstablishedModelMemoryOnJob(modelSizeStats.getTimestamp(), modelSizeStats);
}
processModelSizeStats(context, modelSizeStats);
}
ModelSnapshot modelSnapshot = result.getModelSnapshot();
if (modelSnapshot != null) {
@ -293,6 +278,25 @@ public class AutoDetectResultProcessor {
}
}
private void processModelSizeStats(Context context, ModelSizeStats modelSizeStats) {
LOGGER.trace("[{}] Parsed ModelSizeStats: {} / {} / {} / {} / {} / {}",
context.jobId, modelSizeStats.getModelBytes(), modelSizeStats.getTotalByFieldCount(),
modelSizeStats.getTotalOverFieldCount(), modelSizeStats.getTotalPartitionFieldCount(),
modelSizeStats.getBucketAllocationFailuresCount(), modelSizeStats.getMemoryStatus());
latestModelSizeStats = modelSizeStats;
haveNewLatestModelSizeStats = true;
persister.persistModelSizeStats(modelSizeStats);
// This is a crude way to NOT refresh the index and NOT attempt to update established model memory during the first 20 buckets
// because this is when the model size stats are likely to be least stable and lots of updates will be coming through, and
// we'll NEVER consider memory usage to be established during this period
if (restoredSnapshot || bucketCount >= JobProvider.BUCKETS_FOR_ESTABLISHED_MEMORY_SIZE) {
// We need to make all results written up to and including these stats available for the established memory calculation
persister.commitResultWrites(context.jobId);
updateEstablishedModelMemoryOnJob(modelSizeStats.getTimestamp(), modelSizeStats);
}
}
protected void updateModelSnapshotOnJob(ModelSnapshot modelSnapshot) {
JobUpdate update = new JobUpdate.Builder(jobId)
.setModelSnapshotId(modelSnapshot.getSnapshotId())