honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Uplift model memory limit on job migration (#37126) 

When a 6.1-6.3 job is opened in a later version
we increase the model memory limit by 30% if it's
below 0.5GB. The migration of jobs from cluster
state to the config index changes the job version,
so we need to also do this uplift as part of that
config migration.

Relates #36961
This commit is contained in:
David Roberts 2019-01-04 12:21:28 +00:00 committed by GitHub
parent 21d52f0dab
commit ff7df40b20
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
@ -403,11 +404,23 @@ public class MlConfigMigrator {
Map<String, Object> custom = job.getCustomSettings() == null ? new HashMap<>() : new HashMap<>(job.getCustomSettings()); Map<String, Object> custom = job.getCustomSettings() == null ? new HashMap<>() : new HashMap<>(job.getCustomSettings());
custom.put(MIGRATED_FROM_VERSION, job.getJobVersion()); custom.put(MIGRATED_FROM_VERSION, job.getJobVersion());
builder.setCustomSettings(custom); builder.setCustomSettings(custom);
// Increase the model memory limit for 6.1 - 6.3 jobs
Version jobVersion = job.getJobVersion();
if (jobVersion != null && jobVersion.onOrAfter(Version.V_6_1_0) && jobVersion.before(Version.V_6_3_0)) {
// Increase model memory limit if < 512MB
if (job.getAnalysisLimits() != null && job.getAnalysisLimits().getModelMemoryLimit() != null &&
job.getAnalysisLimits().getModelMemoryLimit() < 512L) {
long updatedModelMemoryLimit = (long) (job.getAnalysisLimits().getModelMemoryLimit() * 1.3);
AnalysisLimits limits = new AnalysisLimits(updatedModelMemoryLimit,
job.getAnalysisLimits().getCategorizationExamplesLimit());
builder.setAnalysisLimits(limits);
}
}
// Pre v5.5 (ml beta) jobs do not have a version. // Pre v5.5 (ml beta) jobs do not have a version.
// These jobs cannot be opened, we rely on the missing version // These jobs cannot be opened, we rely on the missing version
// to indicate this. // to indicate this.
// See TransportOpenJobAction.validate() // See TransportOpenJobAction.validate()
if (job.getJobVersion() != null) { if (jobVersion != null) {
builder.setJobVersion(Version.CURRENT); builder.setJobVersion(Version.CURRENT);
} }
return builder.build(); return builder.build();

41
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java
View File

@ -55,9 +55,6 @@ import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.MlTasks; import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction;
import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
import org.elasticsearch.xpack.core.ml.action.PutJobAction;
import org.elasticsearch.xpack.core.ml.action.UpdateJobAction;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule;
import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobState;
@ -540,50 +537,16 @@ public class TransportOpenJobAction extends TransportMasterNodeAction<OpenJobAct
); );
// Tell the job tracker to refresh the memory requirement for this job and all other jobs that have persistent tasks // Tell the job tracker to refresh the memory requirement for this job and all other jobs that have persistent tasks
ActionListener<PutJobAction.Response> jobUpdateListener = ActionListener.wrap( ActionListener<Boolean> jobUpdateListener = ActionListener.wrap(
response -> memoryTracker.refreshJobMemoryAndAllOthers(jobParams.getJobId(), memoryRequirementRefreshListener), response -> memoryTracker.refreshJobMemoryAndAllOthers(jobParams.getJobId(), memoryRequirementRefreshListener),
listener::onFailure listener::onFailure
); );
// Increase the model memory limit for 6.1 - 6.3 jobs
ActionListener<Boolean> missingMappingsListener = ActionListener.wrap(
response -> {
Job job = jobParams.getJob();
if (job != null) {
Version jobVersion = job.getJobVersion();
if (jobVersion != null &&
(jobVersion.onOrAfter(Version.V_6_1_0) && jobVersion.before(Version.V_6_3_0))) {
// Increase model memory limit if < 512MB
if (job.getAnalysisLimits() != null && job.getAnalysisLimits().getModelMemoryLimit() != null &&
job.getAnalysisLimits().getModelMemoryLimit() < 512L) {
long updatedModelMemoryLimit = (long) (job.getAnalysisLimits().getModelMemoryLimit() * 1.3);
AnalysisLimits limits = new AnalysisLimits(updatedModelMemoryLimit,
job.getAnalysisLimits().getCategorizationExamplesLimit());
JobUpdate update = new JobUpdate.Builder(job.getId()).setJobVersion(Version.CURRENT)
.setAnalysisLimits(limits).build();
UpdateJobAction.Request updateRequest = UpdateJobAction.Request.internal(job.getId(), update);
executeAsyncWithOrigin(client, ML_ORIGIN, UpdateJobAction.INSTANCE, updateRequest,
jobUpdateListener);
} else {
jobUpdateListener.onResponse(null);
}
}
else {
jobUpdateListener.onResponse(null);
}
} else {
jobUpdateListener.onResponse(null);
}
}, listener::onFailure
);
// Try adding state doc mapping // Try adding state doc mapping
ActionListener<Boolean> resultsPutMappingHandler = ActionListener.wrap( ActionListener<Boolean> resultsPutMappingHandler = ActionListener.wrap(
response -> { response -> {
addDocMappingIfMissing(AnomalyDetectorsIndex.jobStateIndexName(), ElasticsearchMappings::stateMapping, addDocMappingIfMissing(AnomalyDetectorsIndex.jobStateIndexName(), ElasticsearchMappings::stateMapping,
state, missingMappingsListener); state, jobUpdateListener);
}, listener::onFailure }, listener::onFailure
); );