honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Improve logging while removing expired data (elastic/x-pack-elasticsearch#1554) 

relates elastic/x-pack-elasticsearch#1286

Original commit: elastic/x-pack-elasticsearch@4f938fa14b
This commit is contained in:
Dimitris Athanasiou 2017-05-25 14:40:09 +01:00 committed by GitHub
parent 779e6f6dba
commit 9b655ce6f1
6 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteModelSnapshotAction.java
View File

@ -184,8 +184,10 @@ public class DeleteModelSnapshotAction extends Action<DeleteModelSnapshotAction.
deleter.deleteModelSnapshots(Collections.singletonList(deleteCandidate), new ActionListener<BulkResponse>() { deleter.deleteModelSnapshots(Collections.singletonList(deleteCandidate), new ActionListener<BulkResponse>() {
@Override @Override
public void onResponse(BulkResponse bulkResponse) { public void onResponse(BulkResponse bulkResponse) {
auditor.info(request.getJobId(), Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, String msg = Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, deleteCandidate.getSnapshotId(),
deleteCandidate.getDescription())); deleteCandidate.getDescription());
auditor.info(request.getJobId(), msg);
logger.debug("[{}] {}", request.getJobId(), msg);
// We don't care about the bulk response, just that it succeeded // We don't care about the bulk response, just that it succeeded
listener.onResponse(new DeleteModelSnapshotAction.Response(true)); listener.onResponse(new DeleteModelSnapshotAction.Response(true));
} }

2
plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
View File

@ -274,7 +274,7 @@ public class JobManager extends AbstractComponent {
// ------- // -------
CheckedConsumer<Boolean, Exception> apiResponseHandler = jobDeleted -> { CheckedConsumer<Boolean, Exception> apiResponseHandler = jobDeleted -> {
if (jobDeleted) { if (jobDeleted) {
logger.info("Job [" + jobId + "] deleted."); logger.info("Job [" + jobId + "] deleted");
auditor.info(jobId, Messages.getMessage(Messages.JOB_AUDIT_DELETED)); auditor.info(jobId, Messages.getMessage(Messages.JOB_AUDIT_DELETED));
actionListener.onResponse(new DeleteJobAction.Response(true)); actionListener.onResponse(new DeleteJobAction.Response(true));
} else { } else {

2
plugin/src/main/java/org/elasticsearch/xpack/ml/job/messages/Messages.java
View File

@ -51,7 +51,7 @@ public final class Messages {
public static final String JOB_AUDIT_DELETED = "Job deleted"; public static final String JOB_AUDIT_DELETED = "Job deleted";
public static final String JOB_AUDIT_OLD_RESULTS_DELETED = "Deleted results prior to {1}"; public static final String JOB_AUDIT_OLD_RESULTS_DELETED = "Deleted results prior to {1}";
public static final String JOB_AUDIT_REVERTED = "Job model snapshot reverted to ''{0}''"; public static final String JOB_AUDIT_REVERTED = "Job model snapshot reverted to ''{0}''";
public static final String JOB_AUDIT_SNAPSHOT_DELETED = "Job model snapshot ''{0}'' deleted"; public static final String JOB_AUDIT_SNAPSHOT_DELETED = "Model snapshot [{0}] with description ''{1}'' deleted";
public static final String JOB_CONFIG_BYFIELD_INCOMPATIBLE_FUNCTION = "by_field_name cannot be used with function ''{0}''"; public static final String JOB_CONFIG_BYFIELD_INCOMPATIBLE_FUNCTION = "by_field_name cannot be used with function ''{0}''";
public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES = "categorization_filters contain duplicates"; public static final String JOB_CONFIG_CATEGORIZATION_FILTERS_CONTAINS_DUPLICATES = "categorization_filters contain duplicates";

3
plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
View File

@ -63,7 +63,7 @@ public class ExpiredModelSnapshotsRemover extends AbstractExpiredJobDataRemover
onFinish.run(); onFinish.run();
return; return;
} }
LOGGER.info("Removing model snapshots of job [{}] that have a timestamp before [{}]", job.getId(), cutoffEpochMs); LOGGER.debug("Removing model snapshots of job [{}] that have a timestamp before [{}]", job.getId(), cutoffEpochMs);
SearchRequest searchRequest = new SearchRequest(); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId())); searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()));
@ -111,7 +111,6 @@ public class ExpiredModelSnapshotsRemover extends AbstractExpiredJobDataRemover
client.execute(DeleteModelSnapshotAction.INSTANCE, deleteSnapshotRequest, new ActionListener<DeleteModelSnapshotAction.Response>() { client.execute(DeleteModelSnapshotAction.INSTANCE, deleteSnapshotRequest, new ActionListener<DeleteModelSnapshotAction.Response>() {
@Override @Override
public void onResponse(DeleteModelSnapshotAction.Response response) { public void onResponse(DeleteModelSnapshotAction.Response response) {
LOGGER.trace("[{}] Deleted expired snapshot [{}]", modelSnapshot.getJobId(), modelSnapshot.getSnapshotId());
try { try {
deleteModelSnapshots(modelSnapshotIterator, onFinish); deleteModelSnapshots(modelSnapshotIterator, onFinish);
} catch (Exception e) { } catch (Exception e) {

7
plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
View File

@ -56,14 +56,16 @@ public class ExpiredResultsRemover extends AbstractExpiredJobDataRemover {
@Override @Override
protected void removeDataBefore(Job job, long cutoffEpochMs, Runnable onFinish) { protected void removeDataBefore(Job job, long cutoffEpochMs, Runnable onFinish) {
LOGGER.info("Removing results of job [{}] that have a timestamp before [{}]", job.getId(), cutoffEpochMs); LOGGER.debug("Removing results of job [{}] that have a timestamp before [{}]", job.getId(), cutoffEpochMs);
DeleteByQueryRequest request = createDBQRequest(job, cutoffEpochMs); DeleteByQueryRequest request = createDBQRequest(job, cutoffEpochMs);
client.execute(DeleteByQueryAction.INSTANCE, request, new ActionListener<BulkByScrollResponse>() { client.execute(DeleteByQueryAction.INSTANCE, request, new ActionListener<BulkByScrollResponse>() {
@Override @Override
public void onResponse(BulkByScrollResponse bulkByScrollResponse) { public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
try { try {
auditResultsWereDeleted(job.getId(), cutoffEpochMs); if (bulkByScrollResponse.getDeleted() > 0) {
auditResultsWereDeleted(job.getId(), cutoffEpochMs);
}
onFinish.run(); onFinish.run();
} catch (Exception e) { } catch (Exception e) {
onFailure(e); onFailure(e);
@ -99,6 +101,7 @@ public class ExpiredResultsRemover extends AbstractExpiredJobDataRemover {
ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(instant, ZoneOffset.systemDefault()); ZonedDateTime zonedDateTime = ZonedDateTime.ofInstant(instant, ZoneOffset.systemDefault());
String formatted = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(zonedDateTime); String formatted = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(zonedDateTime);
String msg = Messages.getMessage(Messages.JOB_AUDIT_OLD_RESULTS_DELETED, formatted); String msg = Messages.getMessage(Messages.JOB_AUDIT_OLD_RESULTS_DELETED, formatted);
LOGGER.debug("[{}] {}", jobId, msg);
auditor.info(jobId, msg); auditor.info(jobId, msg);
} }
} }

4
plugin/src/test/java/org/elasticsearch/xpack/ml/job/messages/MessagesTests.java
View File

@ -18,7 +18,7 @@ public class MessagesTests extends ESTestCase {
String formattedMessage = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "field-name", "field-value"); String formattedMessage = Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "field-name", "field-value");
assertEquals("Invalid field-name value 'field-value' in datafeed configuration", formattedMessage); assertEquals("Invalid field-name value 'field-value' in datafeed configuration", formattedMessage);
formattedMessage = Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, "foo-job"); formattedMessage = Messages.getMessage(Messages.JOB_AUDIT_SNAPSHOT_DELETED, "snapshot_foo", "snapshot description");
assertEquals("Job model snapshot 'foo-job' deleted", formattedMessage); assertEquals("Model snapshot [snapshot_foo] with description 'snapshot description' deleted", formattedMessage);
} }
} }