From f96a40c61af2a283f931971f93d56fce908ab60f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 20 Mar 2017 16:19:52 +0000 Subject: [PATCH] [ML] Rename score fields (elastic/x-pack-elasticsearch#769) * Rename normalized_probability to record_score * Rename anomaly_score on influencers to influencer_score * Remove record level anomaly_score * Remove bucket level max_normalized_probability Relates elastic/x-pack-elasticsearch#678 Original commit: elastic/x-pack-elasticsearch@c2c05e73d827f43eb316e553efcc14e69237cf80 --- .../xpack/ml/action/GetBucketsAction.java | 29 +-- .../xpack/ml/action/GetInfluencersAction.java | 2 +- .../xpack/ml/action/GetRecordsAction.java | 43 ++--- .../job/persistence/BucketsQueryBuilder.java | 19 +- .../persistence/ElasticsearchMappings.java | 20 +- .../persistence/InfluencersQueryBuilder.java | 4 +- .../xpack/ml/job/persistence/JobProvider.java | 26 +-- .../job/persistence/RecordsQueryBuilder.java | 24 +-- .../normalizer/BucketNormalizable.java | 12 +- .../normalizer/InfluencerNormalizable.java | 4 +- .../PartitionScoreNormalizable.java | 4 +- .../normalizer/RecordNormalizable.java | 6 +- .../xpack/ml/job/results/AnomalyRecord.java | 63 +++---- .../xpack/ml/job/results/Bucket.java | 23 +-- .../xpack/ml/job/results/Influencer.java | 48 ++--- .../xpack/ml/job/results/PartitionScore.java | 42 ++--- .../results/PerPartitionMaxProbabilities.java | 33 ++-- .../ml/job/results/ReservedFieldNames.java | 12 +- .../ml/rest/results/RestGetBucketsAction.java | 5 - .../results/RestGetInfluencersAction.java | 2 +- .../ml/rest/results/RestGetRecordsAction.java | 8 +- .../action/GetBucketActionRequestTests.java | 3 - .../action/GetBucketActionResponseTests.java | 4 - .../GetInfluencersActionResponseTests.java | 4 +- .../action/GetRecordsActionRequestTests.java | 5 +- .../xpack/ml/integration/MlJobIT.java | 2 +- .../persistence/BucketsQueryBuilderTests.java | 9 +- .../InfluencersQueryBuilderTests.java | 4 +- .../ml/job/persistence/JobProviderTests.java | 175 +++++------------- .../persistence/JobResultsPersisterTests.java | 21 +-- .../output/AutodetectResultsParserTests.java | 84 ++++----- .../normalizer/BucketNormalizableTests.java | 9 +- .../InfluencerNormalizableTests.java | 6 +- .../normalizer/ScoresUpdaterTests.java | 8 - .../ml/job/results/AnomalyRecordTests.java | 5 +- .../xpack/ml/job/results/BucketTests.java | 19 +- .../xpack/ml/job/results/InfluencerTests.java | 12 +- .../PerPartitionMaxProbabilitiesTests.java | 8 +- .../api/xpack.ml.get_buckets.json | 4 - .../api/xpack.ml.get_influencers.json | 4 +- .../api/xpack.ml.get_records.json | 5 +- .../test/ml/jobs_get_result_influencers.yaml | 8 +- .../test/ml/jobs_get_result_records.yaml | 4 +- .../test/ml/revert_model_snapshot.yaml | 4 +- 44 files changed, 288 insertions(+), 548 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java index 7a20dd81bd4..2105602a153 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetBucketsAction.java @@ -69,7 +69,6 @@ public class GetBucketsAction extends Action PARSER = new ObjectParser<>(NAME, Request::new); @@ -86,7 +85,6 @@ public class GetBucketsAction extends Action PARSER = new ObjectParser<>(NAME, Request::new); @@ -83,8 +82,7 @@ public class GetRecordsAction extends ActionIncludeInterim- Include interim results. Default = false *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= * this value. Default = 0.0
    • - *
  • normalizedProbabilityThreshold- Return only buckets with a - * maxNormalizedProbability >= this value. Default = 0.0
    • *
  • start- The start bucket time. A bucket with this timestamp will be * included in the results. If 0 all buckets up to endEpochMs are * returned. Default = -1
    • @@ -66,13 +64,6 @@ public final class BucketsQueryBuilder { return this; } - public BucketsQueryBuilder normalizedProbabilityThreshold(Double normalizedProbability) { - if (normalizedProbability != null) { - bucketsQuery.normalizedProbability = normalizedProbability; - } - return this; - } - /** * @param partitionValue Not set if null or empty */ @@ -133,8 +124,7 @@ public final class BucketsQueryBuilder { private int size = DEFAULT_SIZE; private boolean expand = false; private boolean includeInterim = false; - private double anomalyScoreFilter = 0.0d; - private double normalizedProbability = 0.0d; + private double anomalyScoreFilter = 0.0; private String start; private String end; private String timestamp; @@ -162,10 +152,6 @@ public final class BucketsQueryBuilder { return anomalyScoreFilter; } - public double getNormalizedProbability() { - return normalizedProbability; - } - public String getStart() { return start; } @@ -195,7 +181,7 @@ public final class BucketsQueryBuilder { @Override public int hashCode() { - return Objects.hash(from, size, expand, includeInterim, anomalyScoreFilter, normalizedProbability, start, end, + return Objects.hash(from, size, expand, includeInterim, anomalyScoreFilter, start, end, timestamp, partitionValue, sortField, sortDescending); } @@ -221,7 +207,6 @@ public final class BucketsQueryBuilder { Objects.equals(end, other.end) && Objects.equals(timestamp, other.timestamp) && Objects.equals(anomalyScoreFilter, other.anomalyScoreFilter) && - Objects.equals(normalizedProbability, other.normalizedProbability) && Objects.equals(partitionValue, other.partitionValue) && Objects.equals(sortField, other.sortField) && this.sortDescending == other.sortDescending; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java index 8e1d6d1c50c..3a0f6370d61 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java @@ -145,9 +145,6 @@ public class ElasticsearchMappings { .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() - .startObject(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() .startObject(Bucket.IS_INTERIM.getPreferredName()) .field(TYPE, BOOLEAN) .endObject() @@ -175,9 +172,6 @@ public class ElasticsearchMappings { .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() - .startObject(AnomalyRecord.ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) .field(TYPE, DOUBLE) .endObject() @@ -230,7 +224,7 @@ public class ElasticsearchMappings { .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) .field(TYPE, KEYWORD) .endObject() - .startObject(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName()) + .startObject(PerPartitionMaxProbabilities.MAX_RECORD_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() .endObject() @@ -343,10 +337,10 @@ public class ElasticsearchMappings { .field(TYPE, KEYWORD) .field(COPY_TO, ALL_FIELD_VALUES) .endObject() - .startObject(AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName()) + .startObject(AnomalyRecord.RECORD_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() - .startObject(AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY.getPreferredName()) + .startObject(AnomalyRecord.INITIAL_RECORD_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() .startObject(AnomalyRecord.CAUSES.getPreferredName()) @@ -415,7 +409,13 @@ public class ElasticsearchMappings { } private static XContentBuilder addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException { - builder.startObject(Influencer.INFLUENCER_FIELD_NAME.getPreferredName()) + builder.startObject(Influencer.INFLUENCER_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Influencer.INITIAL_INFLUENCER_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Influencer.INFLUENCER_FIELD_NAME.getPreferredName()) .field(TYPE, KEYWORD) .endObject() .startObject(Influencer.INFLUENCER_FIELD_VALUE.getPreferredName()) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java index 5e7f68c04ae..5116ce18b3f 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilder.java @@ -25,8 +25,6 @@ import java.util.Objects; *
  • end- The end influencer timestamp. Influencers up to but NOT including this * timestamp are returned. If 0 all influencers from start are * returned. Default = -1
    • - *
  • partitionValue Set the bucket's max normalized probability to this - * partition field value's max normalized probability. Default = null
    • * */ public final class InfluencersQueryBuilder { @@ -96,7 +94,7 @@ public final class InfluencersQueryBuilder { private double anomalyScoreFilter = 0.0d; private String start; private String end; - private String sortField = Influencer.ANOMALY_SCORE.getPreferredName(); + private String sortField = Influencer.INFLUENCER_SCORE.getPreferredName(); private boolean sortDescending = false; public int getFrom() { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index 9bd42f6a1d8..dda228af06b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -99,7 +99,7 @@ public class JobProvider { private static final Logger LOGGER = Loggers.getLogger(JobProvider.class); private static final List SECONDARY_SORT = Arrays.asList( - AnomalyRecord.ANOMALY_SCORE.getPreferredName(), + AnomalyRecord.RECORD_SCORE.getPreferredName(), AnomalyRecord.OVER_FIELD_VALUE.getPreferredName(), AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), AnomalyRecord.BY_FIELD_VALUE.getPreferredName(), @@ -330,7 +330,6 @@ public class JobProvider { } else { rfb.timeRange(Result.TIMESTAMP.getPreferredName(), query.getStart(), query.getEnd()) .score(Bucket.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreFilter()) - .score(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), query.getNormalizedProbability()) .interim(Bucket.IS_INTERIM.getPreferredName(), query.isIncludeInterim()); } @@ -399,9 +398,6 @@ public class JobProvider { errorHandler.accept(item2.getFailure()); return; } - List partitionProbs = - handlePartitionMaxNormailizedProbabilitiesResponse(item2.getResponse()); - mergePartitionScoresIntoBucket(partitionProbs, buckets.results(), query.getPartitionValue()); if (query.isExpand()) { Iterator bucketsToExpand = buckets.results().stream() @@ -433,23 +429,6 @@ public class JobProvider { } } - void mergePartitionScoresIntoBucket(List partitionProbs, List buckets, String partitionValue) { - Iterator itr = partitionProbs.iterator(); - PerPartitionMaxProbabilities partitionProb = itr.hasNext() ? itr.next() : null; - for (Bucket b : buckets) { - if (partitionProb == null) { - b.setMaxNormalizedProbability(0.0); - } else { - if (partitionProb.getTimestamp().equals(b.getTimestamp())) { - b.setMaxNormalizedProbability(partitionProb.getMaxProbabilityForPartition(partitionValue)); - partitionProb = itr.hasNext() ? itr.next() : null; - } else { - b.setMaxNormalizedProbability(0.0); - } - } - } - } - private SearchRequest createPartitionMaxNormailizedProbabilitiesRequest(String jobId, Object epochStart, Object epochEnd, String partitionFieldValue) { QueryBuilder timeRangeQuery = new ResultsFilterBuilder() @@ -643,8 +622,7 @@ public class JobProvider { Consumer errorHandler, Client client) { QueryBuilder fb = new ResultsFilterBuilder() .timeRange(Result.TIMESTAMP.getPreferredName(), query.getStart(), query.getEnd()) - .score(AnomalyRecord.ANOMALY_SCORE.getPreferredName(), query.getAnomalyScoreThreshold()) - .score(AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName(), query.getNormalizedProbabilityThreshold()) + .score(AnomalyRecord.RECORD_SCORE.getPreferredName(), query.getRecordScoreThreshold()) .interim(AnomalyRecord.IS_INTERIM.getPreferredName(), query.isIncludeInterim()) .term(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), query.getPartitionFieldValue()).build(); FieldSortBuilder sb = null; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java index fe01627066d..824adeb4061 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/RecordsQueryBuilder.java @@ -17,10 +17,8 @@ package org.elasticsearch.xpack.ml.job.persistence; *
  • SortField- The field to sort results by if null no sort is * applied. Default = null
    • *
  • SortDescending- Sort in descending order. Default = true
    • - *
  • anomalyScoreThreshold- Return only buckets with an anomalyScore >= + *
  • recordScoreThreshold- Return only records with a record_score >= * this value. Default = 0.0
    • - *
  • normalizedProbabilityThreshold. Return only buckets with a - * maxNormalizedProbability >= this value. Default = 0.0
    • *
  • start- The start bucket time. A bucket with this timestamp will be * included in the results. If 0 all buckets up to endEpochMs are * returned. Default = -1
    • @@ -70,13 +68,8 @@ public final class RecordsQueryBuilder { return this; } - public RecordsQueryBuilder anomalyScoreThreshold(double anomalyScoreFilter) { - recordsQuery.anomalyScoreFilter = anomalyScoreFilter; - return this; - } - - public RecordsQueryBuilder normalizedProbability(double normalizedProbability) { - recordsQuery.normalizedProbability = normalizedProbability; + public RecordsQueryBuilder recordScore(double recordScore) { + recordsQuery.recordScore = recordScore; return this; } @@ -100,8 +93,7 @@ public final class RecordsQueryBuilder { private boolean includeInterim = false; private String sortField; private boolean sortDescending = true; - private double anomalyScoreFilter = 0.0d; - private double normalizedProbability = 0.0d; + private double recordScore = 0.0; private String partitionFieldValue; private String start; private String end; @@ -123,12 +115,8 @@ public final class RecordsQueryBuilder { return sortDescending; } - public double getAnomalyScoreThreshold() { - return anomalyScoreFilter; - } - - public double getNormalizedProbabilityThreshold() { - return normalizedProbability; + public double getRecordScoreThreshold() { + return recordScore; } public String getPartitionFieldValue() { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java index 0ad9d4da99a..d78cc89b75c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizable.java @@ -140,22 +140,18 @@ public class BucketNormalizable extends Normalizable { @Override public boolean setMaxChildrenScore(ChildType childrenType, double maxScore) { - double oldScore = 0.0; switch (childrenType) { case BUCKET_INFLUENCER: - oldScore = bucket.getAnomalyScore(); + double oldScore = bucket.getAnomalyScore(); bucket.setAnomalyScore(maxScore); - break; + return maxScore != oldScore; case RECORD: - oldScore = bucket.getMaxNormalizedProbability(); - bucket.setMaxNormalizedProbability(maxScore); - break; case PARTITION_SCORE: - break; + return false; default: throw new IllegalArgumentException("Invalid type: " + childrenType); } - return maxScore != oldScore; + } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java index d54666d686a..baaf1fb5853 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizable.java @@ -61,12 +61,12 @@ class InfluencerNormalizable extends AbstractLeafNormalizable { @Override public double getNormalizedScore() { - return influencer.getAnomalyScore(); + return influencer.getInfluencerScore(); } @Override public void setNormalizedScore(double normalizedScore) { - influencer.setAnomalyScore(normalizedScore); + influencer.setInfluencerScore(normalizedScore); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java index 78c3455cebb..0a7e8662d2b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/PartitionScoreNormalizable.java @@ -62,12 +62,12 @@ public class PartitionScoreNormalizable extends AbstractLeafNormalizable { @Override public double getNormalizedScore() { - return score.getAnomalyScore(); + return score.getRecordScore(); } @Override public void setNormalizedScore(double normalizedScore) { - score.setAnomalyScore(normalizedScore); + score.setRecordScore(normalizedScore); } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java index eedf640d954..e43f75671f1 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/normalizer/RecordNormalizable.java @@ -63,17 +63,17 @@ class RecordNormalizable extends AbstractLeafNormalizable { @Override public double getNormalizedScore() { - return record.getNormalizedProbability(); + return record.getRecordScore(); } @Override public void setNormalizedScore(double normalizedScore) { - record.setNormalizedProbability(normalizedScore); + record.setRecordScore(normalizedScore); } @Override public void setParentScore(double parentScore) { - record.setAnomalyScore(parentScore); + // nothing to do } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java index c57c94f6561..018d281c456 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecord.java @@ -76,9 +76,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { /** * Normalization */ - public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); - public static final ParseField NORMALIZED_PROBABILITY = new ParseField("normalized_probability"); - public static final ParseField INITIAL_NORMALIZED_PROBABILITY = new ParseField("initial_normalized_probability"); + public static final ParseField RECORD_SCORE = new ParseField("record_score"); + public static final ParseField INITIAL_RECORD_SCORE = new ParseField("initial_record_score"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, @@ -99,9 +98,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { PARSER.declareInt(ConstructingObjectParser.constructorArg(), SEQUENCE_NUM); PARSER.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE); PARSER.declareDouble(AnomalyRecord::setProbability, PROBABILITY); - PARSER.declareDouble(AnomalyRecord::setAnomalyScore, ANOMALY_SCORE); - PARSER.declareDouble(AnomalyRecord::setNormalizedProbability, NORMALIZED_PROBABILITY); - PARSER.declareDouble(AnomalyRecord::setInitialNormalizedProbability, INITIAL_NORMALIZED_PROBABILITY); + PARSER.declareDouble(AnomalyRecord::setRecordScore, RECORD_SCORE); + PARSER.declareDouble(AnomalyRecord::setInitialRecordScore, INITIAL_RECORD_SCORE); PARSER.declareInt(AnomalyRecord::setDetectorIndex, DETECTOR_INDEX); PARSER.declareBoolean(AnomalyRecord::setInterim, IS_INTERIM); PARSER.declareString(AnomalyRecord::setByFieldName, BY_FIELD_NAME); @@ -141,10 +139,9 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { private String overFieldValue; private List causes; - private double anomalyScore; - private double normalizedProbability; + private double recordScore; - private double initialNormalizedProbability; + private double initialRecordScore; private final Date timestamp; private final long bucketSpan; @@ -184,9 +181,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { if (in.readBoolean()) { causes = in.readList(AnomalyCause::new); } - anomalyScore = in.readDouble(); - normalizedProbability = in.readDouble(); - initialNormalizedProbability = in.readDouble(); + recordScore = in.readDouble(); + initialRecordScore = in.readDouble(); timestamp = new Date(in.readLong()); bucketSpan = in.readLong(); if (in.readBoolean()) { @@ -226,9 +222,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { if (hasCauses) { out.writeList(causes); } - out.writeDouble(anomalyScore); - out.writeDouble(normalizedProbability); - out.writeDouble(initialNormalizedProbability); + out.writeDouble(recordScore); + out.writeDouble(initialRecordScore); out.writeLong(timestamp.getTime()); out.writeLong(bucketSpan); boolean hasInfluencers = influences != null; @@ -244,9 +239,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { builder.field(Job.ID.getPreferredName(), jobId); builder.field(Result.RESULT_TYPE.getPreferredName(), RESULT_TYPE_VALUE); builder.field(PROBABILITY.getPreferredName(), probability); - builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); - builder.field(NORMALIZED_PROBABILITY.getPreferredName(), normalizedProbability); - builder.field(INITIAL_NORMALIZED_PROBABILITY.getPreferredName(), initialNormalizedProbability); + builder.field(RECORD_SCORE.getPreferredName(), recordScore); + builder.field(INITIAL_RECORD_SCORE.getPreferredName(), initialRecordScore); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(DETECTOR_INDEX.getPreferredName(), detectorIndex); builder.field(SEQUENCE_NUM.getPreferredName(), sequenceNum); @@ -350,28 +344,20 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { this.detectorIndex = detectorIndex; } - public double getAnomalyScore() { - return anomalyScore; + public double getRecordScore() { + return recordScore; } - public void setAnomalyScore(double anomalyScore) { - this.anomalyScore = anomalyScore; + public void setRecordScore(double recordScore) { + this.recordScore = recordScore; } - public double getNormalizedProbability() { - return normalizedProbability; + public double getInitialRecordScore() { + return initialRecordScore; } - public void setNormalizedProbability(double normalizedProbability) { - this.normalizedProbability = normalizedProbability; - } - - public double getInitialNormalizedProbability() { - return initialNormalizedProbability; - } - - public void setInitialNormalizedProbability(double initialNormalizedProbability) { - this.initialNormalizedProbability = initialNormalizedProbability; + public void setInitialRecordScore(double initialRecordScore) { + this.initialRecordScore = initialRecordScore; } public Date getTimestamp() { @@ -523,8 +509,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { @Override public int hashCode() { - return Objects.hash(jobId, detectorIndex, sequenceNum, bucketSpan, probability, anomalyScore, - normalizedProbability, initialNormalizedProbability, typical, actual, + return Objects.hash(jobId, detectorIndex, sequenceNum, bucketSpan, probability, + recordScore, initialRecordScore, typical, actual, function, functionDescription, fieldName, byFieldName, byFieldValue, correlatedByFieldValue, partitionFieldName, partitionFieldValue, overFieldName, overFieldValue, timestamp, isInterim, causes, influences, jobId); @@ -548,9 +534,8 @@ public class AnomalyRecord extends ToXContentToBytes implements Writeable { && this.sequenceNum == that.sequenceNum && this.bucketSpan == that.bucketSpan && this.probability == that.probability - && this.anomalyScore == that.anomalyScore - && this.normalizedProbability == that.normalizedProbability - && this.initialNormalizedProbability == that.initialNormalizedProbability + && this.recordScore == that.recordScore + && this.initialRecordScore == that.initialRecordScore && Objects.deepEquals(this.typical, that.typical) && Objects.deepEquals(this.actual, that.actual) && Objects.equals(this.function, that.function) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java index 0f3af3f5cfe..bb39f56b56d 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Bucket.java @@ -38,7 +38,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score"); - public static final ParseField MAX_NORMALIZED_PROBABILITY = new ParseField("max_normalized_probability"); public static final ParseField IS_INTERIM = new ParseField("is_interim"); public static final ParseField RECORD_COUNT = new ParseField("record_count"); public static final ParseField EVENT_COUNT = new ParseField("event_count"); @@ -74,7 +73,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); PARSER.declareDouble(Bucket::setAnomalyScore, ANOMALY_SCORE); PARSER.declareDouble(Bucket::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); - PARSER.declareDouble(Bucket::setMaxNormalizedProbability, MAX_NORMALIZED_PROBABILITY); PARSER.declareBoolean(Bucket::setInterim, IS_INTERIM); PARSER.declareInt(Bucket::setRecordCount, RECORD_COUNT); PARSER.declareLong(Bucket::setEventCount, EVENT_COUNT); @@ -90,7 +88,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { private final long bucketSpan; private double anomalyScore; private double initialAnomalyScore; - private double maxNormalizedProbability; private int recordCount; private List records = new ArrayList<>(); private long eventCount; @@ -112,7 +109,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { this.bucketSpan = other.bucketSpan; this.anomalyScore = other.anomalyScore; this.initialAnomalyScore = other.initialAnomalyScore; - this.maxNormalizedProbability = other.maxNormalizedProbability; this.recordCount = other.recordCount; this.records = new ArrayList<>(other.records); this.eventCount = other.eventCount; @@ -130,7 +126,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { anomalyScore = in.readDouble(); bucketSpan = in.readLong(); initialAnomalyScore = in.readDouble(); - maxNormalizedProbability = in.readDouble(); recordCount = in.readInt(); records = in.readList(AnomalyRecord::new); eventCount = in.readLong(); @@ -148,7 +143,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { out.writeDouble(anomalyScore); out.writeLong(bucketSpan); out.writeDouble(initialAnomalyScore); - out.writeDouble(maxNormalizedProbability); out.writeInt(recordCount); out.writeList(records); out.writeLong(eventCount); @@ -167,7 +161,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); - builder.field(MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability); builder.field(RECORD_COUNT.getPreferredName(), recordCount); if (!records.isEmpty()) { builder.field(RECORDS.getPreferredName(), records); @@ -225,14 +218,6 @@ public class Bucket extends ToXContentToBytes implements Writeable { this.initialAnomalyScore = initialAnomalyScore; } - public double getMaxNormalizedProbability() { - return maxNormalizedProbability; - } - - public void setMaxNormalizedProbability(double maxNormalizedProbability) { - this.maxNormalizedProbability = maxNormalizedProbability; - } - public int getRecordCount() { return recordCount; } @@ -316,19 +301,19 @@ public class Bucket extends ToXContentToBytes implements Writeable { Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) .findFirst(); - return first.isPresent() ? first.get().getInitialAnomalyScore() : 0.0; + return first.isPresent() ? first.get().getInitialRecordScore() : 0.0; } public double partitionAnomalyScore(String partitionValue) { Optional first = partitionScores.stream().filter(s -> partitionValue.equals(s.getPartitionFieldValue())) .findFirst(); - return first.isPresent() ? first.get().getAnomalyScore() : 0.0; + return first.isPresent() ? first.get().getRecordScore() : 0.0; } @Override public int hashCode() { - return Objects.hash(jobId, timestamp, eventCount, initialAnomalyScore, anomalyScore, maxNormalizedProbability, recordCount, records, + return Objects.hash(jobId, timestamp, eventCount, initialAnomalyScore, anomalyScore, recordCount, records, isInterim, bucketSpan, bucketInfluencers); } @@ -349,8 +334,8 @@ public class Bucket extends ToXContentToBytes implements Writeable { return Objects.equals(this.jobId, that.jobId) && Objects.equals(this.timestamp, that.timestamp) && (this.eventCount == that.eventCount) && (this.bucketSpan == that.bucketSpan) + && (this.recordCount == that.recordCount) && (this.anomalyScore == that.anomalyScore) && (this.initialAnomalyScore == that.initialAnomalyScore) - && (this.maxNormalizedProbability == that.maxNormalizedProbability) && (this.recordCount == that.recordCount) && Objects.equals(this.records, that.records) && Objects.equals(this.isInterim, that.isInterim) && Objects.equals(this.bucketInfluencers, that.bucketInfluencers); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java index 8bbbaa161d3..473063bf0f4 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/Influencer.java @@ -37,8 +37,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name"); public static final ParseField INFLUENCER_FIELD_VALUE = new ParseField("influencer_field_value"); - public static final ParseField INITIAL_ANOMALY_SCORE = new ParseField("initial_anomaly_score"); - public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); + public static final ParseField INITIAL_INFLUENCER_SCORE = new ParseField("initial_influencer_score"); + public static final ParseField INFLUENCER_SCORE = new ParseField("influencer_score"); // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("influencers"); @@ -64,8 +64,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { PARSER.declareInt(ConstructingObjectParser.constructorArg(), SEQUENCE_NUM); PARSER.declareString((influencer, s) -> {}, Result.RESULT_TYPE); PARSER.declareDouble(Influencer::setProbability, PROBABILITY); - PARSER.declareDouble(Influencer::setAnomalyScore, ANOMALY_SCORE); - PARSER.declareDouble(Influencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + PARSER.declareDouble(Influencer::setInfluencerScore, INFLUENCER_SCORE); + PARSER.declareDouble(Influencer::setInitialInfluencerScore, INITIAL_INFLUENCER_SCORE); PARSER.declareBoolean(Influencer::setInterim, Bucket.IS_INTERIM); } @@ -76,8 +76,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { private String influenceField; private String influenceValue; private double probability; - private double initialAnomalyScore; - private double anomalyScore; + private double initialInfluencerScore; + private double influencerScore; private boolean isInterim; public Influencer(String jobId, String fieldName, String fieldValue, Date timestamp, long bucketSpan, int sequenceNum) { @@ -95,8 +95,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { influenceField = in.readString(); influenceValue = in.readString(); probability = in.readDouble(); - initialAnomalyScore = in.readDouble(); - anomalyScore = in.readDouble(); + initialInfluencerScore = in.readDouble(); + influencerScore = in.readDouble(); isInterim = in.readBoolean(); bucketSpan = in.readLong(); sequenceNum = in.readInt(); @@ -109,8 +109,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { out.writeString(influenceField); out.writeString(influenceValue); out.writeDouble(probability); - out.writeDouble(initialAnomalyScore); - out.writeDouble(anomalyScore); + out.writeDouble(initialInfluencerScore); + out.writeDouble(influencerScore); out.writeBoolean(isInterim); out.writeLong(bucketSpan); out.writeInt(sequenceNum); @@ -126,8 +126,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { if (ReservedFieldNames.isValidFieldName(influenceField)) { builder.field(influenceField, influenceValue); } - builder.field(ANOMALY_SCORE.getPreferredName(), anomalyScore); - builder.field(INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); + builder.field(INFLUENCER_SCORE.getPreferredName(), influencerScore); + builder.field(INITIAL_INFLUENCER_SCORE.getPreferredName(), initialInfluencerScore); builder.field(PROBABILITY.getPreferredName(), probability); builder.field(SEQUENCE_NUM.getPreferredName(), sequenceNum); builder.field(BUCKET_SPAN.getPreferredName(), bucketSpan); @@ -165,20 +165,20 @@ public class Influencer extends ToXContentToBytes implements Writeable { return influenceValue; } - public double getInitialAnomalyScore() { - return initialAnomalyScore; + public double getInitialInfluencerScore() { + return initialInfluencerScore; } - public void setInitialAnomalyScore(double influenceScore) { - initialAnomalyScore = influenceScore; + public void setInitialInfluencerScore(double score) { + initialInfluencerScore = score; } - public double getAnomalyScore() { - return anomalyScore; + public double getInfluencerScore() { + return influencerScore; } - public void setAnomalyScore(double score) { - anomalyScore = score; + public void setInfluencerScore(double score) { + influencerScore = score; } public boolean isInterim() { @@ -191,8 +191,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { @Override public int hashCode() { - return Objects.hash(jobId, timestamp, influenceField, influenceValue, initialAnomalyScore, anomalyScore, probability, isInterim, - bucketSpan, sequenceNum); + return Objects.hash(jobId, timestamp, influenceField, influenceValue, initialInfluencerScore, + influencerScore, probability, isInterim, bucketSpan, sequenceNum); } @Override @@ -213,8 +213,8 @@ public class Influencer extends ToXContentToBytes implements Writeable { return Objects.equals(jobId, other.jobId) && Objects.equals(timestamp, other.timestamp) && Objects.equals(influenceField, other.influenceField) && Objects.equals(influenceValue, other.influenceValue) - && Double.compare(initialAnomalyScore, other.initialAnomalyScore) == 0 - && Double.compare(anomalyScore, other.anomalyScore) == 0 && Double.compare(probability, other.probability) == 0 + && Double.compare(initialInfluencerScore, other.initialInfluencerScore) == 0 + && Double.compare(influencerScore, other.influencerScore) == 0 && Double.compare(probability, other.probability) == 0 && (isInterim == other.isInterim) && (bucketSpan == other.bucketSpan) && (sequenceNum == other.sequenceNum); } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java index 097ed71aa24..912e0f7a446 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PartitionScore.java @@ -21,8 +21,8 @@ public class PartitionScore extends ToXContentToBytes implements Writeable { private final String partitionFieldValue; private final String partitionFieldName; - private final double initialAnomalyScore; - private double anomalyScore; + private final double initialRecordScore; + private double recordScore; private double probability; public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( @@ -32,24 +32,24 @@ public class PartitionScore extends ToXContentToBytes implements Writeable { static { PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_NAME); PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); - PARSER.declareDouble(ConstructingObjectParser.constructorArg(), Bucket.INITIAL_ANOMALY_SCORE); - PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.ANOMALY_SCORE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.INITIAL_RECORD_SCORE); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.RECORD_SCORE); PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.PROBABILITY); } - public PartitionScore(String fieldName, String fieldValue, double initialAnomalyScore, double anomalyScore, double probability) { + public PartitionScore(String fieldName, String fieldValue, double initialRecordScore, double recordScore, double probability) { partitionFieldName = fieldName; partitionFieldValue = fieldValue; - this.initialAnomalyScore = initialAnomalyScore; - this.anomalyScore = anomalyScore; + this.initialRecordScore = initialRecordScore; + this.recordScore = recordScore; this.probability = probability; } public PartitionScore(StreamInput in) throws IOException { partitionFieldName = in.readString(); partitionFieldValue = in.readString(); - initialAnomalyScore = in.readDouble(); - anomalyScore = in.readDouble(); + initialRecordScore = in.readDouble(); + recordScore = in.readDouble(); probability = in.readDouble(); } @@ -57,8 +57,8 @@ public class PartitionScore extends ToXContentToBytes implements Writeable { public void writeTo(StreamOutput out) throws IOException { out.writeString(partitionFieldName); out.writeString(partitionFieldValue); - out.writeDouble(initialAnomalyScore); - out.writeDouble(anomalyScore); + out.writeDouble(initialRecordScore); + out.writeDouble(recordScore); out.writeDouble(probability); } @@ -67,23 +67,23 @@ public class PartitionScore extends ToXContentToBytes implements Writeable { builder.startObject(); builder.field(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName(), partitionFieldName); builder.field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionFieldValue); - builder.field(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName(), initialAnomalyScore); - builder.field(AnomalyRecord.ANOMALY_SCORE.getPreferredName(), anomalyScore); + builder.field(AnomalyRecord.INITIAL_RECORD_SCORE.getPreferredName(), initialRecordScore); + builder.field(AnomalyRecord.RECORD_SCORE.getPreferredName(), recordScore); builder.field(AnomalyRecord.PROBABILITY.getPreferredName(), probability); builder.endObject(); return builder; } - public double getInitialAnomalyScore() { - return initialAnomalyScore; + public double getInitialRecordScore() { + return initialRecordScore; } - public double getAnomalyScore() { - return anomalyScore; + public double getRecordScore() { + return recordScore; } - public void setAnomalyScore(double anomalyScore) { - this.anomalyScore = anomalyScore; + public void setRecordScore(double recordScore) { + this.recordScore = recordScore; } public String getPartitionFieldName() { @@ -104,7 +104,7 @@ public class PartitionScore extends ToXContentToBytes implements Writeable { @Override public int hashCode() { - return Objects.hash(partitionFieldName, partitionFieldValue, probability, initialAnomalyScore, anomalyScore); + return Objects.hash(partitionFieldName, partitionFieldValue, probability, initialRecordScore, recordScore); } @Override @@ -122,6 +122,6 @@ public class PartitionScore extends ToXContentToBytes implements Writeable { // id is excluded from the test as it is generated by the datastore return Objects.equals(this.partitionFieldValue, that.partitionFieldValue) && Objects.equals(this.partitionFieldName, that.partitionFieldName) && (this.probability == that.probability) - && (this.initialAnomalyScore == that.initialAnomalyScore) && (this.anomalyScore == that.anomalyScore); + && (this.initialRecordScore == that.initialRecordScore) && (this.recordScore == that.recordScore); } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java index 4e3a1b3312d..f2d657b4f99 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilities.java @@ -43,6 +43,7 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W * Field Names */ public static final ParseField PER_PARTITION_MAX_PROBABILITIES = new ParseField("per_partition_max_probabilities"); + public static final ParseField MAX_RECORD_SCORE = new ParseField("max_record_score"); @SuppressWarnings("unchecked") public static final ConstructingObjectParser PARSER = @@ -85,7 +86,7 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W this.jobId = records.get(0).getJobId(); this.timestamp = records.get(0).getTimestamp(); this.bucketSpan = records.get(0).getBucketSpan(); - this.perPartitionMaxProbabilities = calcMaxNormalizedProbabilityPerPartition(records); + this.perPartitionMaxProbabilities = calcMaxRecordScorePerPartition(records); } public PerPartitionMaxProbabilities(StreamInput in) throws IOException { @@ -123,7 +124,7 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W Optional first = perPartitionMaxProbabilities.stream().filter(pp -> partitionValue.equals(pp.getPartitionValue())).findFirst(); - return first.isPresent() ? first.get().getMaxNormalizedProbability() : 0.0; + return first.isPresent() ? first.get().getMaxRecordScore() : 0.0; } /** @@ -150,10 +151,10 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W } } - private List calcMaxNormalizedProbabilityPerPartition(List anomalyRecords) { + private List calcMaxRecordScorePerPartition(List anomalyRecords) { Map maxValueByPartition = anomalyRecords.stream().collect( Collectors.groupingBy(AnomalyRecord::getPartitionFieldValue, - Collector.of(DoubleMaxBox::new, (m, ar) -> m.accept(ar.getNormalizedProbability()), + Collector.of(DoubleMaxBox::new, (m, ar) -> m.accept(ar.getRecordScore()), DoubleMaxBox::combine, DoubleMaxBox::value))); List pProbs = new ArrayList<>(); @@ -200,7 +201,7 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W } /** - * Class for partitionValue, maxNormalizedProb pairs + * Class for partitionValue, maxRecordScore pairs */ public static class PartitionProbability extends ToXContentToBytes implements Writeable { @@ -210,48 +211,48 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W static { PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); - PARSER.declareDouble(ConstructingObjectParser.constructorArg(), Bucket.MAX_NORMALIZED_PROBABILITY); + PARSER.declareDouble(ConstructingObjectParser.constructorArg(), MAX_RECORD_SCORE); } private final String partitionValue; - private final double maxNormalizedProbability; + private final double maxRecordScore; - PartitionProbability(String partitionName, double maxNormalizedProbability) { + PartitionProbability(String partitionName, double maxRecordScore) { this.partitionValue = partitionName; - this.maxNormalizedProbability = maxNormalizedProbability; + this.maxRecordScore = maxRecordScore; } public PartitionProbability(StreamInput in) throws IOException { partitionValue = in.readString(); - maxNormalizedProbability = in.readDouble(); + maxRecordScore = in.readDouble(); } public String getPartitionValue() { return partitionValue; } - public double getMaxNormalizedProbability() { - return maxNormalizedProbability; + public double getMaxRecordScore() { + return maxRecordScore; } @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(partitionValue); - out.writeDouble(maxNormalizedProbability); + out.writeDouble(maxRecordScore); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject() .field(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName(), partitionValue) - .field(Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), maxNormalizedProbability) + .field(MAX_RECORD_SCORE.getPreferredName(), maxRecordScore) .endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(partitionValue, maxNormalizedProbability); + return Objects.hash(partitionValue, maxRecordScore); } @Override @@ -267,7 +268,7 @@ public class PerPartitionMaxProbabilities extends ToXContentToBytes implements W PartitionProbability that = (PartitionProbability) other; return Objects.equals(this.partitionValue, that.partitionValue) - && this.maxNormalizedProbability == that.maxNormalizedProbability; + && this.maxRecordScore == that.maxRecordScore; } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java index d9b6886482f..04dc2c29fda 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java @@ -10,7 +10,6 @@ import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; import java.util.Arrays; import java.util.HashSet; @@ -72,16 +71,14 @@ public final class ReservedFieldNames { AnomalyRecord.OVER_FIELD_NAME.getPreferredName(), AnomalyRecord.OVER_FIELD_VALUE.getPreferredName(), AnomalyRecord.CAUSES.getPreferredName(), - AnomalyRecord.ANOMALY_SCORE.getPreferredName(), - AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName(), - AnomalyRecord.INITIAL_NORMALIZED_PROBABILITY.getPreferredName(), + AnomalyRecord.RECORD_SCORE.getPreferredName(), + AnomalyRecord.INITIAL_RECORD_SCORE.getPreferredName(), AnomalyRecord.BUCKET_SPAN.getPreferredName(), AnomalyRecord.SEQUENCE_NUM.getPreferredName(), Bucket.ANOMALY_SCORE.getPreferredName(), Bucket.BUCKET_INFLUENCERS.getPreferredName(), Bucket.BUCKET_SPAN.getPreferredName(), - Bucket.MAX_NORMALIZED_PROBABILITY.getPreferredName(), Bucket.IS_INTERIM.getPreferredName(), Bucket.RECORD_COUNT.getPreferredName(), Bucket.EVENT_COUNT.getPreferredName(), @@ -121,8 +118,8 @@ public final class ReservedFieldNames { Influencer.PROBABILITY.getPreferredName(), Influencer.INFLUENCER_FIELD_NAME.getPreferredName(), Influencer.INFLUENCER_FIELD_VALUE.getPreferredName(), - Influencer.INITIAL_ANOMALY_SCORE.getPreferredName(), - Influencer.ANOMALY_SCORE.getPreferredName(), + Influencer.INITIAL_INFLUENCER_SCORE.getPreferredName(), + Influencer.INFLUENCER_SCORE.getPreferredName(), Influencer.BUCKET_SPAN.getPreferredName(), Influencer.SEQUENCE_NUM.getPreferredName(), @@ -148,6 +145,7 @@ public final class ReservedFieldNames { ModelSnapshot.LATEST_RESULT_TIME.getPreferredName(), PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName(), + PerPartitionMaxProbabilities.MAX_RECORD_SCORE.getPreferredName(), Result.RESULT_TYPE.getPreferredName(), Result.TIMESTAMP.getPreferredName() diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java index 0c1120fc785..415b2ffe5be 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetBucketsAction.java @@ -71,11 +71,6 @@ public class RestGetBucketsAction extends BaseRestHandler { request.setAnomalyScore( Double.parseDouble(restRequest.param(GetBucketsAction.Request.ANOMALY_SCORE.getPreferredName(), "0.0"))); } - if (restRequest.hasParam(GetBucketsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName())) { - request.setMaxNormalizedProbability( - Double.parseDouble(restRequest.param( - GetBucketsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName(), "0.0"))); - } if (restRequest.hasParam(GetBucketsAction.Request.PARTITION_VALUE.getPreferredName())) { request.setPartitionValue(restRequest.param(GetBucketsAction.Request.PARTITION_VALUE.getPreferredName())); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java index 2e6c8f0966a..8e122d1f5b2 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetInfluencersAction.java @@ -50,7 +50,7 @@ public class RestGetInfluencersAction extends BaseRestHandler { request.setAnomalyScore( Double.parseDouble(restRequest.param(GetInfluencersAction.Request.ANOMALY_SCORE.getPreferredName(), "0.0"))); request.setSort(restRequest.param(GetInfluencersAction.Request.SORT_FIELD.getPreferredName(), - Influencer.ANOMALY_SCORE.getPreferredName())); + Influencer.INFLUENCER_SCORE.getPreferredName())); request.setDecending(restRequest.paramAsBoolean(GetInfluencersAction.Request.DESCENDING_SORT.getPreferredName(), true)); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java index f74c347e799..27bd9fa77e7 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/rest/results/RestGetRecordsAction.java @@ -45,13 +45,11 @@ public class RestGetRecordsAction extends BaseRestHandler { request.setIncludeInterim(restRequest.paramAsBoolean(GetRecordsAction.Request.INCLUDE_INTERIM.getPreferredName(), false)); request.setPageParams(new PageParams(restRequest.paramAsInt(PageParams.FROM.getPreferredName(), PageParams.DEFAULT_FROM), restRequest.paramAsInt(PageParams.SIZE.getPreferredName(), PageParams.DEFAULT_SIZE))); - request.setAnomalyScore( - Double.parseDouble(restRequest.param(GetRecordsAction.Request.ANOMALY_SCORE_FILTER.getPreferredName(), "0.0"))); + request.setRecordScore( + Double.parseDouble(restRequest.param(GetRecordsAction.Request.RECORD_SCORE_FILTER.getPreferredName(), "0.0"))); request.setSort(restRequest.param(GetRecordsAction.Request.SORT.getPreferredName(), - AnomalyRecord.NORMALIZED_PROBABILITY.getPreferredName())); + AnomalyRecord.RECORD_SCORE.getPreferredName())); request.setDecending(restRequest.paramAsBoolean(GetRecordsAction.Request.DESCENDING.getPreferredName(), true)); - request.setMaxNormalizedProbability( - Double.parseDouble(restRequest.param(GetRecordsAction.Request.MAX_NORMALIZED_PROBABILITY.getPreferredName(), "0.0"))); String partitionValue = restRequest.param(GetRecordsAction.Request.PARTITION_VALUE.getPreferredName()); if (partitionValue != null) { request.setPartitionValue(partitionValue); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java index 723dd56e3ff..cff53d3080c 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionRequestTests.java @@ -19,9 +19,6 @@ public class GetBucketActionRequestTests extends AbstractStreamableXContentTestC if (randomBoolean()) { request.setTimestamp(String.valueOf(randomLong())); } else { - if (randomBoolean()) { - request.setMaxNormalizedProbability(randomDouble()); - } if (randomBoolean()) { request.setPartitionValue(randomAsciiOfLengthBetween(1, 20)); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java index 7d884522373..d8fd9e5b386 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetBucketActionResponseTests.java @@ -58,9 +58,6 @@ public class GetBucketActionResponseTests extends AbstractStreamableTestCase partitionScores = new ArrayList<>(size); @@ -89,7 +86,6 @@ public class GetBucketActionResponseTests extends AbstractStreamableTestCase records = new ArrayList<>(size); for (int i = 0; i < size; i++) { AnomalyRecord anomalyRecord = new AnomalyRecord(jobId, new Date(randomLong()), randomNonNegativeLong(), sequenceNum++); - anomalyRecord.setAnomalyScore(randomDouble()); anomalyRecord.setActual(Collections.singletonList(randomDouble())); anomalyRecord.setTypical(Collections.singletonList(randomDouble())); anomalyRecord.setProbability(randomDouble()); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java index 352bee43bd6..29230e3409f 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetInfluencersActionResponseTests.java @@ -23,8 +23,8 @@ public class GetInfluencersActionResponseTests extends AbstractStreamableTestCas for (int j = 0; j < listSize; j++) { Influencer influencer = new Influencer(randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), new Date(randomNonNegativeLong()), randomNonNegativeLong(), j + 1); - influencer.setAnomalyScore(randomDouble()); - influencer.setInitialAnomalyScore(randomDouble()); + influencer.setInfluencerScore(randomDouble()); + influencer.setInitialInfluencerScore(randomDouble()); influencer.setProbability(randomDouble()); influencer.setInterim(randomBoolean()); hits.add(influencer); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java index 3f2cfbb3ead..8292ad38e09 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/action/GetRecordsActionRequestTests.java @@ -38,14 +38,11 @@ public class GetRecordsActionRequestTests extends AbstractStreamableXContentTest request.setDecending(randomBoolean()); } if (randomBoolean()) { - request.setAnomalyScore(randomDouble()); + request.setRecordScore(randomDouble()); } if (randomBoolean()) { request.setIncludeInterim(randomBoolean()); } - if (randomBoolean()) { - request.setMaxNormalizedProbability(randomDouble()); - } if (randomBoolean()) { int from = randomInt(PageParams.MAX_FROM_SIZE_SUM); int maxSize = PageParams.MAX_FROM_SIZE_SUM - from; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java index 471297b22cc..1c3b83f224d 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java @@ -51,7 +51,7 @@ public class MlJobIT extends ESRestTestCase { "\"result_type\": { \"type\" : \"keyword\" }," + "\"timestamp\": { \"type\" : \"date\" }, " + "\"anomaly_score\": { \"type\" : \"double\" }, " + - "\"normalized_probability\": { \"type\" : \"double\" }, " + + "\"record_score\": { \"type\" : \"double\" }, " + "\"over_field_value\": { \"type\" : \"keyword\" }, " + "\"partition_field_value\": { \"type\" : \"keyword\" }, " + "\"by_field_value\": { \"type\" : \"keyword\" }, " + diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java index a5a0c4edb1b..94d3a481b5c 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/BucketsQueryBuilderTests.java @@ -17,7 +17,6 @@ public class BucketsQueryBuilderTests extends ESTestCase { assertEquals(false, query.isIncludeInterim()); assertEquals(false, query.isExpand()); assertEquals(0.0, query.getAnomalyScoreFilter(), 0.0001); - assertEquals(0.0, query.getNormalizedProbability(), 0.0001); assertNull(query.getStart()); assertNull(query.getEnd()); assertEquals("timestamp", query.getSortField()); @@ -31,7 +30,6 @@ public class BucketsQueryBuilderTests extends ESTestCase { .includeInterim(true) .expand(true) .anomalyScoreThreshold(50.0d) - .normalizedProbabilityThreshold(70.0d) .start("1000") .end("2000") .partitionValue("foo") @@ -44,7 +42,6 @@ public class BucketsQueryBuilderTests extends ESTestCase { assertEquals(true, query.isIncludeInterim()); assertEquals(true, query.isExpand()); assertEquals(50.0d, query.getAnomalyScoreFilter(), 0.00001); - assertEquals(70.0d, query.getNormalizedProbability(), 0.00001); assertEquals("1000", query.getStart()); assertEquals("2000", query.getEnd()); assertEquals("foo", query.getPartitionValue()); @@ -59,7 +56,6 @@ public class BucketsQueryBuilderTests extends ESTestCase { .includeInterim(true) .expand(true) .anomalyScoreThreshold(50.0d) - .normalizedProbabilityThreshold(70.0d) .start("1000") .end("2000") .partitionValue("foo"); @@ -70,7 +66,6 @@ public class BucketsQueryBuilderTests extends ESTestCase { .includeInterim(true) .expand(true) .anomalyScoreThreshold(50.0d) - .normalizedProbabilityThreshold(70.0d) .start("1000") .end("2000") .partitionValue("foo"); @@ -85,7 +80,6 @@ public class BucketsQueryBuilderTests extends ESTestCase { .includeInterim(true) .expand(true) .anomalyScoreThreshold(50.0d) - .normalizedProbabilityThreshold(70.0d) .start("1000") .end("2000") .partitionValue("foo"); @@ -97,10 +91,9 @@ public class BucketsQueryBuilderTests extends ESTestCase { .includeInterim(true) .expand(true) .anomalyScoreThreshold(50.1d) - .normalizedProbabilityThreshold(70.0d) .start("1000") .end("2000") .partitionValue("foo"); assertFalse(query.build().equals(query2.build())); } -} \ No newline at end of file +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java index 97abbb2b787..c7bdd79d5dd 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/InfluencersQueryBuilderTests.java @@ -19,7 +19,7 @@ public class InfluencersQueryBuilderTests extends ESTestCase { assertEquals(0.0, query.getAnomalyScoreFilter(), 0.0001); assertNull(query.getStart()); assertNull(query.getEnd()); - assertEquals(Influencer.ANOMALY_SCORE.getPreferredName(), query.getSortField()); + assertEquals(Influencer.INFLUENCER_SCORE.getPreferredName(), query.getSortField()); assertFalse(query.isSortDescending()); } @@ -84,4 +84,4 @@ public class InfluencersQueryBuilderTests extends ESTestCase { .end("2000"); assertFalse(query.build().equals(query2.build())); } -} \ No newline at end of file +} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java index 5887cced56a..c9fd145b329 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobProviderTests.java @@ -297,8 +297,7 @@ public class JobProviderTests extends ESTestCase { Client client = getMockedClient(queryBuilder -> {queryBuilderHolder[0] = queryBuilder;}, response); JobProvider provider = createProvider(client); - BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(0.0) - .normalizedProbabilityThreshold(1.0); + BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(1.0); @SuppressWarnings({"unchecked", "rawtypes"}) QueryPage[] holder = new QueryPage[1]; @@ -308,7 +307,7 @@ public class JobProviderTests extends ESTestCase { QueryBuilder query = queryBuilderHolder[0]; String queryString = query.toString(); assertTrue( - queryString.matches("(?s).*max_normalized_probability[^}]*from. : 1\\.0.*must_not[^}]*term[^}]*is_interim.*value. : .true" + + queryString.matches("(?s).*anomaly_score[^}]*from. : 1\\.0.*must_not[^}]*term[^}]*is_interim.*value. : .true" + ".*")); } @@ -333,7 +332,7 @@ public class JobProviderTests extends ESTestCase { JobProvider provider = createProvider(client); BucketsQueryBuilder bq = new BucketsQueryBuilder().from(from).size(size).anomalyScoreThreshold(5.1) - .normalizedProbabilityThreshold(10.9).includeInterim(true); + .includeInterim(true); @SuppressWarnings({"unchecked", "rawtypes"}) QueryPage[] holder = new QueryPage[1]; @@ -342,7 +341,6 @@ public class JobProviderTests extends ESTestCase { assertEquals(1L, buckets.count()); QueryBuilder query = queryBuilderHolder[0]; String queryString = query.toString(); - assertTrue(queryString.matches("(?s).*max_normalized_probability[^}]*from. : 10\\.9.*")); assertTrue(queryString.matches("(?s).*anomaly_score[^}]*from. : 5\\.1.*")); assertFalse(queryString.matches("(?s).*is_interim.*")); } @@ -371,7 +369,6 @@ public class JobProviderTests extends ESTestCase { bq.from(from); bq.size(size); bq.anomalyScoreThreshold(5.1); - bq.normalizedProbabilityThreshold(10.9); bq.includeInterim(true); @SuppressWarnings({"unchecked", "rawtypes"}) @@ -381,7 +378,6 @@ public class JobProviderTests extends ESTestCase { assertEquals(1L, buckets.count()); QueryBuilder query = queryBuilderHolder[0]; String queryString = query.toString(); - assertTrue(queryString.matches("(?s).*max_normalized_probability[^}]*from. : 10\\.9.*")); assertTrue(queryString.matches("(?s).*anomaly_score[^}]*from. : 5\\.1.*")); assertFalse(queryString.matches("(?s).*is_interim.*")); } @@ -488,8 +484,8 @@ public class JobProviderTests extends ESTestCase { JobProvider provider = createProvider(client); RecordsQueryBuilder rqb = new RecordsQueryBuilder().from(from).size(size).epochStart(String.valueOf(now.getTime())) - .epochEnd(String.valueOf(now.getTime())).includeInterim(true).sortField(sortfield).anomalyScoreThreshold(11.1) - .normalizedProbability(2.2); + .epochEnd(String.valueOf(now.getTime())).includeInterim(true).sortField(sortfield) + .recordScore(2.2); @SuppressWarnings({"unchecked", "rawtypes"}) QueryPage[] holder = new QueryPage[1]; @@ -545,8 +541,7 @@ public class JobProviderTests extends ESTestCase { rqb.epochEnd(String.valueOf(now.getTime())); rqb.includeInterim(true); rqb.sortField(sortfield); - rqb.anomalyScoreThreshold(11.1); - rqb.normalizedProbability(2.2); + rqb.recordScore(2.2); @SuppressWarnings({"unchecked", "rawtypes"}) QueryPage[] holder = new QueryPage[1]; @@ -729,27 +724,27 @@ public class JobProviderTests extends ESTestCase { Date now = new Date(); List> source = new ArrayList<>(); - Map recordMap1 = new HashMap<>(); - recordMap1.put("job_id", "foo"); - recordMap1.put("probability", 0.555); - recordMap1.put("influencer_field_name", "Builder"); - recordMap1.put("timestamp", now.getTime()); - recordMap1.put("influencer_field_value", "Bob"); - recordMap1.put("initial_anomaly_score", 22.2); - recordMap1.put("anomaly_score", 22.6); - recordMap1.put("bucket_span", 123); - recordMap1.put("sequence_num", 1); + Map influencerMap1 = new HashMap<>(); + influencerMap1.put("job_id", "foo"); + influencerMap1.put("probability", 0.555); + influencerMap1.put("influencer_field_name", "Builder"); + influencerMap1.put("timestamp", now.getTime()); + influencerMap1.put("influencer_field_value", "Bob"); + influencerMap1.put("initial_influencer_score", 22.2); + influencerMap1.put("influencer_score", 22.6); + influencerMap1.put("bucket_span", 123); + influencerMap1.put("sequence_num", 1); Map recordMap2 = new HashMap<>(); recordMap2.put("job_id", "foo"); recordMap2.put("probability", 0.99); recordMap2.put("influencer_field_name", "Builder"); recordMap2.put("timestamp", now.getTime()); recordMap2.put("influencer_field_value", "James"); - recordMap2.put("initial_anomaly_score", 5.0); - recordMap2.put("anomaly_score", 5.0); + recordMap2.put("initial_influencer_score", 5.0); + recordMap2.put("influencer_score", 5.0); recordMap2.put("bucket_span", 123); recordMap2.put("sequence_num", 2); - source.add(recordMap1); + source.add(influencerMap1); source.add(recordMap2); int from = 4; @@ -775,15 +770,15 @@ public class JobProviderTests extends ESTestCase { assertEquals("Builder", records.get(0).getInfluencerFieldName()); assertEquals(now, records.get(0).getTimestamp()); assertEquals(0.555, records.get(0).getProbability(), 0.00001); - assertEquals(22.6, records.get(0).getAnomalyScore(), 0.00001); - assertEquals(22.2, records.get(0).getInitialAnomalyScore(), 0.00001); + assertEquals(22.6, records.get(0).getInfluencerScore(), 0.00001); + assertEquals(22.2, records.get(0).getInitialInfluencerScore(), 0.00001); assertEquals("James", records.get(1).getInfluencerFieldValue()); assertEquals("Builder", records.get(1).getInfluencerFieldName()); assertEquals(now, records.get(1).getTimestamp()); assertEquals(0.99, records.get(1).getProbability(), 0.00001); - assertEquals(5.0, records.get(1).getAnomalyScore(), 0.00001); - assertEquals(5.0, records.get(1).getInitialAnomalyScore(), 0.00001); + assertEquals(5.0, records.get(1).getInfluencerScore(), 0.00001); + assertEquals(5.0, records.get(1).getInitialInfluencerScore(), 0.00001); } public void testInfluencers_WithInterim() throws InterruptedException, ExecutionException, IOException { @@ -791,28 +786,28 @@ public class JobProviderTests extends ESTestCase { Date now = new Date(); List> source = new ArrayList<>(); - Map recordMap1 = new HashMap<>(); - recordMap1.put("job_id", "foo"); - recordMap1.put("probability", 0.555); - recordMap1.put("influencer_field_name", "Builder"); - recordMap1.put("timestamp", now.getTime()); - recordMap1.put("influencer_field_value", "Bob"); - recordMap1.put("initial_anomaly_score", 22.2); - recordMap1.put("anomaly_score", 22.6); - recordMap1.put("bucket_span", 123); - recordMap1.put("sequence_num", 1); - Map recordMap2 = new HashMap<>(); - recordMap2.put("job_id", "foo"); - recordMap2.put("probability", 0.99); - recordMap2.put("influencer_field_name", "Builder"); - recordMap2.put("timestamp", now.getTime()); - recordMap2.put("influencer_field_value", "James"); - recordMap2.put("initial_anomaly_score", 5.0); - recordMap2.put("anomaly_score", 5.0); - recordMap2.put("bucket_span", 123); - recordMap2.put("sequence_num", 2); - source.add(recordMap1); - source.add(recordMap2); + Map influencerMap1 = new HashMap<>(); + influencerMap1.put("job_id", "foo"); + influencerMap1.put("probability", 0.555); + influencerMap1.put("influencer_field_name", "Builder"); + influencerMap1.put("timestamp", now.getTime()); + influencerMap1.put("influencer_field_value", "Bob"); + influencerMap1.put("initial_influencer_score", 22.2); + influencerMap1.put("influencer_score", 22.6); + influencerMap1.put("bucket_span", 123); + influencerMap1.put("sequence_num", 1); + Map influencerMap2 = new HashMap<>(); + influencerMap2.put("job_id", "foo"); + influencerMap2.put("probability", 0.99); + influencerMap2.put("influencer_field_name", "Builder"); + influencerMap2.put("timestamp", now.getTime()); + influencerMap2.put("influencer_field_value", "James"); + influencerMap2.put("initial_influencer_score", 5.0); + influencerMap2.put("influencer_score", 5.0); + influencerMap2.put("bucket_span", 123); + influencerMap2.put("sequence_num", 2); + source.add(influencerMap1); + source.add(influencerMap2); int from = 4; int size = 3; @@ -837,15 +832,15 @@ public class JobProviderTests extends ESTestCase { assertEquals("Builder", records.get(0).getInfluencerFieldName()); assertEquals(now, records.get(0).getTimestamp()); assertEquals(0.555, records.get(0).getProbability(), 0.00001); - assertEquals(22.6, records.get(0).getAnomalyScore(), 0.00001); - assertEquals(22.2, records.get(0).getInitialAnomalyScore(), 0.00001); + assertEquals(22.6, records.get(0).getInfluencerScore(), 0.00001); + assertEquals(22.2, records.get(0).getInitialInfluencerScore(), 0.00001); assertEquals("James", records.get(1).getInfluencerFieldValue()); assertEquals("Builder", records.get(1).getInfluencerFieldName()); assertEquals(now, records.get(1).getTimestamp()); assertEquals(0.99, records.get(1).getProbability(), 0.00001); - assertEquals(5.0, records.get(1).getAnomalyScore(), 0.00001); - assertEquals(5.0, records.get(1).getInitialAnomalyScore(), 0.00001); + assertEquals(5.0, records.get(1).getInfluencerScore(), 0.00001); + assertEquals(5.0, records.get(1).getInitialInfluencerScore(), 0.00001); } public void testModelSnapshots() throws InterruptedException, ExecutionException, IOException { @@ -951,80 +946,13 @@ public class JobProviderTests extends ESTestCase { assertTrue(queryString.matches("(?s).*snapshot_id.*value. : .snappyId.*description.*value. : .description1.*")); } - public void testMergePartitionScoresIntoBucket() throws InterruptedException, ExecutionException { - MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); - - JobProvider provider = createProvider(clientBuilder.build()); - - List partitionMaxProbs = new ArrayList<>(); - - List records = new ArrayList<>(); - records.add(createAnomalyRecord("partitionValue1", new Date(2), 1.0)); - records.add(createAnomalyRecord("partitionValue2", new Date(2), 4.0)); - partitionMaxProbs.add(new PerPartitionMaxProbabilities(records)); - - records.clear(); - records.add(createAnomalyRecord("partitionValue1", new Date(3), 2.0)); - records.add(createAnomalyRecord("partitionValue2", new Date(3), 1.0)); - partitionMaxProbs.add(new PerPartitionMaxProbabilities(records)); - - records.clear(); - records.add(createAnomalyRecord("partitionValue1", new Date(5), 3.0)); - records.add(createAnomalyRecord("partitionValue2", new Date(5), 2.0)); - partitionMaxProbs.add(new PerPartitionMaxProbabilities(records)); - - List buckets = new ArrayList<>(); - buckets.add(createBucketAtEpochTime(1)); - buckets.add(createBucketAtEpochTime(2)); - buckets.add(createBucketAtEpochTime(3)); - buckets.add(createBucketAtEpochTime(4)); - buckets.add(createBucketAtEpochTime(5)); - buckets.add(createBucketAtEpochTime(6)); - - provider.mergePartitionScoresIntoBucket(partitionMaxProbs, buckets, "partitionValue1"); - assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); - assertEquals(1.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); - assertEquals(2.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); - assertEquals(3.0, buckets.get(4).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(5).getMaxNormalizedProbability(), 0.001); - - provider.mergePartitionScoresIntoBucket(partitionMaxProbs, buckets, "partitionValue2"); - assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); - assertEquals(4.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); - assertEquals(1.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); - assertEquals(2.0, buckets.get(4).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(5).getMaxNormalizedProbability(), 0.001); - } - - private AnomalyRecord createAnomalyRecord(String partitionFieldValue, Date timestamp, double normalizedProbability) { + private AnomalyRecord createAnomalyRecord(String partitionFieldValue, Date timestamp, double recordScore) { AnomalyRecord record = new AnomalyRecord("foo", timestamp, 600, 42); record.setPartitionFieldValue(partitionFieldValue); - record.setNormalizedProbability(normalizedProbability); + record.setRecordScore(recordScore); return record; } - public void testMergePartitionScoresIntoBucket_WithEmptyScoresList() throws InterruptedException, ExecutionException { - MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME); - - JobProvider provider = createProvider(clientBuilder.build()); - - List scores = new ArrayList<>(); - - List buckets = new ArrayList<>(); - buckets.add(createBucketAtEpochTime(1)); - buckets.add(createBucketAtEpochTime(2)); - buckets.add(createBucketAtEpochTime(3)); - buckets.add(createBucketAtEpochTime(4)); - - provider.mergePartitionScoresIntoBucket(scores, buckets, "partitionValue"); - assertEquals(0.0, buckets.get(0).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(1).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(2).getMaxNormalizedProbability(), 0.001); - assertEquals(0.0, buckets.get(3).getMaxNormalizedProbability(), 0.001); - } - public void testRestoreStateToStream() throws Exception { Map categorizerState = new HashMap<>(); categorizerState.put("catName", "catVal"); @@ -1120,7 +1048,6 @@ public class JobProviderTests extends ESTestCase { private Bucket createBucketAtEpochTime(long epoch) { Bucket b = new Bucket("foo", new Date(epoch), 123); - b.setMaxNormalizedProbability(10.0); return b; } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java index e392df7bd10..f04532d710b 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersisterTests.java @@ -40,7 +40,6 @@ public class JobResultsPersisterTests extends ESTestCase { bucket.setAnomalyScore(99.9); bucket.setEventCount(57); bucket.setInitialAnomalyScore(88.8); - bucket.setMaxNormalizedProbability(42.0); bucket.setProcessingTimeMs(8888); bucket.setRecordCount(1); @@ -54,7 +53,6 @@ public class JobResultsPersisterTests extends ESTestCase { // We are adding a record but it shouldn't be persisted as part of the bucket AnomalyRecord record = new AnomalyRecord(JOB_ID, new Date(), 600, 2); - record.setAnomalyScore(99.8); bucket.setRecords(Arrays.asList(record)); JobResultsPersister persister = new JobResultsPersister(Settings.EMPTY, client); @@ -65,7 +63,6 @@ public class JobResultsPersisterTests extends ESTestCase { String s = ((IndexRequest)bulkRequest.requests().get(0)).source().utf8ToString(); assertTrue(s.matches(".*anomaly_score.:99\\.9.*")); assertTrue(s.matches(".*initial_anomaly_score.:88\\.8.*")); - assertTrue(s.matches(".*max_normalized_probability.:42\\.0.*")); assertTrue(s.matches(".*record_count.:1.*")); assertTrue(s.matches(".*event_count.:57.*")); assertTrue(s.matches(".*bucket_span.:123456.*")); @@ -92,7 +89,6 @@ public class JobResultsPersisterTests extends ESTestCase { actuals.add(5.0); actuals.add(5.1); r1.setActual(actuals); - r1.setAnomalyScore(99.8); r1.setByFieldName("byName"); r1.setByFieldValue("byValue"); r1.setCorrelatedByFieldValue("testCorrelations"); @@ -100,8 +96,8 @@ public class JobResultsPersisterTests extends ESTestCase { r1.setFieldName("testFieldName"); r1.setFunction("testFunction"); r1.setFunctionDescription("testDescription"); - r1.setInitialNormalizedProbability(23.4); - r1.setNormalizedProbability(0.005); + r1.setInitialRecordScore(23.4); + r1.setRecordScore(0.005); r1.setOverFieldName("overName"); r1.setOverFieldValue("overValue"); r1.setPartitionFieldName("partName"); @@ -120,9 +116,8 @@ public class JobResultsPersisterTests extends ESTestCase { String s = ((IndexRequest) bulkRequest.requests().get(0)).source().utf8ToString(); assertTrue(s.matches(".*detector_index.:3.*")); assertTrue(s.matches(".*\"probability\":0\\.1.*")); - assertTrue(s.matches(".*\"anomaly_score\":99\\.8.*")); - assertTrue(s.matches(".*\"normalized_probability\":0\\.005.*")); - assertTrue(s.matches(".*initial_normalized_probability.:23.4.*")); + assertTrue(s.matches(".*\"record_score\":0\\.005.*")); + assertTrue(s.matches(".*initial_record_score.:23.4.*")); assertTrue(s.matches(".*bucket_span.:42.*")); assertTrue(s.matches(".*by_field_name.:.byName.*")); assertTrue(s.matches(".*by_field_value.:.byValue.*")); @@ -144,8 +139,8 @@ public class JobResultsPersisterTests extends ESTestCase { List influencers = new ArrayList<>(); Influencer inf = new Influencer(JOB_ID, "infName1", "infValue1", new Date(), 600, 1); - inf.setAnomalyScore(16); - inf.setInitialAnomalyScore(55.5); + inf.setInfluencerScore(16); + inf.setInitialInfluencerScore(55.5); inf.setProbability(0.4); influencers.add(inf); @@ -158,8 +153,8 @@ public class JobResultsPersisterTests extends ESTestCase { assertTrue(s.matches(".*probability.:0\\.4.*")); assertTrue(s.matches(".*influencer_field_name.:.infName1.*")); assertTrue(s.matches(".*influencer_field_value.:.infValue1.*")); - assertTrue(s.matches(".*initial_anomaly_score.:55\\.5.*")); - assertTrue(s.matches(".*anomaly_score.:16\\.0.*")); + assertTrue(s.matches(".*initial_influencer_score.:55\\.5.*")); + assertTrue(s.matches(".*influencer_score.:16\\.0.*")); } @SuppressWarnings({"unchecked", "rawtypes"}) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java index 0c5871bb18a..80f92709f4a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/output/AutodetectResultsParserTests.java @@ -30,7 +30,7 @@ public class AutodetectResultsParserTests extends ESTestCase { public static final String METRIC_OUTPUT_SAMPLE = "[{\"bucket\": {\"job_id\":\"foo\",\"timestamp\":1359450000000," + "\"bucket_span\":22, \"records\":[]," - + "\"max_normalized_probability\":0, \"anomaly_score\":0,\"record_count\":0,\"event_count\":806,\"bucket_influencers\":[" + + "\"anomaly_score\":0,\"record_count\":0,\"event_count\":806,\"bucket_influencers\":[" + "{\"sequence_num\":1,\"timestamp\":1359450000000,\"bucket_span\":22,\"job_id\":\"foo\",\"anomaly_score\":0," + "\"probability\":0.0, \"influencer_field_name\":\"bucket_time\"," + "\"initial_anomaly_score\":0.0}]}},{\"quantiles\": {\"job_id\":\"foo\", \"quantile_state\":\"[normalizer 1.1, normalizer 2" + @@ -50,7 +50,7 @@ public class AutodetectResultsParserTests extends ESTestCase { + "\"probability\":0.0473552,\"by_field_name\":\"airline\",\"by_field_value\":\"SWA\", \"typical\":[152.148]," + "\"actual\":[96.6425],\"field_name\":\"responsetime\",\"function\":\"min\",\"partition_field_name\":\"\"," + "\"partition_field_value\":\"\"}]," - + "\"initial_anomaly_score\":0.0140005, \"anomaly_score\":20.22688, \"max_normalized_probability\":10.5688, \"record_count\":4," + + "\"initial_anomaly_score\":0.0140005, \"anomaly_score\":20.22688, \"record_count\":4," + "\"event_count\":820,\"bucket_influencers\":[{\"timestamp\":1359453600000,\"bucket_span\":22,\"sequence_num\":5," + "\"job_id\":\"foo\", \"raw_anomaly_score\":0.0140005, \"probability\":0.01,\"influencer_field_name\":\"bucket_time\"," + "\"initial_anomaly_score\":20.22688,\"anomaly_score\":20.22688} ,{\"timestamp\":1359453600000,\"bucket_span\":22," @@ -65,168 +65,168 @@ public class AutodetectResultsParserTests extends ESTestCase { + "\"sequence_num\":1,\"function\":\"max\"," + "\"causes\":[{\"probability\":1.38951e-08,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"mail.google.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[9.19027e+07]}]," - + "\"normalized_probability\":100,\"anomaly_score\":44.7324},{\"probability\":3.86587e-07,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":100,\"anomaly_score\":44.7324},{\"probability\":3.86587e-07,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":3.86587e-07,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[3.20093e+07]}]," - + "\"normalized_probability\":89.5834,\"anomaly_score\":44.7324},{\"probability\":0.00500083,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":89.5834,\"anomaly_score\":44.7324},{\"probability\":0.00500083,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.00500083,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[6.61812e+06]}]," - + "\"normalized_probability\":1.19856,\"anomaly_score\":44.7324},{\"probability\":0.0152333,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":1.19856,\"anomaly_score\":44.7324},{\"probability\":0.0152333,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"emea.salesforce.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0152333,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"emea.salesforce.com\",\"function\":\"max\",\"typical\":[101534],\"actual\":[5.36373e+06]}]," - + "\"normalized_probability\":0.303996,\"anomaly_score\":44.7324}],\"raw_anomaly_score\":1.30397,\"anomaly_score\":44.7324," - + "\"max_normalized_probability\":100,\"record_count\":4,\"event_count\":1235}" + ",{\"flush\":\"testing2\"}" + + "\"record_score\":0.303996,\"anomaly_score\":44.7324}],\"raw_anomaly_score\":1.30397,\"anomaly_score\":44.7324," + + "\"record_count\":4,\"event_count\":1235}" + ",{\"flush\":\"testing2\"}" + ",{\"timestamp\":1379590800,\"records\":[{\"probability\":1.9008e-08,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"mail.google.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":1.9008e-08,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"mail.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.1498e+08]}]," - + "\"normalized_probability\":93.6213,\"anomaly_score\":1.19192},{\"probability\":1.01013e-06,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":93.6213,\"anomaly_score\":1.19192},{\"probability\":1.01013e-06,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":1.01013e-06,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"armmf.adobe.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[3.25808e+07]}]," - + "\"normalized_probability\":86.5825,\"anomaly_score\":1.19192},{\"probability\":0.000386185,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":86.5825,\"anomaly_score\":1.19192},{\"probability\":0.000386185,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.000386185,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"0.docs.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[3.22855e+06]}]," - + "\"normalized_probability\":17.1179,\"anomaly_score\":1.19192},{\"probability\":0.00208033,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":17.1179,\"anomaly_score\":1.19192},{\"probability\":0.00208033,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"docs.google.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.00208033,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"docs.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.43328e+06]}]," - + "\"normalized_probability\":3.0692,\"anomaly_score\":1.19192},{\"probability\":0.00312988,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":3.0692,\"anomaly_score\":1.19192},{\"probability\":0.00312988,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"booking2.airasia.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.00312988,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"booking2.airasia.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.15764e+06]}]," - + "\"normalized_probability\":1.99532,\"anomaly_score\":1.19192},{\"probability\":0.00379229,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":1.99532,\"anomaly_score\":1.19192},{\"probability\":0.00379229,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.facebook.com\",\"function\":\"max\",\"causes\":[" + "{\"probability\":0.00379229,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.facebook.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[1.0443e+06]}]," - + "\"normalized_probability\":1.62352,\"anomaly_score\":1.19192},{\"probability\":0.00623576,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":1.62352,\"anomaly_score\":1.19192},{\"probability\":0.00623576,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.airasia.com\",\"function\":\"max\",\"causes\":[" + "{\"probability\":0.00623576,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.airasia.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[792699]}]," - + "\"normalized_probability\":0.935134,\"anomaly_score\":1.19192},{\"probability\":0.00665308,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.935134,\"anomaly_score\":1.19192},{\"probability\":0.00665308,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.google.com\",\"function\":\"max\",\"causes\":[" + "{\"probability\":0.00665308,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[763985]}]," - + "\"normalized_probability\":0.868119,\"anomaly_score\":1.19192},{\"probability\":0.00709315,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.868119,\"anomaly_score\":1.19192},{\"probability\":0.00709315,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"0.drive.google.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.00709315,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"0.drive.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[736442]}]," - + "\"normalized_probability\":0.805994,\"anomaly_score\":1.19192},{\"probability\":0.00755789,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.805994,\"anomaly_score\":1.19192},{\"probability\":0.00755789,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources2.news.com.au\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.00755789,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"resources2.news.com.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[709962]}]," - + "\"normalized_probability\":0.748239,\"anomaly_score\":1.19192},{\"probability\":0.00834974,\"field_name\":" + + "\"record_score\":0.748239,\"anomaly_score\":1.19192},{\"probability\":0.00834974,\"field_name\":" + "\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.calypso.net.au\",\"function\":\"max\"," + "\"causes\":[{\"probability\":0.00834974,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.calypso.net.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[669968]}]," - + "\"normalized_probability\":0.664644,\"anomaly_score\":1.19192},{\"probability\":0.0107711,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.664644,\"anomaly_score\":1.19192},{\"probability\":0.0107711,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"ad.yieldmanager.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0107711,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"ad.yieldmanager.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[576067]}]," - + "\"normalized_probability\":0.485277,\"anomaly_score\":1.19192},{\"probability\":0.0123367,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.485277,\"anomaly_score\":1.19192},{\"probability\":0.0123367,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.google-analytics.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0123367,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.google-analytics.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[530594]}]," - + "\"normalized_probability\":0.406783,\"anomaly_score\":1.19192},{\"probability\":0.0125647,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.406783,\"anomaly_score\":1.19192},{\"probability\":0.0125647,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"bs.serving-sys.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0125647,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"bs.serving-sys.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[524690]}]," - + "\"normalized_probability\":0.396986,\"anomaly_score\":1.19192},{\"probability\":0.0141652,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.396986,\"anomaly_score\":1.19192},{\"probability\":0.0141652,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.google.com.au\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0141652,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.google.com.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[487328]}]," - + "\"normalized_probability\":0.337075,\"anomaly_score\":1.19192},{\"probability\":0.0141742,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.337075,\"anomaly_score\":1.19192},{\"probability\":0.0141742,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources1.news.com.au\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0141742,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"resources1.news.com.au\",\"function\":\"max\",\"typical\":[31356],\"actual\":[487136]}]," - + "\"normalized_probability\":0.336776,\"anomaly_score\":1.19192},{\"probability\":0.0145263,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.336776,\"anomaly_score\":1.19192},{\"probability\":0.0145263,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"b.mail.google.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0145263,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"b.mail.google.com\",\"function\":\"max\",\"typical\":[31356],\"actual\":[479766]}]," - + "\"normalized_probability\":0.325385,\"anomaly_score\":1.19192},{\"probability\":0.0151447,\"field_name\":\"sum_cs_bytes_\"," + + "\"record_score\":0.325385,\"anomaly_score\":1.19192},{\"probability\":0.0151447,\"field_name\":\"sum_cs_bytes_\"," + "\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.rei.com\",\"function\":\"max\",\"causes\":[{" + "\"probability\":0.0151447,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.rei" + ".com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[467450]}],\"normalized_probability\":0.306657,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[467450]}],\"record_score\":0.306657,\"anomaly_score\":1" + ".19192}," + "{\"probability\":0.0164073,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"s3.amazonaws.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0164073," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"s3.amazonaws.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[444511]}],\"normalized_probability\":0.272805,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[444511]}],\"record_score\":0.272805,\"anomaly_score\":1" + ".19192}," + "{\"probability\":0.0201927,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"0-p-06-ash2.channel.facebook.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0201927," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"0-p-06-ash2.channel.facebook.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[389243]}],\"normalized_probability\":0.196685,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[389243]}],\"record_score\":0.196685,\"anomaly_score\":1" + ".19192}," + "{\"probability\":0.0218721,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"booking.airasia.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0218721," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"booking.airasia.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[369509]}],\"normalized_probability\":0.171353," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[369509]}],\"record_score\":0.171353," + "\"anomaly_score\":1.19192},{\"probability\":0.0242411,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.yammer.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0242411," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.yammer.com\"," + "\"function\":\"max\"," - + "\"typical\":[31356],\"actual\":[345295]}],\"normalized_probability\":0.141585,\"anomaly_score\":1.19192}," + + "\"typical\":[31356],\"actual\":[345295]}],\"record_score\":0.141585,\"anomaly_score\":1.19192}," + "{\"probability\":0.0258232,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"safebrowsing-cache.google.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0258232," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"safebrowsing-cache.google.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[331051]}],\"normalized_probability\":0.124748,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[331051]}],\"record_score\":0.124748,\"anomaly_score\":1" + ".19192}," + "{\"probability\":0.0259695,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"fbcdn-profile-a.akamaihd.net\",\"function\":\"max\",\"causes\":[{\"probability\":0.0259695," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"fbcdn-profile-a.akamaihd.net\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[329801]}],\"normalized_probability\":0.123294,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[329801]}],\"record_score\":0.123294,\"anomaly_score\":1" + ".19192}," + "{\"probability\":0.0268874,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.oag.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0268874," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.oag.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[322200]}],\"normalized_probability\":0.114537," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[322200]}],\"record_score\":0.114537," + "\"anomaly_score\":1.19192},{\"probability\":0.0279146,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"booking.qatarairways.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0279146," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"booking.qatarairways.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[314153]}],\"normalized_probability\":0.105419,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[314153]}],\"record_score\":0.105419,\"anomaly_score\":1" + ".19192}," + "{\"probability\":0.0309351,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"resources3.news.com.au\",\"function\":\"max\",\"causes\":[{\"probability\":0.0309351," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources3.news.com.au\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[292918]}],\"normalized_probability\":0.0821156,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[292918]}],\"record_score\":0.0821156,\"anomaly_score\":1" + ".19192}" + ",{\"probability\":0.0335204,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"resources0.news.com.au\",\"function\":\"max\",\"causes\":[{\"probability\":0.0335204," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"resources0.news.com.au\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[277136]}],\"normalized_probability\":0.0655063,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[277136]}],\"record_score\":0.0655063,\"anomaly_score\":1" + ".19192}" + ",{\"probability\":0.0354927,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.southwest.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0354927," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.southwest.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[266310]}],\"normalized_probability\":0.0544615," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[266310]}],\"record_score\":0.0544615," + "\"anomaly_score\":1.19192},{\"probability\":0.0392043,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"syndication.twimg.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0392043," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"syndication.twimg.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[248276]}],\"normalized_probability\":0.0366913,\"anomaly_score\":1" + + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[248276]}],\"record_score\":0.0366913,\"anomaly_score\":1" + ".19192}" + ",{\"probability\":0.0400853,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"" + ",\"over_field_value\":\"mts0.google.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0400853," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"mts0.google.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[244381]}],\"normalized_probability\":0.0329562," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[244381]}],\"record_score\":0.0329562," + "\"anomaly_score\":1.19192},{\"probability\":0.0407335,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"www.onthegotours.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0407335," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"www.onthegotours.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[241600]}],\"normalized_probability\":0.0303116," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[241600]}],\"record_score\":0.0303116," + "\"anomaly_score\":1.19192},{\"probability\":0.0470889,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"chatenabled.mail.google.com\",\"function\":\"max\",\"causes\":[{\"probability\":0.0470889," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"chatenabled.mail.google.com\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[217573]}],\"normalized_probability\":0.00823738," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[217573]}],\"record_score\":0.00823738," + "\"anomaly_score\":1.19192},{\"probability\":0.0491243,\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\"," + "\"over_field_value\":\"googleads.g.doubleclick.net\",\"function\":\"max\",\"causes\":[{\"probability\":0.0491243," + "\"field_name\":\"sum_cs_bytes_\",\"over_field_name\":\"cs_host\",\"over_field_value\":\"googleads.g.doubleclick.net\"," - + "\"function\":\"max\",\"typical\":[31356],\"actual\":[210926]}],\"normalized_probability\":0.00237509," - + "\"anomaly_score\":1.19192}],\"raw_anomaly_score\":1.26918,\"anomaly_score\":1.19192,\"max_normalized_probability\":93.6213," + + "\"function\":\"max\",\"typical\":[31356],\"actual\":[210926]}],\"record_score\":0.00237509," + + "\"anomaly_score\":1.19192}],\"raw_anomaly_score\":1.26918,\"anomaly_score\":1.19192," + "\"record_count\":34,\"event_count\":1159}" + "]"; public void testParser() throws IOException { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java index 5531df7e4c5..7314f674d2e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/BucketNormalizableTests.java @@ -41,12 +41,11 @@ public class BucketNormalizableTests extends ESTestCase { bucket.setBucketInfluencers(Arrays.asList(bucketInfluencer1, bucketInfluencer2)); bucket.setAnomalyScore(88.0); - bucket.setMaxNormalizedProbability(2.0); AnomalyRecord record1 = new AnomalyRecord("foo", bucket.getTimestamp(), 600, 3); - record1.setNormalizedProbability(1.0); + record1.setRecordScore(1.0); AnomalyRecord record2 = new AnomalyRecord("foo", bucket.getTimestamp(), 600, 4); - record2.setNormalizedProbability(2.0); + record2.setRecordScore(2.0); bucket.setRecords(Arrays.asList(record1, record2)); List partitionScores = new ArrayList<>(); @@ -149,10 +148,9 @@ public class BucketNormalizableTests extends ESTestCase { BucketNormalizable bucketNormalizable = new BucketNormalizable(bucket, INDEX_NAME); assertTrue(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.BUCKET_INFLUENCER, 95.0)); - assertTrue(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.RECORD, 42.0)); + assertFalse(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.RECORD, 42.0)); assertEquals(95.0, bucket.getAnomalyScore(), EPSILON); - assertEquals(42.0, bucket.getMaxNormalizedProbability(), EPSILON); } public void testSetMaxChildrenScore_GivenSameScores() { @@ -162,7 +160,6 @@ public class BucketNormalizableTests extends ESTestCase { assertFalse(bucketNormalizable.setMaxChildrenScore(Normalizable.ChildType.RECORD, 2.0)); assertEquals(88.0, bucket.getAnomalyScore(), EPSILON); - assertEquals(2.0, bucket.getMaxNormalizedProbability(), EPSILON); } public void testSetParentScore() { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java index 9056a71cb7b..68f0420a844 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/InfluencerNormalizableTests.java @@ -19,8 +19,8 @@ public class InfluencerNormalizableTests extends ESTestCase { @Before public void setUpInfluencer() { influencer = new Influencer("foo", "airline", "AAL", new Date(), 600, 1); - influencer.setAnomalyScore(1.0); - influencer.setInitialAnomalyScore(2.0); + influencer.setInfluencerScore(1.0); + influencer.setInitialInfluencerScore(2.0); influencer.setProbability(0.05); } @@ -66,7 +66,7 @@ public class InfluencerNormalizableTests extends ESTestCase { normalizable.setNormalizedScore(99.0); assertEquals(99.0, normalizable.getNormalizedScore(), EPSILON); - assertEquals(99.0, influencer.getAnomalyScore(), EPSILON); + assertEquals(99.0, influencer.getInfluencerScore(), EPSILON); } public void testGetChildrenTypes() { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java index 29d829ddf05..dbf36691e87 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/process/normalizer/ScoresUpdaterTests.java @@ -124,7 +124,6 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket = generateBucket(new Date(0)); bucket.setAnomalyScore(42.0); bucket.addBucketInfluencer(createTimeBucketInfluencer(bucket.getTimestamp(), 0.04, 42.0)); - bucket.setMaxNormalizedProbability(50.0); Deque buckets = new ArrayDeque<>(); buckets.add(bucket); @@ -166,14 +165,12 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket = generateBucket(new Date(i * 1000)); bucket.setAnomalyScore(42.0); bucket.addBucketInfluencer(createTimeBucketInfluencer(bucket.getTimestamp(), 0.04, 42.0)); - bucket.setMaxNormalizedProbability(50.0); batch1.add(bucket); } Bucket secondBatchBucket = generateBucket(new Date(10000 * 1000)); secondBatchBucket.addBucketInfluencer(createTimeBucketInfluencer(secondBatchBucket.getTimestamp(), 0.04, 42.0)); secondBatchBucket.setAnomalyScore(42.0); - secondBatchBucket.setMaxNormalizedProbability(50.0); Deque batch2 = new ArrayDeque<>(); batch2.add(secondBatchBucket); @@ -193,7 +190,6 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket1 = generateBucket(new Date(0)); bucket1.setAnomalyScore(42.0); bucket1.addBucketInfluencer(createTimeBucketInfluencer(bucket1.getTimestamp(), 0.04, 42.0)); - bucket1.setMaxNormalizedProbability(50.0); List> records = new ArrayList<>(); Date date = new Date(); for (int i=0; i<100000; i++) { @@ -203,7 +199,6 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket2 = generateBucket(new Date(10000 * 1000)); bucket2.addBucketInfluencer(createTimeBucketInfluencer(bucket2.getTimestamp(), 0.04, 42.0)); bucket2.setAnomalyScore(42.0); - bucket2.setMaxNormalizedProbability(50.0); Deque batch = new ArrayDeque<>(); batch.add(bucket1); @@ -240,7 +235,6 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket = generateBucket(new Date(2509200000L)); bucket.setAnomalyScore(42.0); bucket.addBucketInfluencer(createTimeBucketInfluencer(bucket.getTimestamp(), 0.04, 42.0)); - bucket.setMaxNormalizedProbability(50.0); Deque buckets = new ArrayDeque<>(); buckets.add(bucket); @@ -258,7 +252,6 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket = generateBucket(new Date(3600000)); bucket.setAnomalyScore(42.0); bucket.addBucketInfluencer(createTimeBucketInfluencer(bucket.getTimestamp(), 0.04, 42.0)); - bucket.setMaxNormalizedProbability(50.0); Deque buckets = new ArrayDeque<>(); buckets.add(bucket); @@ -277,7 +270,6 @@ public class ScoresUpdaterTests extends ESTestCase { Bucket bucket = generateBucket(new Date(2700000)); bucket.setAnomalyScore(42.0); bucket.addBucketInfluencer(createTimeBucketInfluencer(bucket.getTimestamp(), 0.04, 42.0)); - bucket.setMaxNormalizedProbability(50.0); Deque buckets = new ArrayDeque<>(); buckets.add(bucket); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecordTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecordTests.java index 667c8ee134d..183f16eb29f 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecordTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/AnomalyRecordTests.java @@ -30,10 +30,9 @@ public class AnomalyRecordTests extends AbstractSerializingTestCase { if (randomBoolean()) { bucket.setInterim(randomBoolean()); } - if (randomBoolean()) { - bucket.setMaxNormalizedProbability(randomDouble()); - } if (randomBoolean()) { int size = randomInt(10); List partitionScores = new ArrayList<>(size); @@ -130,16 +127,6 @@ public class BucketTests extends AbstractSerializingTestCase { assertTrue(b1.equals(b2)); } - public void testEquals_GivenDifferentMaxNormalizedProbability() { - Bucket bucket1 = new Bucket("foo", new Date(123), 123); - bucket1.setMaxNormalizedProbability(55.0); - Bucket bucket2 = new Bucket("foo", new Date(123), 123); - bucket2.setMaxNormalizedProbability(55.1); - - assertFalse(bucket1.equals(bucket2)); - assertFalse(bucket2.equals(bucket1)); - } - public void testEquals_GivenDifferentEventCount() { Bucket bucket1 = new Bucket("foo", new Date(123), 123); bucket1.setEventCount(3); @@ -183,9 +170,9 @@ public class BucketTests extends AbstractSerializingTestCase { public void testEquals_GivenSameNumberOfRecordsButDifferent() { AnomalyRecord anomalyRecord1 = new AnomalyRecord("foo", new Date(123), 123, 1); - anomalyRecord1.setAnomalyScore(1.0); + anomalyRecord1.setRecordScore(1.0); AnomalyRecord anomalyRecord2 = new AnomalyRecord("foo", new Date(123), 123, 2); - anomalyRecord1.setAnomalyScore(2.0); + anomalyRecord1.setRecordScore(2.0); Bucket bucket1 = new Bucket("foo", new Date(123), 123); bucket1.setRecords(Arrays.asList(anomalyRecord1)); @@ -231,7 +218,6 @@ public class BucketTests extends AbstractSerializingTestCase { bucket1.setInitialAnomalyScore(92.0); bucket1.setEventCount(134); bucket1.setInterim(true); - bucket1.setMaxNormalizedProbability(33.3); bucket1.setRecordCount(4); bucket1.setRecords(Arrays.asList(record)); bucket1.addBucketInfluencer(bucketInfluencer); @@ -241,7 +227,6 @@ public class BucketTests extends AbstractSerializingTestCase { bucket2.setInitialAnomalyScore(92.0); bucket2.setEventCount(134); bucket2.setInterim(true); - bucket2.setMaxNormalizedProbability(33.3); bucket2.setRecordCount(4); bucket2.setRecords(Arrays.asList(record)); bucket2.addBucketInfluencer(bucketInfluencer); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluencerTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluencerTests.java index 897516a7089..bc34d0f0b96 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluencerTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluencerTests.java @@ -16,12 +16,12 @@ import java.util.Date; public class InfluencerTests extends AbstractSerializingTestCase { - public Influencer createTestInstance(String jobId) { + public Influencer createTestInstance(String jobId) { Influencer influencer = new Influencer(jobId, randomAsciiOfLengthBetween(1, 20), randomAsciiOfLengthBetween(1, 20), new Date(randomNonNegativeLong()), randomNonNegativeLong(), randomIntBetween(1, 1000)); influencer.setInterim(randomBoolean()); - influencer.setAnomalyScore(randomDouble()); - influencer.setInitialAnomalyScore(randomDouble()); + influencer.setInfluencerScore(randomDouble()); + influencer.setInitialInfluencerScore(randomDouble()); influencer.setProbability(randomDouble()); return influencer; } @@ -50,12 +50,12 @@ public class InfluencerTests extends AbstractSerializingTestCase { } public void testToXContentDoesNotIncludeNameValueFieldWhenReservedWord() throws IOException { - Influencer influencer = new Influencer("foo", AnomalyRecord.ANOMALY_SCORE.getPreferredName(), "bar", new Date(), 300L, 0); + Influencer influencer = new Influencer("foo", Influencer.INFLUENCER_SCORE.getPreferredName(), "bar", new Date(), 300L, 0); XContentBuilder builder = toXContent(influencer, XContentType.JSON); XContentParser parser = createParser(builder); - Object serialisedFieldValue = parser.map().get(AnomalyRecord.ANOMALY_SCORE.getPreferredName()); + Object serialisedFieldValue = parser.map().get(Influencer.INFLUENCER_SCORE.getPreferredName()); + assertNotNull(serialisedFieldValue); assertNotEquals("bar", serialisedFieldValue); - assertEquals(0.0, (Double)serialisedFieldValue, 0.0001); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilitiesTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilitiesTests.java index 3df2fb8f4bb..b7c0c486313 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilitiesTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/results/PerPartitionMaxProbabilitiesTests.java @@ -52,9 +52,9 @@ public class PerPartitionMaxProbabilitiesTests extends AbstractSerializingTestCa assertEquals(2, pProbs.size()); for (PerPartitionMaxProbabilities.PartitionProbability pProb : pProbs) { if (pProb.getPartitionValue().equals("A")) { - assertEquals(40.0, pProb.getMaxNormalizedProbability(), 0.0001); + assertEquals(40.0, pProb.getMaxRecordScore(), 0.0001); } else { - assertEquals(90.0, pProb.getMaxNormalizedProbability(), 0.0001); + assertEquals(90.0, pProb.getMaxRecordScore(), 0.0001); } } } @@ -73,10 +73,10 @@ public class PerPartitionMaxProbabilitiesTests extends AbstractSerializingTestCa assertEquals(90.0, ppMax.getMaxProbabilityForPartition("B"), 0.0001); } - private AnomalyRecord createAnomalyRecord(String partitionFieldValue, double normalizedProbability) { + private AnomalyRecord createAnomalyRecord(String partitionFieldValue, double recordScore) { AnomalyRecord record = new AnomalyRecord("foo", new Date(), 600, 1); record.setPartitionFieldValue(partitionFieldValue); - record.setNormalizedProbability(normalizedProbability); + record.setRecordScore(recordScore); return record; } } diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_buckets.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_buckets.json index f6ee2471d30..496103831b0 100644 --- a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_buckets.json +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_buckets.json @@ -50,10 +50,6 @@ "anomaly_score": { "type": "double", "description": "Filter for the most anomalous buckets" - }, - "max_normalized_probability": { - "type": "double", - "description": "Filter for buckets containing the most anomalous records" } } }, diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_influencers.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_influencers.json index b5a2e5f1e31..4bf6e733588 100644 --- a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_influencers.json +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_influencers.json @@ -31,9 +31,9 @@ "type": "string", "description": "end timestamp for the requested influencers" }, - "anomaly_score": { + "influencer_score": { "type": "double", - "description": "anomaly score threshold for the requested influencers" + "description": "influencer score threshold for the requested influencers" }, "sort": { "type": "string", diff --git a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_records.json b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_records.json index 59169cf15bf..889aa233c66 100644 --- a/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_records.json +++ b/plugin/src/test/resources/rest-api-spec/api/xpack.ml.get_records.json @@ -36,10 +36,7 @@ "type": "string", "description": "End time filter for records" }, - "anomaly_score": { - "type": "double" - }, - "max_normalized_probability": { + "record_score": { "type": "double" }, "partition_value": { diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yaml b/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yaml index 18d74700fc2..ac82a12c5b7 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yaml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_influencers.yaml @@ -10,8 +10,8 @@ setup: type: keyword "timestamp": type: date - "anomaly_score": - type: float + "influencer_score": + type: double "result_type": type: keyword - do: @@ -25,7 +25,7 @@ setup: "timestamp": "2016-06-01T00:00:00Z", "influencer_field_name": "foo", "influencer_field_value": "bar", - "anomaly_score": 80.0, + "influencer_score": 80.0, "result_type" : "influencer", "bucket_span" : 1, "sequence_num" : 1 @@ -42,7 +42,7 @@ setup: "timestamp": "2016-06-02T00:00:00Z", "influencer_field_name": "foo", "influencer_field_value": "zoo", - "anomaly_score": 50.0, + "influencer_score": 50.0, "result_type" : "influencer", "bucket_span" : 1, "sequence_num" : 2 diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yaml b/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yaml index 68e7e38fa3e..d92f6e9a9bb 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yaml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/jobs_get_result_records.yaml @@ -23,7 +23,7 @@ setup: "job_id": "farequote", "result_type": "record", "timestamp": "2016-06-01T00:00:00Z", - "anomaly_score": 60.0, + "record_score": 60.0, "bucket_span": 1, "sequence_num": 1 } @@ -38,7 +38,7 @@ setup: "job_id": "farequote", "result_type": "record", "timestamp": "2016-06-02T00:00:00Z", - "anomaly_score": 80.0, + "record_score": 80.0, "bucket_span": 1, "sequence_num": 2 } diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yaml b/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yaml index d52206ceb46..7b4abe7263e 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yaml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yaml @@ -101,7 +101,7 @@ setup: "timestamp": "2016-06-02T00:00:00Z", "influencer_field_name": "foo", "influencer_field_value": "zoo", - "anomaly_score": 50.0, + "influencer_score": 50.0, "bucket_span": 1, "sequence_num": 3 } @@ -118,7 +118,7 @@ setup: "timestamp": "2016-05-01T00:00:00Z", "influencer_field_name": "foo", "influencer_field_value": "zoo", - "anomaly_score": 50.0, + "influencer_score": 50.0, "bucket_span": 1, "sequence_num": 4 }