From 2aeff7713c66ca226af7371a140eb2f2437280c4 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 29 Mar 2018 17:32:57 +0100 Subject: [PATCH] [ML] Parsing objects from internal indices should be lenient (elastic/x-pack-elasticsearch#4256) All ML objects stored in internal indices are currently parsed strictly. This means unknown fields lead to parsing failures. In turn, this means we cannot add new fields in any of those objects (e.g. bucket, record, calendar, etc.) as it is not backwards compatible. This commit changes this by introducing lenient parsing when it comes to reading those objects from the internal indices. Note we still use strict parsing for the objects we read from the c++ process, which is nice as it guarantees we would detect if any of the fields were renamed on one side but not the other. Also note that even though this is going in from 6.3, we cannot introduce new fields until 7.0. relates elastic/x-pack-elasticsearch#4232 Original commit: elastic/x-pack-elasticsearch@3f95d3c7b96c67cbe9b5a9639560537c1a462bc0 --- .../ml/action/PostCalendarEventsAction.java | 2 +- .../core/ml/action/PutCalendarAction.java | 2 +- .../xpack/core/ml/action/PutFilterAction.java | 2 +- .../xpack/core/ml/calendars/Calendar.java | 17 +++-- .../core/ml/calendars/ScheduledEvent.java | 20 +++--- .../xpack/core/ml/job/config/MlFilter.java | 15 +++-- .../process/autodetect/state/DataCounts.java | 10 +-- .../autodetect/state/ModelSizeStats.java | 32 +++++---- .../autodetect/state/ModelSnapshot.java | 37 ++++++----- .../process/autodetect/state/Quantiles.java | 17 +++-- .../core/ml/job/results/AnomalyCause.java | 41 +++++++----- .../core/ml/job/results/AnomalyRecord.java | 66 ++++++++++--------- .../xpack/core/ml/job/results/Bucket.java | 45 +++++++------ .../core/ml/job/results/BucketInfluencer.java | 35 +++++----- .../ml/job/results/CategoryDefinition.java | 23 ++++--- .../xpack/core/ml/job/results/Forecast.java | 39 ++++++----- .../ml/job/results/ForecastRequestStats.java | 39 ++++++----- .../xpack/core/ml/job/results/Influence.java | 17 +++-- .../xpack/core/ml/job/results/Influencer.java | 30 ++++----- .../xpack/core/ml/job/results/ModelPlot.java | 44 +++++++------ .../core/ml/job/results/PartitionScore.java | 22 ++++--- .../core/ml/notifications/AuditMessage.java | 3 +- .../core/ml/calendars/CalendarTests.java | 21 +++++- .../ml/calendars/ScheduledEventTests.java | 20 +++++- .../core/ml/job/config/MlFilterTests.java | 23 ++++++- .../autodetect/state/ModelSizeStatsTests.java | 24 ++++++- .../autodetect/state/ModelSnapshotTests.java | 27 +++++++- .../autodetect/state/QuantilesTests.java | 23 ++++++- .../ml/job/results/AnomalyCauseTests.java | 24 ++++++- .../ml/job/results/AnomalyRecordTests.java | 41 ++++++++---- .../ml/job/results/BucketInfluencerTests.java | 33 +++++----- .../core/ml/job/results/InfluencerTests.java | 22 +++---- .../ml/action/TransportGetFiltersAction.java | 4 +- .../persistence/BatchedBucketsIterator.java | 2 +- .../BatchedInfluencersIterator.java | 2 +- .../persistence/BatchedRecordsIterator.java | 2 +- .../xpack/ml/job/persistence/JobProvider.java | 34 +++++----- .../ml/job/results/AutodetectResult.java | 21 +++--- .../retention/ExpiredForecastsRemover.java | 2 +- .../xpack/ml/job/results/BucketTests.java | 22 ++++++- .../job/results/CategoryDefinitionTests.java | 23 ++++++- .../results/ForecastRequestStatsTests.java | 23 ++++++- .../xpack/ml/job/results/ForecastTests.java | 17 ++++- .../xpack/ml/job/results/InfluenceTests.java | 22 ++++++- .../xpack/ml/job/results/ModelPlotTests.java | 20 +++++- .../ml/job/results/PartitionScoreTests.java | 25 ++++++- .../MlNativeAutodetectIntegTestCase.java | 12 ++-- .../ml/integration/RevertModelSnapshotIT.java | 2 +- 48 files changed, 704 insertions(+), 345 deletions(-) diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java index 25709db45bf..82e9072751e 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PostCalendarEventsAction.java @@ -56,7 +56,7 @@ public class PostCalendarEventsAction extends Action, Void> PARSER = new ObjectParser<>(NAME, ArrayList::new); static { - PARSER.declareObjectArray(List::addAll, (p, c) -> ScheduledEvent.PARSER.apply(p, null), ScheduledEvent.RESULTS_FIELD); + PARSER.declareObjectArray(List::addAll, (p, c) -> ScheduledEvent.STRICT_PARSER.apply(p, null), ScheduledEvent.RESULTS_FIELD); } public static Request parseRequest(String calendarId, XContentParser parser) throws IOException { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java index 4781faf1f3d..5644d05af04 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutCalendarAction.java @@ -49,7 +49,7 @@ public class PutCalendarAction extends Action PARSER = new ObjectParser<>(ID.getPreferredName(), Builder::new); + public static final ObjectParser STRICT_PARSER = createParser(false); + public static final ObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(Builder::setId, ID); - PARSER.declareStringArray(Builder::setJobIds, JOB_IDS); - PARSER.declareString((builder, s) -> {}, TYPE); - PARSER.declareStringOrNull(Builder::setDescription, DESCRIPTION); + private static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>(ID.getPreferredName(), ignoreUnknownFields, Builder::new); + + parser.declareString(Builder::setId, ID); + parser.declareStringArray(Builder::setJobIds, JOB_IDS); + parser.declareString((builder, s) -> {}, TYPE); + parser.declareStringOrNull(Builder::setDescription, DESCRIPTION); + + return parser; } public static String documentId(String calendarId) { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java index 3d5547f61d7..bba25b9131f 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java @@ -47,12 +47,14 @@ public class ScheduledEvent implements ToXContentObject, Writeable { public static final String SCHEDULED_EVENT_TYPE = "scheduled_event"; public static final String DOCUMENT_ID_PREFIX = "event_"; - public static final ObjectParser PARSER = - new ObjectParser<>("scheduled_event", Builder::new); + public static final ObjectParser STRICT_PARSER = createParser(false); + public static final ObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ScheduledEvent.Builder::description, DESCRIPTION); - PARSER.declareField(ScheduledEvent.Builder::startTime, p -> { + private static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>("scheduled_event", ignoreUnknownFields, Builder::new); + + parser.declareString(ScheduledEvent.Builder::description, DESCRIPTION); + parser.declareField(ScheduledEvent.Builder::startTime, p -> { if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(p.longValue()), ZoneOffset.UTC); } else if (p.currentToken() == XContentParser.Token.VALUE_STRING) { @@ -61,7 +63,7 @@ public class ScheduledEvent implements ToXContentObject, Writeable { throw new IllegalArgumentException( "unexpected token [" + p.currentToken() + "] for [" + START_TIME.getPreferredName() + "]"); }, START_TIME, ObjectParser.ValueType.VALUE); - PARSER.declareField(ScheduledEvent.Builder::endTime, p -> { + parser.declareField(ScheduledEvent.Builder::endTime, p -> { if (p.currentToken() == XContentParser.Token.VALUE_NUMBER) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(p.longValue()), ZoneOffset.UTC); } else if (p.currentToken() == XContentParser.Token.VALUE_STRING) { @@ -71,8 +73,10 @@ public class ScheduledEvent implements ToXContentObject, Writeable { "unexpected token [" + p.currentToken() + "] for [" + END_TIME.getPreferredName() + "]"); }, END_TIME, ObjectParser.ValueType.VALUE); - PARSER.declareString(ScheduledEvent.Builder::calendarId, Calendar.ID); - PARSER.declareString((builder, s) -> {}, TYPE); + parser.declareString(ScheduledEvent.Builder::calendarId, Calendar.ID); + parser.declareString((builder, s) -> {}, TYPE); + + return parser; } public static String documentId(String eventId) { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java index 780b9437455..de6ee3d509c 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/MlFilter.java @@ -35,12 +35,17 @@ public class MlFilter implements ToXContentObject, Writeable { // For QueryPage public static final ParseField RESULTS_FIELD = new ParseField("filters"); - public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), Builder::new); + public static final ObjectParser STRICT_PARSER = createParser(false); + public static final ObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString((builder, s) -> {}, TYPE); - PARSER.declareString(Builder::setId, ID); - PARSER.declareStringArray(Builder::setItems, ITEMS); + private static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>(TYPE.getPreferredName(), ignoreUnknownFields, Builder::new); + + parser.declareString((builder, s) -> {}, TYPE); + parser.declareString(Builder::setId, ID); + parser.declareStringArray(Builder::setItems, ITEMS); + + return parser; } private final String id; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java index 7c461340c38..f2545c5abf7 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/DataCounts.java @@ -73,10 +73,10 @@ public class DataCounts implements ToXContentObject, Writeable { public static final ParseField TYPE = new ParseField("data_counts"); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("data_counts", a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3], - (long) a[4], (long) a[5], (long) a[6], (long) a[7], (long) a[8], (long) a[9], (long) a[10], - (Date) a[11], (Date) a[12], (Date) a[13], (Date) a[14], (Date) a[15])); + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("data_counts", true, + a -> new DataCounts((String) a[0], (long) a[1], (long) a[2], (long) a[3], (long) a[4], (long) a[5], (long) a[6], + (long) a[7], (long) a[8], (long) a[9], (long) a[10], (Date) a[11], (Date) a[12], (Date) a[13], (Date) a[14], + (Date) a[15])); static { PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); @@ -123,7 +123,7 @@ public class DataCounts implements ToXContentObject, Writeable { } else if (p.currentToken() == Token.VALUE_STRING) { return new Date(TimeUtils.dateStringToEpoch(p.text())); } - throw new IllegalArgumentException( + throw new IllegalArgumentException( "unexpected token [" + p.currentToken() + "] for [" + LATEST_EMPTY_BUCKET_TIME.getPreferredName() + "]"); }, LATEST_EMPTY_BUCKET_TIME, ValueType.VALUE); PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java index 80429e0c3da..f0e4ef8367b 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -46,18 +47,21 @@ public class ModelSizeStats implements ToXContentObject, Writeable { public static final ParseField LOG_TIME_FIELD = new ParseField("log_time"); public static final ParseField TIMESTAMP_FIELD = new ParseField("timestamp"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - RESULT_TYPE_FIELD.getPreferredName(), a -> new Builder((String) a[0])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareString((modelSizeStat, s) -> {}, Result.RESULT_TYPE); - PARSER.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); - PARSER.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); - PARSER.declareLong(Builder::setTotalByFieldCount, TOTAL_BY_FIELD_COUNT_FIELD); - PARSER.declareLong(Builder::setTotalOverFieldCount, TOTAL_OVER_FIELD_COUNT_FIELD); - PARSER.declareLong(Builder::setTotalPartitionFieldCount, TOTAL_PARTITION_FIELD_COUNT_FIELD); - PARSER.declareField(Builder::setLogTime, p -> { + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_FIELD.getPreferredName(), + ignoreUnknownFields, a -> new Builder((String) a[0])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareString((modelSizeStat, s) -> {}, Result.RESULT_TYPE); + parser.declareLong(Builder::setModelBytes, MODEL_BYTES_FIELD); + parser.declareLong(Builder::setBucketAllocationFailuresCount, BUCKET_ALLOCATION_FAILURES_COUNT_FIELD); + parser.declareLong(Builder::setTotalByFieldCount, TOTAL_BY_FIELD_COUNT_FIELD); + parser.declareLong(Builder::setTotalOverFieldCount, TOTAL_OVER_FIELD_COUNT_FIELD); + parser.declareLong(Builder::setTotalPartitionFieldCount, TOTAL_PARTITION_FIELD_COUNT_FIELD); + parser.declareField(Builder::setLogTime, p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -66,7 +70,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable { throw new IllegalArgumentException( "unexpected token [" + p.currentToken() + "] for [" + LOG_TIME_FIELD.getPreferredName() + "]"); }, LOG_TIME_FIELD, ValueType.VALUE); - PARSER.declareField(Builder::setTimestamp, p -> { + parser.declareField(Builder::setTimestamp, p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -75,7 +79,9 @@ public class ModelSizeStats implements ToXContentObject, Writeable { throw new IllegalArgumentException( "unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP_FIELD.getPreferredName() + "]"); }, TIMESTAMP_FIELD, ValueType.VALUE); - PARSER.declareField(Builder::setMemoryStatus, p -> MemoryStatus.fromString(p.text()), MEMORY_STATUS_FIELD, ValueType.STRING); + parser.declareField(Builder::setMemoryStatus, p -> MemoryStatus.fromString(p.text()), MEMORY_STATUS_FIELD, ValueType.STRING); + + return parser; } /** diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java index 1bcb503fdc7..2fff343904e 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshot.java @@ -56,24 +56,29 @@ public class ModelSnapshot implements ToXContentObject, Writeable { */ public static final ParseField TYPE = new ParseField("model_snapshot"); - public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), Builder::new); + public static final ObjectParser STRICT_PARSER = createParser(false); + public static final ObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(Builder::setJobId, Job.ID); - PARSER.declareString(Builder::setMinVersion, MIN_VERSION); - PARSER.declareField(Builder::setTimestamp, p -> { + private static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>(TYPE.getPreferredName(), ignoreUnknownFields, Builder::new); + + parser.declareString(Builder::setJobId, Job.ID); + parser.declareString(Builder::setMinVersion, MIN_VERSION); + parser.declareField(Builder::setTimestamp, p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { return new Date(TimeUtils.dateStringToEpoch(p.text())); } - throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]"); + throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + + TIMESTAMP.getPreferredName() + "]"); }, TIMESTAMP, ValueType.VALUE); - PARSER.declareString(Builder::setDescription, DESCRIPTION); - PARSER.declareString(Builder::setSnapshotId, ModelSnapshotField.SNAPSHOT_ID); - PARSER.declareInt(Builder::setSnapshotDocCount, SNAPSHOT_DOC_COUNT); - PARSER.declareObject(Builder::setModelSizeStats, ModelSizeStats.PARSER, ModelSizeStats.RESULT_TYPE_FIELD); - PARSER.declareField(Builder::setLatestRecordTimeStamp, p -> { + parser.declareString(Builder::setDescription, DESCRIPTION); + parser.declareString(Builder::setSnapshotId, ModelSnapshotField.SNAPSHOT_ID); + parser.declareInt(Builder::setSnapshotDocCount, SNAPSHOT_DOC_COUNT); + parser.declareObject(Builder::setModelSizeStats, ignoreUnknownFields ? ModelSizeStats.LENIENT_PARSER : ModelSizeStats.STRICT_PARSER, + ModelSizeStats.RESULT_TYPE_FIELD); + parser.declareField(Builder::setLatestRecordTimeStamp, p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -82,7 +87,7 @@ public class ModelSnapshot implements ToXContentObject, Writeable { throw new IllegalArgumentException( "unexpected token [" + p.currentToken() + "] for [" + LATEST_RECORD_TIME.getPreferredName() + "]"); }, LATEST_RECORD_TIME, ValueType.VALUE); - PARSER.declareField(Builder::setLatestResultTimeStamp, p -> { + parser.declareField(Builder::setLatestResultTimeStamp, p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -91,8 +96,10 @@ public class ModelSnapshot implements ToXContentObject, Writeable { throw new IllegalArgumentException( "unexpected token [" + p.currentToken() + "] for [" + LATEST_RESULT_TIME.getPreferredName() + "]"); }, LATEST_RESULT_TIME, ValueType.VALUE); - PARSER.declareObject(Builder::setQuantiles, Quantiles.PARSER, QUANTILES); - PARSER.declareBoolean(Builder::setRetain, RETAIN); + parser.declareObject(Builder::setQuantiles, ignoreUnknownFields ? Quantiles.LENIENT_PARSER : Quantiles.STRICT_PARSER, QUANTILES); + parser.declareBoolean(Builder::setRetain, RETAIN); + + return parser; } @@ -340,7 +347,7 @@ public class ModelSnapshot implements ToXContentObject, Writeable { try (InputStream stream = bytesReference.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(bytesReference)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - return PARSER.apply(parser, null).build(); + return LENIENT_PARSER.apply(parser, null).build(); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse modelSnapshot", e); } diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/Quantiles.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/Quantiles.java index 3ca39bdecc3..ec92433c376 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/Quantiles.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/Quantiles.java @@ -35,13 +35,18 @@ public class Quantiles implements ToXContentObject, Writeable { */ public static final ParseField TYPE = new ParseField("quantiles"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - TYPE.getPreferredName(), a -> new Quantiles((String) a[0], (Date) a[1], (String) a[2])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> new Date(p.longValue()), TIMESTAMP, ValueType.LONG); - PARSER.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(TYPE.getPreferredName(), ignoreUnknownFields, + a -> new Quantiles((String) a[0], (Date) a[1], (String) a[2])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> new Date(p.longValue()), TIMESTAMP, ValueType.LONG); + parser.declareString(ConstructingObjectParser.constructorArg(), QUANTILE_STATE); + + return parser; } public static String documentId(String jobId) { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCause.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCause.java index 0c53bbe686a..50efe24ab0f 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCause.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCause.java @@ -45,23 +45,30 @@ public class AnomalyCause implements ToXContentObject, Writeable { */ public static final ParseField FIELD_NAME = new ParseField("field_name"); - public static final ObjectParser PARSER = new ObjectParser<>(ANOMALY_CAUSE.getPreferredName(), - AnomalyCause::new); - static { - PARSER.declareDouble(AnomalyCause::setProbability, PROBABILITY); - PARSER.declareString(AnomalyCause::setByFieldName, BY_FIELD_NAME); - PARSER.declareString(AnomalyCause::setByFieldValue, BY_FIELD_VALUE); - PARSER.declareString(AnomalyCause::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); - PARSER.declareString(AnomalyCause::setPartitionFieldName, PARTITION_FIELD_NAME); - PARSER.declareString(AnomalyCause::setPartitionFieldValue, PARTITION_FIELD_VALUE); - PARSER.declareString(AnomalyCause::setFunction, FUNCTION); - PARSER.declareString(AnomalyCause::setFunctionDescription, FUNCTION_DESCRIPTION); - PARSER.declareDoubleArray(AnomalyCause::setTypical, TYPICAL); - PARSER.declareDoubleArray(AnomalyCause::setActual, ACTUAL); - PARSER.declareString(AnomalyCause::setFieldName, FIELD_NAME); - PARSER.declareString(AnomalyCause::setOverFieldName, OVER_FIELD_NAME); - PARSER.declareString(AnomalyCause::setOverFieldValue, OVER_FIELD_VALUE); - PARSER.declareObjectArray(AnomalyCause::setInfluencers, Influence.PARSER, INFLUENCERS); + public static final ObjectParser STRICT_PARSER = createParser(false); + public static final ObjectParser LENIENT_PARSER = createParser(true); + + private static ObjectParser createParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>(ANOMALY_CAUSE.getPreferredName(), ignoreUnknownFields, + AnomalyCause::new); + + parser.declareDouble(AnomalyCause::setProbability, PROBABILITY); + parser.declareString(AnomalyCause::setByFieldName, BY_FIELD_NAME); + parser.declareString(AnomalyCause::setByFieldValue, BY_FIELD_VALUE); + parser.declareString(AnomalyCause::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); + parser.declareString(AnomalyCause::setPartitionFieldName, PARTITION_FIELD_NAME); + parser.declareString(AnomalyCause::setPartitionFieldValue, PARTITION_FIELD_VALUE); + parser.declareString(AnomalyCause::setFunction, FUNCTION); + parser.declareString(AnomalyCause::setFunctionDescription, FUNCTION_DESCRIPTION); + parser.declareDoubleArray(AnomalyCause::setTypical, TYPICAL); + parser.declareDoubleArray(AnomalyCause::setActual, ACTUAL); + parser.declareString(AnomalyCause::setFieldName, FIELD_NAME); + parser.declareString(AnomalyCause::setOverFieldName, OVER_FIELD_NAME); + parser.declareString(AnomalyCause::setOverFieldValue, OVER_FIELD_VALUE); + parser.declareObjectArray(AnomalyCause::setInfluencers, ignoreUnknownFields ? Influence.LENIENT_PARSER : Influence.STRICT_PARSER, + INFLUENCERS); + + return parser; } private double probability; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java index 1689f4f5dd6..360bcfaaead 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecord.java @@ -44,7 +44,6 @@ public class AnomalyRecord implements ToXContentObject, Writeable { /** * Result fields (all detector types) */ - public static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); public static final ParseField PROBABILITY = new ParseField("probability"); public static final ParseField BY_FIELD_NAME = new ParseField("by_field_name"); public static final ParseField BY_FIELD_VALUE = new ParseField("by_field_value"); @@ -79,13 +78,18 @@ public class AnomalyRecord implements ToXContentObject, Writeable { public static final ParseField RECORD_SCORE = new ParseField("record_score"); public static final ParseField INITIAL_RECORD_SCORE = new ParseField("initial_record_score"); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, - a -> new AnomalyRecord((String) a[0], (Date) a[1], (long) a[2])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + // As a record contains fields named after the data fields, the parser for the record should always ignore unknown fields. + // However, it makes sense to offer strict/lenient parsing for other members, e.g. influences, anomaly causes, etc. + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, true, + a -> new AnomalyRecord((String) a[0], (Date) a[1], (long) a[2])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -94,29 +98,31 @@ public class AnomalyRecord implements ToXContentObject, Writeable { throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + Result.TIMESTAMP.getPreferredName() + "]"); }, Result.TIMESTAMP, ValueType.VALUE); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); - PARSER.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE); - PARSER.declareDouble(AnomalyRecord::setProbability, PROBABILITY); - PARSER.declareDouble(AnomalyRecord::setRecordScore, RECORD_SCORE); - PARSER.declareDouble(AnomalyRecord::setInitialRecordScore, INITIAL_RECORD_SCORE); - PARSER.declareInt(AnomalyRecord::setDetectorIndex, Detector.DETECTOR_INDEX); - PARSER.declareBoolean(AnomalyRecord::setInterim, Result.IS_INTERIM); - PARSER.declareString(AnomalyRecord::setByFieldName, BY_FIELD_NAME); - PARSER.declareString(AnomalyRecord::setByFieldValue, BY_FIELD_VALUE); - PARSER.declareString(AnomalyRecord::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); - PARSER.declareString(AnomalyRecord::setPartitionFieldName, PARTITION_FIELD_NAME); - PARSER.declareString(AnomalyRecord::setPartitionFieldValue, PARTITION_FIELD_VALUE); - PARSER.declareString(AnomalyRecord::setFunction, FUNCTION); - PARSER.declareString(AnomalyRecord::setFunctionDescription, FUNCTION_DESCRIPTION); - PARSER.declareDoubleArray(AnomalyRecord::setTypical, TYPICAL); - PARSER.declareDoubleArray(AnomalyRecord::setActual, ACTUAL); - PARSER.declareString(AnomalyRecord::setFieldName, FIELD_NAME); - PARSER.declareString(AnomalyRecord::setOverFieldName, OVER_FIELD_NAME); - PARSER.declareString(AnomalyRecord::setOverFieldValue, OVER_FIELD_VALUE); - PARSER.declareObjectArray(AnomalyRecord::setCauses, AnomalyCause.PARSER, CAUSES); - PARSER.declareObjectArray(AnomalyRecord::setInfluencers, Influence.PARSER, INFLUENCERS); - // For bwc with 5.4 - PARSER.declareInt((anomalyRecord, sequenceNum) -> {}, SEQUENCE_NUM); + parser.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + parser.declareString((anomalyRecord, s) -> {}, Result.RESULT_TYPE); + parser.declareDouble(AnomalyRecord::setProbability, PROBABILITY); + parser.declareDouble(AnomalyRecord::setRecordScore, RECORD_SCORE); + parser.declareDouble(AnomalyRecord::setInitialRecordScore, INITIAL_RECORD_SCORE); + parser.declareInt(AnomalyRecord::setDetectorIndex, Detector.DETECTOR_INDEX); + parser.declareBoolean(AnomalyRecord::setInterim, Result.IS_INTERIM); + parser.declareString(AnomalyRecord::setByFieldName, BY_FIELD_NAME); + parser.declareString(AnomalyRecord::setByFieldValue, BY_FIELD_VALUE); + parser.declareString(AnomalyRecord::setCorrelatedByFieldValue, CORRELATED_BY_FIELD_VALUE); + parser.declareString(AnomalyRecord::setPartitionFieldName, PARTITION_FIELD_NAME); + parser.declareString(AnomalyRecord::setPartitionFieldValue, PARTITION_FIELD_VALUE); + parser.declareString(AnomalyRecord::setFunction, FUNCTION); + parser.declareString(AnomalyRecord::setFunctionDescription, FUNCTION_DESCRIPTION); + parser.declareDoubleArray(AnomalyRecord::setTypical, TYPICAL); + parser.declareDoubleArray(AnomalyRecord::setActual, ACTUAL); + parser.declareString(AnomalyRecord::setFieldName, FIELD_NAME); + parser.declareString(AnomalyRecord::setOverFieldName, OVER_FIELD_NAME); + parser.declareString(AnomalyRecord::setOverFieldValue, OVER_FIELD_VALUE); + parser.declareObjectArray(AnomalyRecord::setCauses, ignoreUnknownFields ? AnomalyCause.LENIENT_PARSER : AnomalyCause.STRICT_PARSER, + CAUSES); + parser.declareObjectArray(AnomalyRecord::setInfluencers, ignoreUnknownFields ? Influence.LENIENT_PARSER : Influence.STRICT_PARSER, + INFLUENCERS); + + return parser; } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java index 2dc9110ac30..7c0ab825c26 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Bucket.java @@ -46,9 +46,6 @@ public class Bucket implements ToXContentObject, Writeable { public static final ParseField PARTITION_SCORES = new ParseField("partition_scores"); public static final ParseField SCHEDULED_EVENTS = new ParseField("scheduled_events"); - // Only exists for backwards compatibility; no longer added to mappings - private static final ParseField RECORD_COUNT = new ParseField("record_count"); - // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("buckets"); @@ -58,12 +55,15 @@ public class Bucket implements ToXContentObject, Writeable { public static final String RESULT_TYPE_VALUE = "bucket"; public static final ParseField RESULT_TYPE_FIELD = new ParseField(RESULT_TYPE_VALUE); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new Bucket((String) a[0], (Date) a[1], (long) a[2])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, ignoreUnknownFields, + a -> new Bucket((String) a[0], (Date) a[1], (long) a[2])); + + parser.declareString(ConstructingObjectParser.constructorArg(), JOB_ID); + parser.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -72,19 +72,22 @@ public class Bucket implements ToXContentObject, Writeable { throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + Result.TIMESTAMP.getPreferredName() + "]"); }, Result.TIMESTAMP, ValueType.VALUE); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); - PARSER.declareDouble(Bucket::setAnomalyScore, ANOMALY_SCORE); - PARSER.declareDouble(Bucket::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); - PARSER.declareBoolean(Bucket::setInterim, Result.IS_INTERIM); - PARSER.declareLong(Bucket::setEventCount, EVENT_COUNT); - PARSER.declareObjectArray(Bucket::setRecords, AnomalyRecord.PARSER, RECORDS); - PARSER.declareObjectArray(Bucket::setBucketInfluencers, BucketInfluencer.PARSER, BUCKET_INFLUENCERS); - PARSER.declareLong(Bucket::setProcessingTimeMs, PROCESSING_TIME_MS); - PARSER.declareObjectArray(Bucket::setPartitionScores, PartitionScore.PARSER, PARTITION_SCORES); - PARSER.declareString((bucket, s) -> {}, Result.RESULT_TYPE); - // For bwc with 5.4 - PARSER.declareInt((bucket, recordCount) -> {}, RECORD_COUNT); - PARSER.declareStringArray(Bucket::setScheduledEvents, SCHEDULED_EVENTS); + parser.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + parser.declareDouble(Bucket::setAnomalyScore, ANOMALY_SCORE); + parser.declareDouble(Bucket::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + parser.declareBoolean(Bucket::setInterim, Result.IS_INTERIM); + parser.declareLong(Bucket::setEventCount, EVENT_COUNT); + parser.declareObjectArray(Bucket::setRecords, ignoreUnknownFields ? AnomalyRecord.LENIENT_PARSER : AnomalyRecord.STRICT_PARSER, + RECORDS); + parser.declareObjectArray(Bucket::setBucketInfluencers, ignoreUnknownFields ? + BucketInfluencer.LENIENT_PARSER : BucketInfluencer.STRICT_PARSER, BUCKET_INFLUENCERS); + parser.declareLong(Bucket::setProcessingTimeMs, PROCESSING_TIME_MS); + parser.declareObjectArray(Bucket::setPartitionScores, ignoreUnknownFields ? + PartitionScore.LENIENT_PARSER : PartitionScore.STRICT_PARSER, PARTITION_SCORES); + parser.declareString((bucket, s) -> {}, Result.RESULT_TYPE); + parser.declareStringArray(Bucket::setScheduledEvents, SCHEDULED_EVENTS); + + return parser; } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java index fa8236d2352..fe4b670719c 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencer.java @@ -39,20 +39,21 @@ public class BucketInfluencer implements ToXContentObject, Writeable { public static final ParseField RAW_ANOMALY_SCORE = new ParseField("raw_anomaly_score"); public static final ParseField PROBABILITY = new ParseField("probability"); public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); - public static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); /** * The influencer field name used for time influencers */ public static final String BUCKET_TIME = "bucket_time"; - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(RESULT_TYPE_FIELD.getPreferredName(), a -> new BucketInfluencer((String) a[0], - (Date) a[1], (long) a[2])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_FIELD.getPreferredName(), + ignoreUnknownFields, a -> new BucketInfluencer((String) a[0], (Date) a[1], (long) a[2])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -61,16 +62,16 @@ public class BucketInfluencer implements ToXContentObject, Writeable { throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + Result.TIMESTAMP.getPreferredName() + "]"); }, Result.TIMESTAMP, ValueType.VALUE); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); - PARSER.declareString((bucketInfluencer, s) -> {}, Result.RESULT_TYPE); - PARSER.declareString(BucketInfluencer::setInfluencerFieldName, INFLUENCER_FIELD_NAME); - PARSER.declareDouble(BucketInfluencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); - PARSER.declareDouble(BucketInfluencer::setAnomalyScore, ANOMALY_SCORE); - PARSER.declareDouble(BucketInfluencer::setRawAnomalyScore, RAW_ANOMALY_SCORE); - PARSER.declareDouble(BucketInfluencer::setProbability, PROBABILITY); - PARSER.declareBoolean(BucketInfluencer::setIsInterim, Result.IS_INTERIM); - // For bwc with 5.4 - PARSER.declareInt((bucketInfluencer, sequenceNum) -> {}, SEQUENCE_NUM); + parser.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + parser.declareString((bucketInfluencer, s) -> {}, Result.RESULT_TYPE); + parser.declareString(BucketInfluencer::setInfluencerFieldName, INFLUENCER_FIELD_NAME); + parser.declareDouble(BucketInfluencer::setInitialAnomalyScore, INITIAL_ANOMALY_SCORE); + parser.declareDouble(BucketInfluencer::setAnomalyScore, ANOMALY_SCORE); + parser.declareDouble(BucketInfluencer::setRawAnomalyScore, RAW_ANOMALY_SCORE); + parser.declareDouble(BucketInfluencer::setProbability, PROBABILITY); + parser.declareBoolean(BucketInfluencer::setIsInterim, Result.IS_INTERIM); + + return parser; } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java index 850834d7037..98c38241856 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/CategoryDefinition.java @@ -38,16 +38,21 @@ public class CategoryDefinition implements ToXContentObject, Writeable { // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("categories"); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(TYPE.getPreferredName(), a -> new CategoryDefinition((String) a[0])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareLong(CategoryDefinition::setCategoryId, CATEGORY_ID); - PARSER.declareString(CategoryDefinition::setTerms, TERMS); - PARSER.declareString(CategoryDefinition::setRegex, REGEX); - PARSER.declareLong(CategoryDefinition::setMaxMatchingLength, MAX_MATCHING_LENGTH); - PARSER.declareStringArray(CategoryDefinition::setExamples, EXAMPLES); + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(TYPE.getPreferredName(), + ignoreUnknownFields, a -> new CategoryDefinition((String) a[0])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareLong(CategoryDefinition::setCategoryId, CATEGORY_ID); + parser.declareString(CategoryDefinition::setTerms, TERMS); + parser.declareString(CategoryDefinition::setRegex, REGEX); + parser.declareLong(CategoryDefinition::setMaxMatchingLength, MAX_MATCHING_LENGTH); + parser.declareStringArray(CategoryDefinition::setExamples, EXAMPLES); + + return parser; } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java index 634c8455c31..47f6769a07f 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Forecast.java @@ -43,14 +43,15 @@ public class Forecast implements ToXContentObject, Writeable { public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> - new Forecast((String) a[0], (String) a[1], (Date) a[2], (long) a[3], (int) a[4])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareString(ConstructingObjectParser.constructorArg(), FORECAST_ID); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, ignoreUnknownFields, + a -> new Forecast((String) a[0], (String) a[1], (Date) a[2], (long) a[3], (int) a[4])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareString(ConstructingObjectParser.constructorArg(), FORECAST_ID); + parser.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -59,17 +60,19 @@ public class Forecast implements ToXContentObject, Writeable { throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + Result.TIMESTAMP.getPreferredName() + "]"); }, Result.TIMESTAMP, ValueType.VALUE); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); - PARSER.declareInt(ConstructingObjectParser.constructorArg(), DETECTOR_INDEX); - PARSER.declareString((modelForecast, s) -> {}, Result.RESULT_TYPE); - PARSER.declareString(Forecast::setPartitionFieldName, PARTITION_FIELD_NAME); - PARSER.declareString(Forecast::setPartitionFieldValue, PARTITION_FIELD_VALUE); - PARSER.declareString(Forecast::setByFieldName, BY_FIELD_NAME); - PARSER.declareString(Forecast::setByFieldValue, BY_FIELD_VALUE); - PARSER.declareString(Forecast::setModelFeature, MODEL_FEATURE); - PARSER.declareDouble(Forecast::setForecastLower, FORECAST_LOWER); - PARSER.declareDouble(Forecast::setForecastUpper, FORECAST_UPPER); - PARSER.declareDouble(Forecast::setForecastPrediction, FORECAST_PREDICTION); + parser.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + parser.declareInt(ConstructingObjectParser.constructorArg(), DETECTOR_INDEX); + parser.declareString((modelForecast, s) -> {}, Result.RESULT_TYPE); + parser.declareString(Forecast::setPartitionFieldName, PARTITION_FIELD_NAME); + parser.declareString(Forecast::setPartitionFieldValue, PARTITION_FIELD_VALUE); + parser.declareString(Forecast::setByFieldName, BY_FIELD_NAME); + parser.declareString(Forecast::setByFieldValue, BY_FIELD_VALUE); + parser.declareString(Forecast::setModelFeature, MODEL_FEATURE); + parser.declareDouble(Forecast::setForecastLower, FORECAST_LOWER); + parser.declareDouble(Forecast::setForecastUpper, FORECAST_UPPER); + parser.declareDouble(Forecast::setForecastPrediction, FORECAST_PREDICTION); + + return parser; } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ForecastRequestStats.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ForecastRequestStats.java index 7e92b134784..a9daa78a636 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ForecastRequestStats.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ForecastRequestStats.java @@ -47,30 +47,35 @@ public class ForecastRequestStats implements ToXContentObject, Writeable { public static final ParseField STATUS = new ParseField("forecast_status"); public static final ParseField MEMORY_USAGE = new ParseField("forecast_memory_bytes"); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> new ForecastRequestStats((String) a[0], (String) a[1])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareString(ConstructingObjectParser.constructorArg(), FORECAST_ID); + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, ignoreUnknownFields, + a -> new ForecastRequestStats((String) a[0], (String) a[1])); - PARSER.declareString((modelForecastRequestStats, s) -> {}, Result.RESULT_TYPE); - PARSER.declareLong(ForecastRequestStats::setRecordCount, PROCESSED_RECORD_COUNT); - PARSER.declareStringArray(ForecastRequestStats::setMessages, MESSAGES); - PARSER.declareField(ForecastRequestStats::setTimeStamp, + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareString(ConstructingObjectParser.constructorArg(), FORECAST_ID); + + parser.declareString((modelForecastRequestStats, s) -> {}, Result.RESULT_TYPE); + parser.declareLong(ForecastRequestStats::setRecordCount, PROCESSED_RECORD_COUNT); + parser.declareStringArray(ForecastRequestStats::setMessages, MESSAGES); + parser.declareField(ForecastRequestStats::setTimeStamp, p -> Instant.ofEpochMilli(p.longValue()), Result.TIMESTAMP, ValueType.LONG); - PARSER.declareField(ForecastRequestStats::setStartTime, + parser.declareField(ForecastRequestStats::setStartTime, p -> Instant.ofEpochMilli(p.longValue()), START_TIME, ValueType.LONG); - PARSER.declareField(ForecastRequestStats::setEndTime, + parser.declareField(ForecastRequestStats::setEndTime, p -> Instant.ofEpochMilli(p.longValue()), END_TIME, ValueType.LONG); - PARSER.declareField(ForecastRequestStats::setCreateTime, + parser.declareField(ForecastRequestStats::setCreateTime, p -> Instant.ofEpochMilli(p.longValue()), CREATE_TIME, ValueType.LONG); - PARSER.declareField(ForecastRequestStats::setExpiryTime, + parser.declareField(ForecastRequestStats::setExpiryTime, p -> Instant.ofEpochMilli(p.longValue()), EXPIRY_TIME, ValueType.LONG); - PARSER.declareDouble(ForecastRequestStats::setProgress, PROGRESS); - PARSER.declareLong(ForecastRequestStats::setProcessingTime, PROCESSING_TIME_MS); - PARSER.declareField(ForecastRequestStats::setStatus, p -> ForecastRequestStatus.fromString(p.text()), STATUS, ValueType.STRING); - PARSER.declareLong(ForecastRequestStats::setMemoryUsage, MEMORY_USAGE); + parser.declareDouble(ForecastRequestStats::setProgress, PROGRESS); + parser.declareLong(ForecastRequestStats::setProcessingTime, PROCESSING_TIME_MS); + parser.declareField(ForecastRequestStats::setStatus, p -> ForecastRequestStatus.fromString(p.text()), STATUS, ValueType.STRING); + parser.declareLong(ForecastRequestStats::setMemoryUsage, MEMORY_USAGE); + + return parser; } public enum ForecastRequestStatus implements Writeable { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influence.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influence.java index 7b7c012746f..ab6ca54f3a1 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influence.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influence.java @@ -30,15 +30,18 @@ public class Influence implements ToXContentObject, Writeable { public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name"); public static final ParseField INFLUENCER_FIELD_VALUES = new ParseField("influencer_field_values"); - @SuppressWarnings("unchecked") - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - INFLUENCER.getPreferredName(), a -> new Influence((String) a[0], (List) a[1])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); - PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUES); + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(INFLUENCER.getPreferredName(), + ignoreUnknownFields, a -> new Influence((String) a[0], (List) a[1])); + + parser.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); + parser.declareStringArray(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUES); + + return parser; } - private String field; private List fieldValues; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java index fcb512a3e2a..d91128730dd 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/Influencer.java @@ -34,7 +34,6 @@ public class Influencer implements ToXContentObject, Writeable { * ThrottlerField names */ public static final ParseField PROBABILITY = new ParseField("probability"); - public static final ParseField SEQUENCE_NUM = new ParseField("sequence_num"); public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); public static final ParseField INFLUENCER_FIELD_NAME = new ParseField("influencer_field_name"); public static final ParseField INFLUENCER_FIELD_VALUE = new ParseField("influencer_field_value"); @@ -44,15 +43,16 @@ public class Influencer implements ToXContentObject, Writeable { // Used for QueryPage public static final ParseField RESULTS_FIELD = new ParseField("influencers"); - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - RESULT_TYPE_FIELD.getPreferredName(), true, a -> new Influencer((String) a[0], (String) a[1], (String) a[2], - (Date) a[3], (long) a[4])); + // Influencers contain data fields, thus we always parse them leniently + public static final ConstructingObjectParser LENIENT_PARSER = new ConstructingObjectParser<>( + RESULT_TYPE_FIELD.getPreferredName(), true, + a -> new Influencer((String) a[0], (String) a[1], (String) a[2], (Date) a[3], (long) a[4])); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); - PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUE); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + LENIENT_PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + LENIENT_PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_NAME); + LENIENT_PARSER.declareString(ConstructingObjectParser.constructorArg(), INFLUENCER_FIELD_VALUE); + LENIENT_PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -61,14 +61,12 @@ public class Influencer implements ToXContentObject, Writeable { throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + Result.TIMESTAMP.getPreferredName() + "]"); }, Result.TIMESTAMP, ValueType.VALUE); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); - PARSER.declareString((influencer, s) -> {}, Result.RESULT_TYPE); - PARSER.declareDouble(Influencer::setProbability, PROBABILITY); - PARSER.declareDouble(Influencer::setInfluencerScore, INFLUENCER_SCORE); - PARSER.declareDouble(Influencer::setInitialInfluencerScore, INITIAL_INFLUENCER_SCORE); - PARSER.declareBoolean(Influencer::setInterim, Result.IS_INTERIM); - // For bwc with 5.4 - PARSER.declareInt((influencer, sequenceNum) -> {}, SEQUENCE_NUM); + LENIENT_PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + LENIENT_PARSER.declareString((influencer, s) -> {}, Result.RESULT_TYPE); + LENIENT_PARSER.declareDouble(Influencer::setProbability, PROBABILITY); + LENIENT_PARSER.declareDouble(Influencer::setInfluencerScore, INFLUENCER_SCORE); + LENIENT_PARSER.declareDouble(Influencer::setInitialInfluencerScore, INITIAL_INFLUENCER_SCORE); + LENIENT_PARSER.declareBoolean(Influencer::setInterim, Result.IS_INTERIM); } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java index 5834102fcde..c331d8b0437 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ModelPlot.java @@ -46,13 +46,15 @@ public class ModelPlot implements ToXContentObject, Writeable { public static final ParseField BUCKET_SPAN = new ParseField("bucket_span"); public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); - public static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>(RESULT_TYPE_VALUE, a -> - new ModelPlot((String) a[0], (Date) a[1], (long) a[2], (int) a[3])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(RESULT_TYPE_VALUE, ignoreUnknownFields, + a -> new ModelPlot((String) a[0], (Date) a[1], (long) a[2], (int) a[3])); + + parser.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + parser.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == Token.VALUE_NUMBER) { return new Date(p.longValue()); } else if (p.currentToken() == Token.VALUE_STRING) { @@ -61,20 +63,22 @@ public class ModelPlot implements ToXContentObject, Writeable { throw new IllegalArgumentException("unexpected token [" + p.currentToken() + "] for [" + Result.TIMESTAMP.getPreferredName() + "]"); }, Result.TIMESTAMP, ValueType.VALUE); - PARSER.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); - PARSER.declareInt(ConstructingObjectParser.constructorArg(), DETECTOR_INDEX); - PARSER.declareString((modelPlot, s) -> {}, Result.RESULT_TYPE); - PARSER.declareString(ModelPlot::setPartitionFieldName, PARTITION_FIELD_NAME); - PARSER.declareString(ModelPlot::setPartitionFieldValue, PARTITION_FIELD_VALUE); - PARSER.declareString(ModelPlot::setOverFieldName, OVER_FIELD_NAME); - PARSER.declareString(ModelPlot::setOverFieldValue, OVER_FIELD_VALUE); - PARSER.declareString(ModelPlot::setByFieldName, BY_FIELD_NAME); - PARSER.declareString(ModelPlot::setByFieldValue, BY_FIELD_VALUE); - PARSER.declareString(ModelPlot::setModelFeature, MODEL_FEATURE); - PARSER.declareDouble(ModelPlot::setModelLower, MODEL_LOWER); - PARSER.declareDouble(ModelPlot::setModelUpper, MODEL_UPPER); - PARSER.declareDouble(ModelPlot::setModelMedian, MODEL_MEDIAN); - PARSER.declareDouble(ModelPlot::setActual, ACTUAL); + parser.declareLong(ConstructingObjectParser.constructorArg(), BUCKET_SPAN); + parser.declareInt(ConstructingObjectParser.constructorArg(), DETECTOR_INDEX); + parser.declareString((modelPlot, s) -> {}, Result.RESULT_TYPE); + parser.declareString(ModelPlot::setPartitionFieldName, PARTITION_FIELD_NAME); + parser.declareString(ModelPlot::setPartitionFieldValue, PARTITION_FIELD_VALUE); + parser.declareString(ModelPlot::setOverFieldName, OVER_FIELD_NAME); + parser.declareString(ModelPlot::setOverFieldValue, OVER_FIELD_VALUE); + parser.declareString(ModelPlot::setByFieldName, BY_FIELD_NAME); + parser.declareString(ModelPlot::setByFieldValue, BY_FIELD_VALUE); + parser.declareString(ModelPlot::setModelFeature, MODEL_FEATURE); + parser.declareDouble(ModelPlot::setModelLower, MODEL_LOWER); + parser.declareDouble(ModelPlot::setModelUpper, MODEL_UPPER); + parser.declareDouble(ModelPlot::setModelMedian, MODEL_MEDIAN); + parser.declareDouble(ModelPlot::setActual, ACTUAL); + + return parser; } private final String jobId; diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java index 30afef693a6..3d0acc8fde6 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/PartitionScore.java @@ -25,16 +25,20 @@ public class PartitionScore implements ToXContentObject, Writeable { private double recordScore; private double probability; - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - PARTITION_SCORE.getPreferredName(), a -> new PartitionScore((String) a[0], (String) a[1], (Double) a[2], (Double) a[3], - (Double) a[4])); + public static final ConstructingObjectParser STRICT_PARSER = createParser(false); + public static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_NAME); - PARSER.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); - PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.INITIAL_RECORD_SCORE); - PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.RECORD_SCORE); - PARSER.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.PROBABILITY); + private static ConstructingObjectParser createParser(boolean ignoreUnknownFields) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(PARTITION_SCORE.getPreferredName(), + ignoreUnknownFields, a -> new PartitionScore((String) a[0], (String) a[1], (Double) a[2], (Double) a[3], (Double) a[4])); + + parser.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_NAME); + parser.declareString(ConstructingObjectParser.constructorArg(), AnomalyRecord.PARTITION_FIELD_VALUE); + parser.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.INITIAL_RECORD_SCORE); + parser.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.RECORD_SCORE); + parser.declareDouble(ConstructingObjectParser.constructorArg(), AnomalyRecord.PROBABILITY); + + return parser; } public PartitionScore(String fieldName, String fieldValue, double initialRecordScore, double recordScore, double probability) { diff --git a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java index d423342b8d9..850d89d0a72 100644 --- a/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java +++ b/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java @@ -30,8 +30,7 @@ public class AuditMessage implements ToXContentObject, Writeable { public static final ParseField TIMESTAMP = new ParseField("timestamp"); public static final ParseField NODE_NAME = new ParseField("node_name"); - public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), - AuditMessage::new); + public static final ObjectParser PARSER = new ObjectParser<>(TYPE.getPreferredName(), true, AuditMessage::new); static { PARSER.declareString(AuditMessage::setJobId, Job.ID); diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java index 4260fbfd46e..b51115a47c3 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/CalendarTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.ml.calendars; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.config.JobTests; @@ -15,6 +16,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class CalendarTests extends AbstractSerializingTestCase { @@ -48,7 +50,7 @@ public class CalendarTests extends AbstractSerializingTestCase { @Override protected Calendar doParseInstance(XContentParser parser) throws IOException { - return Calendar.PARSER.apply(parser, null).build(); + return Calendar.STRICT_PARSER.apply(parser, null).build(); } public void testNullId() { @@ -59,4 +61,21 @@ public class CalendarTests extends AbstractSerializingTestCase { public void testDocumentId() { assertThat(Calendar.documentId("foo"), equalTo("calendar_foo")); } + + public void testStrictParser() throws IOException { + String json = "{\"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> Calendar.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + Calendar.LENIENT_PARSER.apply(parser, null); + } + } } \ No newline at end of file diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java index d6cca7a61e1..f98eb9d5dce 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.config.Connective; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; @@ -47,7 +48,7 @@ public class ScheduledEventTests extends AbstractSerializingTestCase ScheduledEvent.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + ScheduledEvent.LENIENT_PARSER.apply(parser, null); + } + } } \ No newline at end of file diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java index 42102b540f9..08efc1c883f 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/MlFilterTests.java @@ -7,12 +7,16 @@ package org.elasticsearch.xpack.core.ml.job.config; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class MlFilterTests extends AbstractSerializingTestCase { @@ -38,7 +42,7 @@ public class MlFilterTests extends AbstractSerializingTestCase { @Override protected MlFilter doParseInstance(XContentParser parser) { - return MlFilter.PARSER.apply(parser, null).build(); + return MlFilter.STRICT_PARSER.apply(parser, null).build(); } public void testNullId() { @@ -55,4 +59,21 @@ public class MlFilterTests extends AbstractSerializingTestCase { public void testDocumentId() { assertThat(MlFilter.documentId("foo"), equalTo("filter_foo")); } + + public void testStrictParser() throws IOException { + String json = "{\"filter_id\":\"filter_1\", \"items\": [], \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> MlFilter.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"filter_id\":\"filter_1\", \"items\": [], \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + MlFilter.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java index fbe8e769085..e66fea90f04 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java @@ -8,12 +8,15 @@ package org.elasticsearch.xpack.core.ml.job.process.autodetect.state; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats.MemoryStatus; +import java.io.IOException; import java.util.Date; +import static org.hamcrest.Matchers.containsString; + public class ModelSizeStatsTests extends AbstractSerializingTestCase { public void testDefaultConstructor() { @@ -83,11 +86,28 @@ public class ModelSizeStatsTests extends AbstractSerializingTestCase ModelSizeStats.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + ModelSizeStats.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java index 0ac8a5be5b2..4026be67461 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSnapshotTests.java @@ -8,12 +8,20 @@ package org.elasticsearch.xpack.core.ml.job.process.autodetect.state; import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContent; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; +import java.io.IOException; import java.util.Arrays; import java.util.Date; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class ModelSnapshotTests extends AbstractSerializingTestCase { @@ -171,7 +179,7 @@ public class ModelSnapshotTests extends AbstractSerializingTestCase ModelSnapshot.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + ModelSnapshot.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java index 93bddbed0ff..84c1a161f1e 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/QuantilesTests.java @@ -8,10 +8,14 @@ package org.elasticsearch.xpack.core.ml.job.process.autodetect.state; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; +import java.io.IOException; import java.util.Date; +import static org.hamcrest.Matchers.containsString; + public class QuantilesTests extends AbstractSerializingTestCase { public void testEquals_GivenSameObject() { @@ -78,6 +82,23 @@ public class QuantilesTests extends AbstractSerializingTestCase { @Override protected Quantiles doParseInstance(XContentParser parser) { - return Quantiles.PARSER.apply(parser, null); + return Quantiles.STRICT_PARSER.apply(parser, null); + } + + public void testStrictParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123456789, \"quantile_state\":\"...\", \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> Quantiles.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123456789, \"quantile_state\":\"...\", \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + Quantiles.LENIENT_PARSER.apply(parser, null); + } } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java index 628239d6f2b..033392eddce 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyCauseTests.java @@ -7,13 +7,15 @@ package org.elasticsearch.xpack.core.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.ml.job.results.AnomalyCause; -import org.elasticsearch.xpack.core.ml.job.results.Influence; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.containsString; + public class AnomalyCauseTests extends AbstractSerializingTestCase { @Override @@ -91,7 +93,23 @@ public class AnomalyCauseTests extends AbstractSerializingTestCase @Override protected AnomalyCause doParseInstance(XContentParser parser) { - return AnomalyCause.PARSER.apply(parser, null); + return AnomalyCause.STRICT_PARSER.apply(parser, null); } + public void testStrictParser() throws IOException { + String json = "{\"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> AnomalyCause.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + AnomalyCause.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java index 5b6060f1f31..1db8a277878 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/AnomalyRecordTests.java @@ -5,14 +5,13 @@ */ package org.elasticsearch.xpack.core.ml.job.results; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; @@ -24,6 +23,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import static org.hamcrest.Matchers.containsString; + public class AnomalyRecordTests extends AbstractSerializingTestCase { @Override @@ -86,7 +87,7 @@ public class AnomalyRecordTests extends AbstractSerializingTestCase AnomalyRecord.STRICT_PARSER.apply(parser, null)); + assertThat(e.getCause().getMessage(), containsString("[anomaly_cause] unknown field [cause_foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123544456, \"bucket_span\": 3600, \"foo\":\"bar\"," + + " \"causes\":[{\"cause_foo\":\"bar\"}]}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + AnomalyRecord.LENIENT_PARSER.apply(parser, null); + } } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java index 0905dc7d9ad..8c56c96c096 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/BucketInfluencerTests.java @@ -6,16 +6,15 @@ package org.elasticsearch.xpack.core.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; import java.util.Date; +import static org.hamcrest.Matchers.containsString; + public class BucketInfluencerTests extends AbstractSerializingTestCase { @Override @@ -50,7 +49,7 @@ public class BucketInfluencerTests extends AbstractSerializingTestCase BucketInfluencer.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } } + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123544456, \"bucket_span\": 3600, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + BucketInfluencer.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java index b3d19014696..7b2da1d7955 100644 --- a/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java +++ b/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/results/InfluencerTests.java @@ -7,12 +7,10 @@ package org.elasticsearch.xpack.core.ml.job.results; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import java.io.IOException; @@ -42,7 +40,7 @@ public class InfluencerTests extends AbstractSerializingTestCase { @Override protected Influencer doParseInstance(XContentParser parser) { - return Influencer.PARSER.apply(parser, null); + return Influencer.LENIENT_PARSER.apply(parser, null); } public void testToXContentIncludesNameValueField() throws IOException { @@ -70,15 +68,11 @@ public class InfluencerTests extends AbstractSerializingTestCase { assertEquals("job-foo_influencer_1000_300_host_" + valueHash + "_" + influencerFieldValue.length(), influencer.getId()); } - public void testParsingv54WithSequenceNumField() throws IOException { - Influencer influencer = createTestInstance(); - XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); - builder.startObject(); - builder.field(Influencer.SEQUENCE_NUM.getPreferredName(), 1); - influencer.innerToXContent(builder, ToXContent.EMPTY_PARAMS); - builder.endObject(); - XContentParser parser = createParser(builder); - Influencer serialised = doParseInstance(parser); - assertEquals(influencer, serialised); + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123544456, \"bucket_span\": 3600," + + "\"influencer_field_name\":\"foo_1\", \"influencer_field_value\": \"foo_2\", \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + Influencer.LENIENT_PARSER.apply(parser, null); + } } } diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java index 08b930dfc73..4264fa2fc2f 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportGetFiltersAction.java @@ -87,7 +87,7 @@ public class TransportGetFiltersAction extends HandledTransportAction(Collections.singletonList(filter), 1, MlFilter.RESULTS_FIELD); GetFiltersAction.Response filterResponse = new GetFiltersAction.Response(responseBody); @@ -128,7 +128,7 @@ public class TransportGetFiltersAction extends HandledTransportAction { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)).createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - Bucket bucket = Bucket.PARSER.apply(parser, null); + Bucket bucket = Bucket.LENIENT_PARSER.apply(parser, null); return new Result<>(hit.getIndex(), bucket); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse bucket", e); diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java index 3ccd8565fbc..d084325350f 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedInfluencersIterator.java @@ -31,7 +31,7 @@ class BatchedInfluencersIterator extends BatchedResultsIterator { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)).createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - Influencer influencer = Influencer.PARSER.apply(parser, null); + Influencer influencer = Influencer.LENIENT_PARSER.apply(parser, null); return new Result<>(hit.getIndex(), influencer); } catch (IOException e) { throw new ElasticsearchParseException("failed to parser influencer", e); diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java index 6bff80d3db1..c0940dfd5aa 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/BatchedRecordsIterator.java @@ -32,7 +32,7 @@ class BatchedRecordsIterator extends BatchedResultsIterator { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)).createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)){ - AnomalyRecord record = AnomalyRecord.PARSER.apply(parser, null); + AnomalyRecord record = AnomalyRecord.LENIENT_PARSER.apply(parser, null); return new Result<>(hit.getIndex(), record); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse record", e); diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index 3f34b3f13db..58fd9ea58f9 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -82,8 +82,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; -import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; -import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; @@ -100,6 +98,8 @@ import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.ml.utils.MlIndicesUtils; import org.elasticsearch.xpack.core.security.support.Exceptions; +import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import java.io.IOException; import java.io.InputStream; @@ -459,15 +459,15 @@ public class JobProvider { if (DataCounts.documentId(jobId).equals(hitId)) { paramsBuilder.setDataCounts(parseSearchHit(hit, DataCounts.PARSER, errorHandler)); } else if (hitId.startsWith(ModelSizeStats.documentIdPrefix(jobId))) { - ModelSizeStats.Builder modelSizeStats = parseSearchHit(hit, ModelSizeStats.PARSER, errorHandler); + ModelSizeStats.Builder modelSizeStats = parseSearchHit(hit, ModelSizeStats.LENIENT_PARSER, errorHandler); paramsBuilder.setModelSizeStats(modelSizeStats == null ? null : modelSizeStats.build()); } else if (hitId.startsWith(ModelSnapshot.documentIdPrefix(jobId))) { - ModelSnapshot.Builder modelSnapshot = parseSearchHit(hit, ModelSnapshot.PARSER, errorHandler); + ModelSnapshot.Builder modelSnapshot = parseSearchHit(hit, ModelSnapshot.LENIENT_PARSER, errorHandler); paramsBuilder.setModelSnapshot(modelSnapshot == null ? null : modelSnapshot.build()); } else if (Quantiles.documentId(jobId).equals(hit.getId())) { - paramsBuilder.setQuantiles(parseSearchHit(hit, Quantiles.PARSER, errorHandler)); + paramsBuilder.setQuantiles(parseSearchHit(hit, Quantiles.LENIENT_PARSER, errorHandler)); } else if (hitId.startsWith(MlFilter.DOCUMENT_ID_PREFIX)) { - paramsBuilder.addFilter(parseSearchHit(hit, MlFilter.PARSER, errorHandler).build()); + paramsBuilder.addFilter(parseSearchHit(hit, MlFilter.LENIENT_PARSER, errorHandler).build()); } else { errorHandler.accept(new IllegalStateException("Unexpected Id [" + hitId + "]")); } @@ -530,7 +530,7 @@ public class JobProvider { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - Bucket bucket = Bucket.PARSER.apply(parser, null); + Bucket bucket = Bucket.LENIENT_PARSER.apply(parser, null); results.add(bucket); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse bucket", e); @@ -662,7 +662,7 @@ public class JobProvider { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - CategoryDefinition categoryDefinition = CategoryDefinition.PARSER.apply(parser, null); + CategoryDefinition categoryDefinition = CategoryDefinition.LENIENT_PARSER.apply(parser, null); results.add(categoryDefinition); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse category definition", e); @@ -697,7 +697,7 @@ public class JobProvider { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - results.add(AnomalyRecord.PARSER.apply(parser, null)); + results.add(AnomalyRecord.LENIENT_PARSER.apply(parser, null)); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse records", e); } @@ -746,7 +746,7 @@ public class JobProvider { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - influencers.add(Influencer.PARSER.apply(parser, null)); + influencers.add(Influencer.LENIENT_PARSER.apply(parser, null)); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse influencer", e); } @@ -779,7 +779,7 @@ public class JobProvider { } String resultsIndex = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); SearchRequestBuilder search = createDocIdSearch(resultsIndex, ModelSnapshot.documentId(jobId, modelSnapshotId)); - searchSingleResult(jobId, ModelSnapshot.TYPE.getPreferredName(), search, ModelSnapshot.PARSER, + searchSingleResult(jobId, ModelSnapshot.TYPE.getPreferredName(), search, ModelSnapshot.LENIENT_PARSER, result -> handler.accept(result.result == null ? null : new Result(result.index, result.result.build())), errorHandler, () -> null); } @@ -891,7 +891,7 @@ public class JobProvider { try (InputStream stream = source.streamInput(); XContentParser parser = XContentFactory.xContent(XContentHelper.xContentType(source)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - ModelPlot modelPlot = ModelPlot.PARSER.apply(parser, null); + ModelPlot modelPlot = ModelPlot.LENIENT_PARSER.apply(parser, null); results.add(modelPlot); } catch (IOException e) { throw new ElasticsearchParseException("failed to parse modelPlot", e); @@ -909,7 +909,7 @@ public class JobProvider { String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); searchSingleResult(jobId, ModelSizeStats.RESULT_TYPE_VALUE, createLatestModelSizeStatsSearch(indexName), - ModelSizeStats.PARSER, + ModelSizeStats.LENIENT_PARSER, result -> handler.accept(result.result.build()), errorHandler, () -> new ModelSizeStats.Builder(jobId)); } @@ -1076,7 +1076,7 @@ public class JobProvider { List events = new ArrayList<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { - ScheduledEvent.Builder event = parseSearchHit(hit, ScheduledEvent.PARSER, handler::onFailure); + ScheduledEvent.Builder event = parseSearchHit(hit, ScheduledEvent.LENIENT_PARSER, handler::onFailure); event.eventId(hit.getId()); events.add(event.build()); } @@ -1094,7 +1094,7 @@ public class JobProvider { GetRequest getRequest = new GetRequest(indexName, ElasticsearchMappings.DOC_TYPE, ForecastRequestStats.documentId(jobId, forecastId)); - getResult(jobId, ForecastRequestStats.RESULTS_FIELD.getPreferredName(), getRequest, ForecastRequestStats.PARSER, + getResult(jobId, ForecastRequestStats.RESULTS_FIELD.getPreferredName(), getRequest, ForecastRequestStats.LENIENT_PARSER, result -> handler.accept(result.result), errorHandler, () -> null); } @@ -1159,7 +1159,7 @@ public class JobProvider { List calendars = new ArrayList<>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit : hits) { - calendars.add(parseSearchHit(hit, Calendar.PARSER, listener::onFailure).build()); + calendars.add(parseSearchHit(hit, Calendar.LENIENT_PARSER, listener::onFailure).build()); } listener.onResponse(new QueryPage(calendars, response.getHits().getTotalHits(), @@ -1228,7 +1228,7 @@ public class JobProvider { XContentFactory.xContent(XContentHelper.xContentType(docSource)) .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - Calendar calendar = Calendar.PARSER.apply(parser, null).build(); + Calendar calendar = Calendar.LENIENT_PARSER.apply(parser, null).build(); listener.onResponse(calendar); } } else { diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java index 5512cdd8281..f3afa98b55a 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/results/AutodetectResult.java @@ -42,18 +42,19 @@ public class AutodetectResult implements ToXContentObject, Writeable { (Forecast) a[7], (ForecastRequestStats) a[8], (CategoryDefinition) a[9], (FlushAcknowledgement) a[10])); static { - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Bucket.PARSER, Bucket.RESULT_TYPE_FIELD); - PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), AnomalyRecord.PARSER, AnomalyRecord.RESULTS_FIELD); - PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), Influencer.PARSER, Influencer.RESULTS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Quantiles.PARSER, Quantiles.TYPE); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSnapshot.PARSER, ModelSnapshot.TYPE); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSizeStats.PARSER, + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Bucket.STRICT_PARSER, Bucket.RESULT_TYPE_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), AnomalyRecord.STRICT_PARSER, + AnomalyRecord.RESULTS_FIELD); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), Influencer.LENIENT_PARSER, Influencer.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Quantiles.STRICT_PARSER, Quantiles.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSnapshot.STRICT_PARSER, ModelSnapshot.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelSizeStats.STRICT_PARSER, ModelSizeStats.RESULT_TYPE_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelPlot.PARSER, ModelPlot.RESULTS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Forecast.PARSER, Forecast.RESULTS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ForecastRequestStats.PARSER, + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ModelPlot.STRICT_PARSER, ModelPlot.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), Forecast.STRICT_PARSER, Forecast.RESULTS_FIELD); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), ForecastRequestStats.STRICT_PARSER, ForecastRequestStats.RESULTS_FIELD); - PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), CategoryDefinition.PARSER, CategoryDefinition.TYPE); + PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), CategoryDefinition.STRICT_PARSER, CategoryDefinition.TYPE); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), FlushAcknowledgement.PARSER, FlushAcknowledgement.TYPE); } diff --git a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java index ed3368f7c33..30c49a834be 100644 --- a/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java +++ b/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredForecastsRemover.java @@ -121,7 +121,7 @@ public class ExpiredForecastsRemover implements MlDataRemover { try (InputStream stream = hit.getSourceRef().streamInput(); XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, stream)) { - ForecastRequestStats forecastRequestStats = ForecastRequestStats.PARSER.apply(parser, null); + ForecastRequestStats forecastRequestStats = ForecastRequestStats.LENIENT_PARSER.apply(parser, null); if (forecastRequestStats.getExpiryTime().toEpochMilli() < cutoffEpochMs) { forecastsToDelete.add(forecastRequestStats); } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java index 375e87249df..966501db43f 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/BucketTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecordTests; @@ -14,6 +15,7 @@ import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.elasticsearch.xpack.core.ml.job.results.BucketInfluencer; import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -21,6 +23,7 @@ import java.util.Date; import java.util.List; import java.util.stream.IntStream; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class BucketTests extends AbstractSerializingTestCase { @@ -95,7 +98,7 @@ public class BucketTests extends AbstractSerializingTestCase { @Override protected Bucket doParseInstance(XContentParser parser) { - return Bucket.PARSER.apply(parser, null); + return Bucket.STRICT_PARSER.apply(parser, null); } public void testEquals_GivenDifferentClass() { @@ -297,4 +300,21 @@ public class BucketTests extends AbstractSerializingTestCase { assertThat(copy, equalTo(bucket)); } } + + public void testStrictParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123544456, \"bucket_span\": 3600, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> Bucket.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\": 123544456, \"bucket_span\": 3600, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + Bucket.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java index f6491a7306d..fdaa2850823 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/CategoryDefinitionTests.java @@ -7,11 +7,15 @@ package org.elasticsearch.xpack.ml.job.results; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; +import java.io.IOException; import java.util.Arrays; +import static org.hamcrest.Matchers.containsString; + public class CategoryDefinitionTests extends AbstractSerializingTestCase { public CategoryDefinition createTestInstance(String jobId) { @@ -36,7 +40,7 @@ public class CategoryDefinitionTests extends AbstractSerializingTestCase CategoryDefinition.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + CategoryDefinition.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java index 7b1a387c49e..c5a33b5cff9 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastRequestStatsTests.java @@ -7,14 +7,18 @@ package org.elasticsearch.xpack.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats.ForecastRequestStatus; +import java.io.IOException; import java.time.Instant; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.containsString; + public class ForecastRequestStatsTests extends AbstractSerializingTestCase { @Override @@ -74,6 +78,23 @@ public class ForecastRequestStatsTests extends AbstractSerializingTestCase ForecastRequestStats.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"forecast_id\":\"forecast_1\", \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + ForecastRequestStats.LENIENT_PARSER.apply(parser, null); + } } } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java index 4060b36a549..b1d9f37dcb4 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ForecastTests.java @@ -7,12 +7,16 @@ package org.elasticsearch.xpack.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.results.Forecast; +import java.io.IOException; import java.util.Date; import java.util.Objects; +import static org.hamcrest.Matchers.containsString; + public class ForecastTests extends AbstractSerializingTestCase { @Override @@ -60,7 +64,7 @@ public class ForecastTests extends AbstractSerializingTestCase { @Override protected Forecast doParseInstance(XContentParser parser) { - return Forecast.PARSER.apply(parser, null); + return Forecast.STRICT_PARSER.apply(parser, null); } public void testId() { @@ -86,4 +90,15 @@ public class ForecastTests extends AbstractSerializingTestCase { valuesHash = Objects.hash(byFieldValue, partitionFieldValue); assertEquals("job-foo_model_forecast_222_100_60_2_" + valuesHash + "_" + length, forecast.getId()); } + + public void testStrictParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"forecast_id\":\"forecast_1\", \"timestamp\":12354667, \"bucket_span\": 3600," + + "\"detector_index\":3, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> Forecast.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java index 30931634ddc..ab6bf0d443d 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/InfluenceTests.java @@ -7,12 +7,16 @@ package org.elasticsearch.xpack.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.results.Influence; +import java.io.IOException; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.containsString; + public class InfluenceTests extends AbstractSerializingTestCase { @Override @@ -32,7 +36,23 @@ public class InfluenceTests extends AbstractSerializingTestCase { @Override protected Influence doParseInstance(XContentParser parser) { - return Influence.PARSER.apply(parser, null); + return Influence.STRICT_PARSER.apply(parser, null); } + public void testStrictParser() throws IOException { + String json = "{\"influencer_field_name\":\"influencer_1\", \"influencer_field_values\":[], \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> Influence.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"influencer_field_name\":\"influencer_1\", \"influencer_field_values\":[], \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + Influence.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java index 377e832920d..2a5ceb8363b 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/ModelPlotTests.java @@ -68,7 +68,7 @@ public class ModelPlotTests extends AbstractSerializingTestCase { @Override protected ModelPlot doParseInstance(XContentParser parser) { - return ModelPlot.PARSER.apply(parser, null); + return ModelPlot.STRICT_PARSER.apply(parser, null); } public void testEquals_GivenSameObject() { @@ -243,6 +243,23 @@ public class ModelPlotTests extends AbstractSerializingTestCase { assertEquals("job-foo_model_plot_100_60_33_" + valuesHash + "_" + length, plot.getId()); } + public void testStrictParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\":12354667, \"bucket_span\": 3600, \"detector_index\":3, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> ModelPlot.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"job_id\":\"job_1\", \"timestamp\":12354667, \"bucket_span\": 3600, \"detector_index\":3, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + ModelPlot.LENIENT_PARSER.apply(parser, null); + } + } + private ModelPlot createFullyPopulated() { ModelPlot modelPlot = new ModelPlot("foo", new Date(12345678L), 360L, 22); modelPlot.setByFieldName("by"); @@ -256,5 +273,4 @@ public class ModelPlotTests extends AbstractSerializingTestCase { modelPlot.setActual(100.0); return modelPlot; } - } diff --git a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java index 9f2a705c579..74c3934c532 100644 --- a/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java +++ b/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/results/PartitionScoreTests.java @@ -7,9 +7,14 @@ package org.elasticsearch.xpack.ml.job.results; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.job.results.PartitionScore; +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; + public class PartitionScoreTests extends AbstractSerializingTestCase { @Override @@ -25,7 +30,25 @@ public class PartitionScoreTests extends AbstractSerializingTestCase PartitionScore.STRICT_PARSER.apply(parser, null)); + + assertThat(e.getMessage(), containsString("unknown field [foo]")); + } + } + + public void testLenientParser() throws IOException { + String json = "{\"partition_field_name\":\"field_1\", \"partition_field_value\":\"x\", \"initial_record_score\": 3," + + " \"record_score\": 3, \"probability\": 0.001, \"foo\":\"bar\"}"; + try (XContentParser parser = createParser(JsonXContent.jsonXContent, json)) { + PartitionScore.LENIENT_PARSER.apply(parser, null); + } + } } diff --git a/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java index 7d81096676c..f3b29a48aa9 100644 --- a/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java +++ b/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java @@ -25,6 +25,9 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.persistent.PersistentTaskParams; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.persistent.PersistentTasksNodeService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -81,9 +84,6 @@ import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.core.ml.job.results.Forecast; import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; import org.elasticsearch.xpack.core.ml.job.results.Result; -import org.elasticsearch.persistent.PersistentTaskParams; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; -import org.elasticsearch.persistent.PersistentTasksNodeService; import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; @@ -359,7 +359,7 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase { XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, hits.getHits()[0].getSourceRef().streamInput()); - return ForecastRequestStats.PARSER.apply(parser, null); + return ForecastRequestStats.STRICT_PARSER.apply(parser, null); } catch (IOException e) { throw new IllegalStateException(e); } @@ -378,7 +378,7 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase { try { XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, hit.getSourceRef().streamInput()); - forecastStats.add(ForecastRequestStats.PARSER.apply(parser, null)); + forecastStats.add(ForecastRequestStats.STRICT_PARSER.apply(parser, null)); } catch (IOException e) { throw new IllegalStateException(e); } @@ -413,7 +413,7 @@ abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase { XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, hit.getSourceRef().streamInput()); - forecasts.add(Forecast.PARSER.apply(parser, null)); + forecasts.add(Forecast.STRICT_PARSER.apply(parser, null)); } catch (IOException e) { throw new IllegalStateException(e); } diff --git a/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java b/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java index 355af23ee04..3400d09ee75 100644 --- a/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java +++ b/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java @@ -165,7 +165,7 @@ public class RevertModelSnapshotIT extends MlNativeAutodetectIntegTestCase { try { XContentParser parser = JsonXContent.jsonXContent .createParser(null, LoggingDeprecationHandler.INSTANCE, hits.getAt(0).getSourceAsString()); - return Quantiles.PARSER.apply(parser, null); + return Quantiles.LENIENT_PARSER.apply(parser, null); } catch (IOException e) { throw new IllegalStateException(e); }