From f594030c9ebc499378721a5ac21e37804e5ec945 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 6 Feb 2017 12:26:03 +0000 Subject: [PATCH] Fix some mappings on the .ml indexes (elastic/elasticsearch#870) Closes elastic/elasticsearch#814 Original commit: elastic/x-pack-elasticsearch@206efacc4c1124ac810474f5e4d0fd84171c09ae --- .../persistence/ElasticsearchMappings.java | 66 ++++++++++++++----- .../ml/job/results/ReservedFieldNames.java | 4 +- .../ElasticsearchMappingsTests.java | 59 ++--------------- 3 files changed, 55 insertions(+), 74 deletions(-) diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java index f8400dd44f8..d62faa22929 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java @@ -29,7 +29,6 @@ import org.elasticsearch.xpack.ml.job.results.Result; import java.io.IOException; import java.util.Collection; -import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -84,7 +83,6 @@ public class ElasticsearchMappings { static final String INTEGER = "integer"; static final String KEYWORD = "keyword"; static final String LONG = "long"; - static final String OBJECT = "object"; static final String TEXT = "text"; private ElasticsearchMappings() { @@ -142,6 +140,9 @@ public class ElasticsearchMappings { .startObject(Bucket.ANOMALY_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() @@ -187,12 +188,39 @@ public class ElasticsearchMappings { .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) .field(TYPE, NESTED) .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) .field(TYPE, KEYWORD) .endObject() + .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() + .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(BucketInfluencer.SEQUENCE_NUM.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(BucketInfluencer.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() .endObject() .endObject() @@ -486,7 +514,7 @@ public class ElasticsearchMappings { .field(TYPE, TEXT) .endObject() .startObject(CategoryDefinition.REGEX.getPreferredName()) - .field(TYPE, TEXT) + .field(TYPE, KEYWORD) .endObject() .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName()) .field(TYPE, LONG) @@ -530,9 +558,6 @@ public class ElasticsearchMappings { .startObject(ModelSnapshot.TIMESTAMP.getPreferredName()) .field(TYPE, DATE) .endObject() - // "description" is analyzed so that it has the same - // mapping as a user field of the same name - this means - // it doesn't have to be a reserved field name .startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) .field(TYPE, TEXT) .endObject() @@ -549,24 +574,20 @@ public class ElasticsearchMappings { .startObject(PROPERTIES) .startObject(Job.ID.getPreferredName()) .field(TYPE, KEYWORD) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName()) + .field(TYPE, DATE) .endObject(); addModelSizeStatsFieldsToMapping(builder); - builder.endObject() + builder.endObject() .endObject() .startObject(Quantiles.TYPE.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Quantiles.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(Quantiles.QUANTILE_STATE.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .endObject() + .field(ENABLED, false) .endObject() .startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName()) .field(TYPE, DATE) @@ -618,6 +639,15 @@ public class ElasticsearchMappings { .startObject() .startObject(AuditMessage.TYPE.getPreferredName()) .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AuditMessage.LEVEL.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AuditMessage.MESSAGE.getPreferredName()) + .field(TYPE, TEXT) + .endObject() .startObject(AuditMessage.TIMESTAMP.getPreferredName()) .field(TYPE, DATE) .endObject() diff --git a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java index ed0e58cca29..539e07d6948 100644 --- a/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java +++ b/elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java @@ -136,7 +136,7 @@ public final class ReservedFieldNames { ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName(), ModelSizeStats.LOG_TIME_FIELD.getPreferredName(), - // ModelSnapshot.DESCRIPTION is not reserved because it is an analyzed string + ModelSnapshot.DESCRIPTION.getPreferredName(), ModelSnapshot.RESTORE_PRIORITY.getPreferredName(), ModelSnapshot.SNAPSHOT_ID.getPreferredName(), ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName(), @@ -145,8 +145,6 @@ public final class ReservedFieldNames { PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName(), - Quantiles.QUANTILE_STATE.getPreferredName(), - Result.RESULT_TYPE.getPreferredName() }; diff --git a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java index 17412505343..61e975aaa9b 100644 --- a/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java @@ -8,21 +8,14 @@ package org.elasticsearch.xpack.ml.job.persistence; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; -import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; -import org.elasticsearch.xpack.ml.notifications.AuditActivity; -import org.elasticsearch.xpack.ml.notifications.AuditMessage; -import org.elasticsearch.xpack.ml.job.metadata.Allocation; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; +import org.elasticsearch.xpack.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; import org.elasticsearch.xpack.ml.job.results.ReservedFieldNames; import org.elasticsearch.xpack.ml.job.results.Result; -import org.elasticsearch.xpack.ml.job.config.MlFilter; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonParseException; @@ -81,52 +74,22 @@ public class ElasticsearchMappingsTests extends ESTestCase { // These are not reserved because they're data types, not field names overridden.add(Result.TYPE.getPreferredName()); - overridden.add(AuditActivity.TYPE.getPreferredName()); - overridden.add(AuditMessage.TYPE.getPreferredName()); overridden.add(DataCounts.TYPE.getPreferredName()); - overridden.add(CategorizerState.TYPE); overridden.add(CategoryDefinition.TYPE.getPreferredName()); - overridden.add(Job.TYPE); - overridden.add(MlFilter.TYPE.getPreferredName()); - overridden.add(ModelState.TYPE.getPreferredName()); overridden.add(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()); overridden.add(ModelSnapshot.TYPE.getPreferredName()); overridden.add(Quantiles.TYPE.getPreferredName()); - // These are not reserved because they're in the ml-int index - // not the job indices - overridden.add(MlFilter.ID.getPreferredName()); - overridden.add(MlFilter.ITEMS.getPreferredName()); - - // These are not reserved because they're analyzed strings, i.e. the - // same type as user-specified fields - overridden.add(Job.DESCRIPTION.getPreferredName()); - overridden.add(Allocation.STATE.getPreferredName()); - overridden.add(ModelSnapshot.DESCRIPTION.getPreferredName()); - Set expected = new HashSet<>(); - XContentBuilder builder = ElasticsearchMappings.auditActivityMapping(); - BufferedInputStream inputStream = new BufferedInputStream( - new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + // Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here. + + XContentBuilder builder = ElasticsearchMappings.resultsMapping(Collections.emptyList()); + BufferedInputStream inputStream = + new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); JsonParser parser = new JsonFactory().createParser(inputStream); parseJson(parser, expected); - builder = ElasticsearchMappings.auditMessageMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - - builder = ElasticsearchMappings.resultsMapping(Collections.emptyList()); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - - builder = ElasticsearchMappings.categorizerStateMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - builder = ElasticsearchMappings.categoryDefinitionMapping(); inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); parser = new JsonFactory().createParser(inputStream); @@ -142,16 +105,6 @@ public class ElasticsearchMappingsTests extends ESTestCase { parser = new JsonFactory().createParser(inputStream); parseJson(parser, expected); - builder = ElasticsearchMappings.modelStateMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - - builder = ElasticsearchMappings.quantilesMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - expected.removeAll(overridden); if (ReservedFieldNames.RESERVED_FIELD_NAMES.size() != expected.size()) {