honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix some mappings on the .ml indexes (elastic/elasticsearch#870) 

Closes elastic/elasticsearch#814

Original commit: elastic/x-pack-elasticsearch@206efacc4c
This commit is contained in:
David Roberts 2017-02-06 12:26:03 +00:00 committed by GitHub
parent 3f9741b85f
commit f594030c9e
3 changed files with 55 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java
View File

@ -29,7 +29,6 @@ import org.elasticsearch.xpack.ml.job.results.Result;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@ -84,7 +83,6 @@ public class ElasticsearchMappings {
static final String INTEGER = "integer";
static final String KEYWORD = "keyword";
static final String LONG = "long";
static final String OBJECT = "object";
static final String TEXT = "text";
private ElasticsearchMappings() {
@ -142,6 +140,9 @@ public class ElasticsearchMappings {
.startObject(Bucket.ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
@ -187,12 +188,39 @@ public class ElasticsearchMappings {
.startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName())
.field(TYPE, NESTED)
.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Result.RESULT_TYPE.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(BucketInfluencer.PROBABILITY.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(BucketInfluencer.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName())
.field(TYPE, LONG)
.endObject()
.startObject(BucketInfluencer.SEQUENCE_NUM.getPreferredName())
.field(TYPE, INTEGER)
.endObject()
.startObject(BucketInfluencer.IS_INTERIM.getPreferredName())
.field(TYPE, BOOLEAN)
.endObject()
.endObject()
.endObject()
@ -486,7 +514,7 @@ public class ElasticsearchMappings {
.field(TYPE, TEXT)
.endObject()
.startObject(CategoryDefinition.REGEX.getPreferredName())
.field(TYPE, TEXT)
.field(TYPE, KEYWORD)
.endObject()
.startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName())
.field(TYPE, LONG)
@ -530,9 +558,6 @@ public class ElasticsearchMappings {
.startObject(ModelSnapshot.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
// "description" is analyzed so that it has the same
// mapping as a user field of the same name - this means
// it doesn't have to be a reserved field name
.startObject(ModelSnapshot.DESCRIPTION.getPreferredName())
.field(TYPE, TEXT)
.endObject()
@ -549,24 +574,20 @@ public class ElasticsearchMappings {
.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Result.RESULT_TYPE.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName())
.field(TYPE, DATE)
.endObject();
addModelSizeStatsFieldsToMapping(builder);
builder.endObject()
builder.endObject()
.endObject()
.startObject(Quantiles.TYPE.getPreferredName())
.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Quantiles.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(Quantiles.QUANTILE_STATE.getPreferredName())
.field(TYPE, TEXT)
.endObject()
.endObject()
.field(ENABLED, false)
.endObject()
.startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName())
.field(TYPE, DATE)
@ -618,6 +639,15 @@ public class ElasticsearchMappings {
.startObject()
.startObject(AuditMessage.TYPE.getPreferredName())
.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()

4
elasticsearch/src/main/java/org/elasticsearch/xpack/ml/job/results/ReservedFieldNames.java
View File

@ -136,7 +136,7 @@ public final class ReservedFieldNames {
ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName(),
ModelSizeStats.LOG_TIME_FIELD.getPreferredName(),
// ModelSnapshot.DESCRIPTION is not reserved because it is an analyzed string
ModelSnapshot.DESCRIPTION.getPreferredName(),
ModelSnapshot.RESTORE_PRIORITY.getPreferredName(),
ModelSnapshot.SNAPSHOT_ID.getPreferredName(),
ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName(),
@ -145,8 +145,6 @@ public final class ReservedFieldNames {
PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName(),
Quantiles.QUANTILE_STATE.getPreferredName(),
Result.RESULT_TYPE.getPreferredName()
};

59
elasticsearch/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java
View File

@ -8,21 +8,14 @@ package org.elasticsearch.xpack.ml.job.persistence;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState;
import org.elasticsearch.xpack.ml.job.results.AnomalyRecord;
import org.elasticsearch.xpack.ml.notifications.AuditActivity;
import org.elasticsearch.xpack.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.ml.job.metadata.Allocation;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.ml.job.results.AnomalyRecord;
import org.elasticsearch.xpack.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.ml.job.results.ReservedFieldNames;
import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.job.config.MlFilter;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParseException;
@ -81,52 +74,22 @@ public class ElasticsearchMappingsTests extends ESTestCase {
// These are not reserved because they're data types, not field names
overridden.add(Result.TYPE.getPreferredName());
overridden.add(AuditActivity.TYPE.getPreferredName());
overridden.add(AuditMessage.TYPE.getPreferredName());
overridden.add(DataCounts.TYPE.getPreferredName());
overridden.add(CategorizerState.TYPE);
overridden.add(CategoryDefinition.TYPE.getPreferredName());
overridden.add(Job.TYPE);
overridden.add(MlFilter.TYPE.getPreferredName());
overridden.add(ModelState.TYPE.getPreferredName());
overridden.add(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName());
overridden.add(ModelSnapshot.TYPE.getPreferredName());
overridden.add(Quantiles.TYPE.getPreferredName());
// These are not reserved because they're in the ml-int index
// not the job indices
overridden.add(MlFilter.ID.getPreferredName());
overridden.add(MlFilter.ITEMS.getPreferredName());
// These are not reserved because they're analyzed strings, i.e. the
// same type as user-specified fields
overridden.add(Job.DESCRIPTION.getPreferredName());
overridden.add(Allocation.STATE.getPreferredName());
overridden.add(ModelSnapshot.DESCRIPTION.getPreferredName());
Set<String> expected = new HashSet<>();
XContentBuilder builder = ElasticsearchMappings.auditActivityMapping();
BufferedInputStream inputStream = new BufferedInputStream(
new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
XContentBuilder builder = ElasticsearchMappings.resultsMapping(Collections.emptyList());
BufferedInputStream inputStream =
new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
JsonParser parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.auditMessageMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.resultsMapping(Collections.emptyList());
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.categorizerStateMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.categoryDefinitionMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
@ -142,16 +105,6 @@ public class ElasticsearchMappingsTests extends ESTestCase {
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.modelStateMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.quantilesMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
expected.removeAll(overridden);
if (ReservedFieldNames.RESERVED_FIELD_NAMES.size() != expected.size()) {