From 71fe59959282d04099d708d3254510126da6cf44 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 24 May 2017 13:24:32 +0100 Subject: [PATCH] [ML] Single doc type for results index (elastic/x-pack-elasticsearch#1528) This commit changes all results to use the single doc type. All searches were adjusted to work without the need to specify type which ensures BWC with 5.4 results. Additional work is needed to put the new doc mapping in indices created with 5.4 but it will be done in separate PR. Relates elastic/x-pack-elasticsearch#668 Original commit: elastic/x-pack-elasticsearch@041c88ac2d4e7beca11d4f2b1c13fd7fe2d64699 --- .../ml/MachineLearningTemplateRegistry.java | 21 +- .../elasticsearch/xpack/ml/MlMetaIndex.java | 17 + .../xpack/ml/action/DeleteFilterAction.java | 3 +- .../xpack/ml/action/GetFiltersAction.java | 2 +- .../xpack/ml/action/PutFilterAction.java | 5 +- .../ml/action/UpdateModelSnapshotAction.java | 40 +- .../xpack/ml/job/JobManager.java | 24 - .../xpack/ml/job/config/MlFilter.java | 10 + .../persistence/ElasticsearchMappings.java | 593 ++++++++---------- .../persistence/JobDataCountsPersister.java | 2 +- .../ml/job/persistence/JobDataDeleter.java | 18 +- .../xpack/ml/job/persistence/JobProvider.java | 71 ++- .../JobRenormalizedResultsPersister.java | 4 +- .../job/persistence/JobResultsPersister.java | 28 +- .../autodetect/state/ModelSizeStats.java | 6 +- .../autodetect/state/ModelSnapshot.java | 6 +- .../ExpiredModelSnapshotsRemover.java | 2 +- .../job/retention/ExpiredResultsRemover.java | 5 +- .../MachineLearningTemplateRegistryTests.java | 14 +- .../AutodetectResultProcessorIT.java | 6 +- .../ml/integration/DeleteExpiredDataIT.java | 9 +- .../integration/MlDistributedFailureIT.java | 1 - .../xpack/ml/job/JobManagerTests.java | 30 - .../xpack/ml/job/config/MlFilterTests.java | 5 + .../ElasticsearchMappingsTests.java | 86 ++- .../test/ml/delete_model_snapshot.yml | 4 +- .../rest-api-spec/test/ml/filter_crud.yml | 2 +- .../test/ml/ml_anomalies_default_mappings.yml | 6 +- .../rest-api-spec/test/ml/post_data.yml | 2 +- .../test/ml/revert_model_snapshot.yml | 18 +- .../test/ml/update_model_snapshot.yml | 4 +- 31 files changed, 498 insertions(+), 546 deletions(-) diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java index c29701630b4..286fa40c55b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java @@ -30,12 +30,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; -import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; -import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.Auditor; @@ -220,8 +216,9 @@ public class MachineLearningTemplateRegistry extends AbstractComponent implemen templateRequest.patterns(Collections.singletonList(MlMetaIndex.INDEX_NAME)); templateRequest.settings(mlNotificationIndexSettings()); templateRequest.version(Version.CURRENT.id); - try (XContentBuilder defaultMapping = ElasticsearchMappings.defaultMapping()) { - templateRequest.mapping(MapperService.DEFAULT_MAPPING, defaultMapping); + + try (XContentBuilder docMapping = MlMetaIndex.docMapping()) { + templateRequest.mapping(MlMetaIndex.TYPE, docMapping); } catch (IOException e) { String msg = "Error creating template mappings for the " + MlMetaIndex.INDEX_NAME + " index"; logger.error(msg, e); @@ -255,20 +252,12 @@ public class MachineLearningTemplateRegistry extends AbstractComponent implemen } void putJobResultsIndexTemplate(BiConsumer listener) { - try (XContentBuilder defaultMapping = ElasticsearchMappings.defaultMapping(); - XContentBuilder resultsMapping = ElasticsearchMappings.resultsMapping(); - XContentBuilder categoryDefinitionMapping = ElasticsearchMappings.categoryDefinitionMapping(); - XContentBuilder dataCountsMapping = ElasticsearchMappings.dataCountsMapping(); - XContentBuilder modelSnapshotMapping = ElasticsearchMappings.modelSnapshotMapping()) { + try (XContentBuilder docMapping = ElasticsearchMappings.docMapping()) { PutIndexTemplateRequest templateRequest = new PutIndexTemplateRequest(AnomalyDetectorsIndex.jobResultsIndexPrefix()); templateRequest.patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*")); templateRequest.settings(mlResultsIndexSettings()); - templateRequest.mapping(MapperService.DEFAULT_MAPPING, defaultMapping); - templateRequest.mapping(Result.TYPE.getPreferredName(), resultsMapping); - templateRequest.mapping(CategoryDefinition.TYPE.getPreferredName(), categoryDefinitionMapping); - templateRequest.mapping(DataCounts.TYPE.getPreferredName(), dataCountsMapping); - templateRequest.mapping(ModelSnapshot.TYPE.getPreferredName(), modelSnapshotMapping); + templateRequest.mapping(ElasticsearchMappings.DOC_TYPE, docMapping); templateRequest.version(Version.CURRENT.id); client.admin().indices().putTemplate(templateRequest, diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java index f90731151f3..70d2b9fe246 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java @@ -5,6 +5,13 @@ */ package org.elasticsearch.xpack.ml; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; + +import java.io.IOException; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + public final class MlMetaIndex { /** * Where to store the ml info in Elasticsearch - must match what's @@ -15,4 +22,14 @@ public final class MlMetaIndex { public static final String TYPE = "doc"; private MlMetaIndex() {} + + public static XContentBuilder docMapping() throws IOException { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.startObject(TYPE); + ElasticsearchMappings.addDefaultMapping(builder); + builder.endObject(); + builder.endObject(); + return builder; + } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java index 87abd2b3b62..0bab0bccab5 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.xpack.ml.MlMetaIndex; import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.job.config.Detector; import org.elasticsearch.xpack.ml.job.config.Job; +import org.elasticsearch.xpack.ml.job.config.MlFilter; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import java.io.IOException; @@ -178,7 +179,7 @@ public class DeleteFilterAction extends Action() { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java index 91082cd53f3..d8a84f1041c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java @@ -245,7 +245,7 @@ public class GetFiltersAction extends Action listener) { - GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId); + GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, MlFilter.documentId(filterId)); transportGetAction.execute(getRequest, new ActionListener() { @Override public void onResponse(GetResponse getDocResponse) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java index db7728fddf1..c4e70563d5b 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java @@ -174,8 +174,7 @@ public class PutFilterAction extends Action listener) { MlFilter filter = request.getFilter(); - final String filterId = filter.getId(); - IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId); + IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId()); try (XContentBuilder builder = XContentFactory.jsonBuilder()) { Payload.XContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlFilter.INCLUDE_TYPE_KEY, "true")); indexRequest.source(filter.toXContent(builder, params)); @@ -192,7 +191,7 @@ public class PutFilterAction extends Action { @@ -257,15 +263,15 @@ public class UpdateModelSnapshotAction extends Action { - private final JobManager jobManager; private final JobProvider jobProvider; + private final TransportBulkAction transportBulkAction; @Inject public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, - IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, JobProvider jobProvider) { + IndexNameExpressionResolver indexNameExpressionResolver, JobProvider jobProvider, TransportBulkAction transportBulkAction) { super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); - this.jobManager = jobManager; this.jobProvider = jobProvider; + this.transportBulkAction = transportBulkAction; } @Override @@ -277,7 +283,7 @@ public class UpdateModelSnapshotAction extends Action updatedSnapshot = applyUpdate(request, modelSnapshot); - jobManager.updateModelSnapshot(updatedSnapshot, b -> { + indexModelSnapshot(updatedSnapshot, b -> { // The quantiles can be large, and totally dominate the output - // it's clearer to remove them listener.onResponse(new Response(new ModelSnapshot.Builder(updatedSnapshot.result).setQuantiles(null).build())); @@ -296,5 +302,29 @@ public class UpdateModelSnapshotAction extends Action modelSnapshot, Consumer handler, Consumer errorHandler) { + IndexRequest indexRequest = new IndexRequest(modelSnapshot.index, ElasticsearchMappings.DOC_TYPE, + ModelSnapshot.documentId(modelSnapshot.result)); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS); + indexRequest.source(builder); + } catch (IOException e) { + errorHandler.accept(e); + return; + } + BulkRequest bulkRequest = new BulkRequest().add(indexRequest); + transportBulkAction.execute(bulkRequest, new ActionListener() { + @Override + public void onResponse(BulkResponse indexResponse) { + handler.accept(true); + } + + @Override + public void onFailure(Exception e) { + errorHandler.accept(e); + } + }); + } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 85dfeabf723..de37dd203f9 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; @@ -21,9 +20,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.action.DeleteJobAction; import org.elasticsearch.xpack.ml.action.PutJobAction; @@ -38,12 +34,10 @@ import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; -import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; -import java.io.IOException; import java.util.Collections; import java.util.Date; import java.util.List; @@ -404,24 +398,6 @@ public class JobManager extends AbstractComponent { }); } - /** - * Update a persisted model snapshot metadata document to match the - * argument supplied. - * - * @param modelSnapshot the updated model snapshot object to be stored - */ - public void updateModelSnapshot(Result modelSnapshot, Consumer handler, Consumer errorHandler) { - IndexRequest indexRequest = new IndexRequest(modelSnapshot.index, ModelSnapshot.TYPE.getPreferredName(), - ModelSnapshot.documentId(modelSnapshot.result)); - try (XContentBuilder builder = XContentFactory.jsonBuilder()) { - modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS); - indexRequest.source(builder); - } catch (IOException e) { - errorHandler.accept(e); - } - client.index(indexRequest, ActionListener.wrap(r -> handler.accept(true), errorHandler)); - } - private static MlMetadata.Builder createMlMetadataBuilder(ClusterState currentState) { MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE); return new MlMetadata.Builder(currentMlMetadata); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java index 21756140d35..470177dd5d3 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java @@ -23,6 +23,8 @@ import java.util.Objects; public class MlFilter extends ToXContentToBytes implements Writeable { + public static final String DOCUMENT_ID_PREFIX = "filter_"; + public static final String INCLUDE_TYPE_KEY = "include_type"; public static final String FILTER_TYPE = "filter"; @@ -99,6 +101,14 @@ public class MlFilter extends ToXContentToBytes implements Writeable { return Objects.hash(id, items); } + public String documentId() { + return documentId(id); + } + + public static String documentId(String filterId) { + return DOCUMENT_ID_PREFIX + filterId; + } + public static class Builder { private String id; diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java index 658fad1ed87..d43f1d6af98 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.ml.job.persistence; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; @@ -52,6 +51,9 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; * using whitespace. */ public class ElasticsearchMappings { + + public static final String DOC_TYPE = "doc"; + /** * String constants used in mappings */ @@ -97,25 +99,56 @@ public class ElasticsearchMappings { * so that the per-job term fields will not be automatically added * as fields of type 'text' to the index mappings of newly rolled indices. * - * @return The default mapping * @throws IOException On write error */ - public static XContentBuilder defaultMapping() throws IOException { - return jsonBuilder() - .startObject() - .startObject(MapperService.DEFAULT_MAPPING) - .startArray("dynamic_templates") - .startObject() - .startObject("strings_as_keywords") - .field("match", "*") - .startObject("mapping") - .field(TYPE, KEYWORD) - .endObject() - .endObject() + public static void addDefaultMapping(XContentBuilder builder) throws IOException { + builder.startArray("dynamic_templates") + .startObject() + .startObject("strings_as_keywords") + .field("match", "*") + .startObject("mapping") + .field(TYPE, KEYWORD) .endObject() - .endArray() + .endObject() .endObject() - .endObject(); + .endArray(); + } + + public static XContentBuilder docMapping() throws IOException { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.startObject(DOC_TYPE); + addDefaultMapping(builder); + builder.startObject(PROPERTIES); + + // Add result all field for easy searches in kibana + builder.startObject(ALL_FIELD_VALUES) + .field(TYPE, TEXT) + .field(ANALYZER, WHITESPACE) + .endObject(); + + builder.startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .field(COPY_TO, ALL_FIELD_VALUES) + .endObject(); + + builder.startObject(Result.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject(); + + addResultsMapping(builder); + addCategoryDefinitionMapping(builder); + addDataCountsMapping(builder); + addModelSnapshotMapping(builder); + + // end properties + builder.endObject(); + // end mapping + builder.endObject(); + // end doc + builder.endObject(); + + return builder; } /** @@ -142,145 +175,120 @@ public class ElasticsearchMappings { *
  • Influencer.influencer_field_value
    • * * - * @return The mapping * @throws IOException On write error */ - public static XContentBuilder resultsMapping() throws IOException { - XContentBuilder builder = jsonBuilder() - .startObject() - .startObject(Result.TYPE.getPreferredName()) - .startObject(PROPERTIES) - .startObject(ALL_FIELD_VALUES) - .field(TYPE, TEXT) - .field(ANALYZER, WHITESPACE) - .endObject() - .startObject(Result.RESULT_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .field(COPY_TO, ALL_FIELD_VALUES) - .endObject() - .startObject(Result.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(Bucket.ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Result.IS_INTERIM.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(Bucket.RECORD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.EVENT_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.BUCKET_SPAN.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.PROCESSING_TIME_MS.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Bucket.PARTITION_SCORES.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() + private static void addResultsMapping(XContentBuilder builder) throws IOException { + builder.startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Bucket.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Result.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(Bucket.RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.EVENT_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.BUCKET_SPAN.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.PROCESSING_TIME_MS.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Bucket.PARTITION_SCORES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(AnomalyRecord.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() - .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Result.RESULT_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(Result.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Result.IS_INTERIM.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .endObject() - .endObject() + .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(BucketInfluencer.PROBABILITY.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(Result.TIMESTAMP.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(Result.IS_INTERIM.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .endObject() + .endObject() - // per-partition max probabilities mapping - .startObject(PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName()) - .field(TYPE, NESTED) - .startObject(PROPERTIES) - .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(PerPartitionMaxProbabilities.MAX_RECORD_SCORE.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .endObject() - .endObject() + // per-partition max probabilities mapping + .startObject(PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName()) + .field(TYPE, NESTED) + .startObject(PROPERTIES) + .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(PerPartitionMaxProbabilities.MAX_RECORD_SCORE.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .endObject() + .endObject() - // Model Plot Output - .startObject(ModelPlot.MODEL_FEATURE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelPlot.MODEL_LOWER.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(ModelPlot.MODEL_UPPER.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject() - .startObject(ModelPlot.MODEL_MEDIAN.getPreferredName()) - .field(TYPE, DOUBLE) - .endObject(); + // Model Plot Output + .startObject(ModelPlot.MODEL_FEATURE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelPlot.MODEL_LOWER.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(ModelPlot.MODEL_UPPER.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject() + .startObject(ModelPlot.MODEL_MEDIAN.getPreferredName()) + .field(TYPE, DOUBLE) + .endObject(); addAnomalyRecordFieldsToMapping(builder); addInfluencerFieldsToMapping(builder); addModelSizeStatsFieldsToMapping(builder); - - // End result properties - builder.endObject(); - // End result - builder.endObject(); - // End mapping - builder.endObject(); - - return builder; } static XContentBuilder termFieldsMapping(String type, Collection termFields) { @@ -312,11 +320,9 @@ public class ElasticsearchMappings { /** * AnomalyRecord fields to be added under the 'properties' section of the mapping * @param builder Add properties to this builder - * @return builder * @throws IOException On write error */ - private static XContentBuilder addAnomalyRecordFieldsToMapping(XContentBuilder builder) - throws IOException { + private static void addAnomalyRecordFieldsToMapping(XContentBuilder builder) throws IOException { builder.startObject(AnomalyRecord.DETECTOR_INDEX.getPreferredName()) .field(TYPE, INTEGER) .endObject() @@ -426,11 +432,9 @@ public class ElasticsearchMappings { .endObject() .endObject() .endObject(); - - return builder; } - private static XContentBuilder addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException { + private static void addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException { builder.startObject(Influencer.INFLUENCER_SCORE.getPreferredName()) .field(TYPE, DOUBLE) .endObject() @@ -444,8 +448,6 @@ public class ElasticsearchMappings { .field(TYPE, KEYWORD) .field(COPY_TO, ALL_FIELD_VALUES) .endObject(); - - return builder; } /** @@ -453,68 +455,57 @@ public class ElasticsearchMappings { * The type is disabled so {@link DataCounts} aren't searchable and * the '_all' field is disabled * - * @return The builder * @throws IOException On builder write error */ - public static XContentBuilder dataCountsMapping() throws IOException { - return jsonBuilder() - .startObject() - .startObject(DataCounts.TYPE.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INPUT_BYTES.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.EMPTY_BUCKET_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.SPARSE_BUCKET_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.BUCKET_COUNT.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(DataCounts.EARLIEST_RECORD_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LATEST_EMPTY_BUCKET_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LATEST_SPARSE_BUCKET_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(DataCounts.LAST_DATA_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .endObject() - .endObject() - .endObject(); + private static void addDataCountsMapping(XContentBuilder builder) throws IOException { + builder.startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_BYTES.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.EMPTY_BUCKET_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.SPARSE_BUCKET_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.BUCKET_COUNT.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(DataCounts.EARLIEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(DataCounts.LATEST_EMPTY_BUCKET_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(DataCounts.LATEST_SPARSE_BUCKET_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(DataCounts.LAST_DATA_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject(); } /** @@ -554,35 +545,24 @@ public class ElasticsearchMappings { * Create the Elasticsearch mapping for {@linkplain CategoryDefinition}. * The '_all' field is disabled as the document isn't meant to be searched. * - * @return The builder * @throws IOException On builder error */ - public static XContentBuilder categoryDefinitionMapping() throws IOException { - return jsonBuilder() - .startObject() - .startObject(CategoryDefinition.TYPE.getPreferredName()) - .startObject(PROPERTIES) - .startObject(CategoryDefinition.CATEGORY_ID.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(CategoryDefinition.TERMS.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(CategoryDefinition.REGEX.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(CategoryDefinition.EXAMPLES.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .endObject() - .endObject() - .endObject(); + private static void addCategoryDefinitionMapping(XContentBuilder builder) throws IOException { + builder.startObject(CategoryDefinition.CATEGORY_ID.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(CategoryDefinition.TERMS.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(CategoryDefinition.REGEX.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(CategoryDefinition.EXAMPLES.getPreferredName()) + .field(TYPE, TEXT) + .endObject(); } /** @@ -605,91 +585,76 @@ public class ElasticsearchMappings { * Create the Elasticsearch mapping for {@linkplain ModelSnapshot}. * The '_all' field is disabled but the type is searchable */ - public static XContentBuilder modelSnapshotMapping() throws IOException { - XContentBuilder builder = jsonBuilder() - .startObject() - .startObject(ModelSnapshot.TYPE.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSnapshot.TIMESTAMP.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) - .field(TYPE, TEXT) - .endObject() - .startObject(ModelSnapshot.SNAPSHOT_ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()) - .field(TYPE, INTEGER) - .endObject() - .startObject(ModelSnapshot.RETAIN.getPreferredName()) - .field(TYPE, BOOLEAN) - .endObject() - .startObject(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()) - .startObject(PROPERTIES) - .startObject(Job.ID.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(Result.RESULT_TYPE.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName()) - .field(TYPE, DATE) - .endObject(); + private static void addModelSnapshotMapping(XContentBuilder builder) throws IOException { + builder.startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) + .field(TYPE, TEXT) + .endObject() + .startObject(ModelSnapshot.SNAPSHOT_ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()) + .field(TYPE, INTEGER) + .endObject() + .startObject(ModelSnapshot.RETAIN.getPreferredName()) + .field(TYPE, BOOLEAN) + .endObject() + .startObject(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()) + .startObject(PROPERTIES) + .startObject(Job.ID.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(Result.RESULT_TYPE.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName()) + .field(TYPE, DATE) + .endObject(); addModelSizeStatsFieldsToMapping(builder); - builder.endObject() - .endObject() - .startObject(Quantiles.TYPE.getPreferredName()) - .field(ENABLED, false) - .endObject() - .startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName()) - .field(TYPE, DATE) - .endObject() - .endObject() - .endObject() - .endObject(); + // end model size stats properties + builder.endObject(); + // end model size stats mapping + builder.endObject(); - return builder; + builder.startObject(Quantiles.TYPE.getPreferredName()) + .field(ENABLED, false) + .endObject() + .startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject() + .startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName()) + .field(TYPE, DATE) + .endObject(); } /** * {@link ModelSizeStats} fields to be added under the 'properties' section of the mapping * @param builder Add properties to this builder - * @return builder * @throws IOException On write error */ - private static XContentBuilder addModelSizeStatsFieldsToMapping(XContentBuilder builder) throws IOException { + private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) throws IOException { builder.startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) - .field(TYPE, LONG) - .endObject() - .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) - .field(TYPE, KEYWORD) - .endObject() - .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) - .field(TYPE, DATE) - .endObject(); - - return builder; + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) + .field(TYPE, LONG) + .endObject() + .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) + .field(TYPE, KEYWORD) + .endObject() + .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) + .field(TYPE, DATE) + .endObject(); } public static XContentBuilder auditMessageMapping() throws IOException { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java index 22e1b5a399c..e318b0cd2bc 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java @@ -47,7 +47,7 @@ public class JobDataCountsPersister extends AbstractComponent { */ public void persistDataCounts(String jobId, DataCounts counts, ActionListener listener) { try (XContentBuilder content = serialiseCounts(counts)) { - client.prepareIndex(AnomalyDetectorsIndex.resultsWriteAlias(jobId), DataCounts.TYPE.getPreferredName(), + client.prepareIndex(AnomalyDetectorsIndex.resultsWriteAlias(jobId), ElasticsearchMappings.DOC_TYPE, DataCounts.documentId(jobId)) .setSource(content).execute(new ActionListener() { @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java index a31a8066b41..05618f74873 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java @@ -10,9 +10,6 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.index.reindex.BulkByScrollResponse; -import org.elasticsearch.index.reindex.DeleteByQueryAction; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; @@ -21,7 +18,9 @@ import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; @@ -61,7 +60,7 @@ public class JobDataDeleter { } bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsAliasedName(modelSnapshot.getJobId()), - ModelSnapshot.TYPE.getPreferredName(), ModelSnapshot.documentId(modelSnapshot))); + ElasticsearchMappings.DOC_TYPE, ModelSnapshot.documentId(modelSnapshot))); } bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); @@ -82,11 +81,11 @@ public class JobDataDeleter { DeleteByQueryHolder deleteByQueryHolder = new DeleteByQueryHolder(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)); deleteByQueryHolder.dbqRequest.setRefresh(true); - RangeQueryBuilder timeRange = QueryBuilders.rangeQuery(Result.TIMESTAMP.getPreferredName()); - timeRange.gte(cutoffEpochMs); + QueryBuilder query = QueryBuilders.boolQuery() + .filter(QueryBuilders.existsQuery(Result.RESULT_TYPE.getPreferredName())) + .filter(QueryBuilders.rangeQuery(Result.TIMESTAMP.getPreferredName()).gte(cutoffEpochMs)); deleteByQueryHolder.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); - deleteByQueryHolder.searchRequest.types(Result.TYPE.getPreferredName()); - deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(timeRange)); + deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(query)); client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryHolder.dbqRequest, new ActionListener() { @Override public void onResponse(BulkByScrollResponse bulkByScrollResponse) { @@ -108,7 +107,6 @@ public class JobDataDeleter { deleteByQueryHolder.dbqRequest.setRefresh(false); deleteByQueryHolder.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); - deleteByQueryHolder.searchRequest.types(Result.TYPE.getPreferredName()); QueryBuilder qb = QueryBuilders.termQuery(Result.IS_INTERIM.getPreferredName(), true); deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(new ConstantScoreQueryBuilder(qb))); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java index c5e1bd6fecc..71877549c06 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.mapper.MapperService; @@ -138,8 +139,9 @@ public class JobProvider { if (!state.getMetaData().hasIndex(indexName)) { LOGGER.trace("ES API CALL: create index {}", indexName); CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); - String type = Result.TYPE.getPreferredName(); - createIndexRequest.mapping(type, ElasticsearchMappings.termFieldsMapping(type, termFields)); + try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(ElasticsearchMappings.DOC_TYPE, termFields)) { + createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, termFieldsMapping); + } client.admin().indices().create(createIndexRequest, ActionListener.wrap( r -> createAliasListener.onResponse(r.isAcknowledged()), @@ -209,19 +211,21 @@ public class JobProvider { } private void updateIndexMappingWithTermFields(String indexName, Collection termFields, ActionListener listener) { - client.admin().indices().preparePutMapping(indexName).setType(Result.TYPE.getPreferredName()) - .setSource(ElasticsearchMappings.termFieldsMapping(null, termFields)) - .execute(new ActionListener() { - @Override - public void onResponse(PutMappingResponse putMappingResponse) { - listener.onResponse(putMappingResponse.isAcknowledged()); - } + try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(null, termFields)) { + client.admin().indices().preparePutMapping(indexName).setType(ElasticsearchMappings.DOC_TYPE) + .setSource(termFieldsMapping) + .execute(new ActionListener() { + @Override + public void onResponse(PutMappingResponse putMappingResponse) { + listener.onResponse(putMappingResponse.isAcknowledged()); + } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } } /** @@ -251,12 +255,11 @@ public class JobProvider { MultiSearchRequestBuilder msearch = client.prepareMultiSearch() .add(createLatestDataCountsSearch(resultsIndex, jobId)) .add(createLatestModelSizeStatsSearch(resultsIndex)) - .add(createDocIdSearch(resultsIndex, ModelSnapshot.TYPE.getPreferredName(), - ModelSnapshot.documentId(jobId, job.getModelSnapshotId()))) - .add(createDocIdSearch(stateIndex, Quantiles.TYPE.getPreferredName(), Quantiles.documentId(jobId))); + .add(createDocIdSearch(resultsIndex, ModelSnapshot.documentId(jobId, job.getModelSnapshotId()))) + .add(createDocIdSearch(stateIndex, Quantiles.documentId(jobId))); for (String filterId : job.getAnalysisConfig().extractReferencedFilters()) { - msearch.add(createDocIdSearch(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId)); + msearch.add(createDocIdSearch(MlMetaIndex.INDEX_NAME, filterId)); } msearch.execute(ActionListener.wrap( @@ -284,7 +287,7 @@ public class JobProvider { SearchRequest searchRequest = msearch.request().requests().get(i); LOGGER.debug("Found 0 hits for [{}/{}]", searchRequest.indices(), searchRequest.types()); } else if (hitsCount == 1) { - parseAutodetectParamSearchHit(paramsBuilder, hits.getAt(0), errorHandler); + parseAutodetectParamSearchHit(jobId, paramsBuilder, hits.getAt(0), errorHandler); } else if (hitsCount > 1) { errorHandler.accept(new IllegalStateException("Expected hits count to be 0 or 1, but got [" + hitsCount + "]")); @@ -298,30 +301,30 @@ public class JobProvider { )); } - private SearchRequestBuilder createDocIdSearch(String index, String type, String id) { + private SearchRequestBuilder createDocIdSearch(String index, String id) { return client.prepareSearch(index).setSize(1) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) - .setQuery(QueryBuilders.idsQuery(type).addIds(id)) + .setQuery(QueryBuilders.idsQuery().addIds(id)) .setRouting(id); } - private void parseAutodetectParamSearchHit(AutodetectParams.Builder paramsBuilder, - SearchHit hit, Consumer errorHandler) { - String type = hit.getType(); - if (DataCounts.TYPE.getPreferredName().equals(type)) { + private void parseAutodetectParamSearchHit(String jobId, AutodetectParams.Builder paramsBuilder, SearchHit hit, + Consumer errorHandler) { + String hitId = hit.getId(); + if (DataCounts.documentId(jobId).equals(hitId)) { paramsBuilder.setDataCounts(parseSearchHit(hit, DataCounts.PARSER, errorHandler)); - } else if (Result.TYPE.getPreferredName().equals(type)) { + } else if (hitId.startsWith(ModelSizeStats.documentIdPrefix(jobId))) { ModelSizeStats.Builder modelSizeStats = parseSearchHit(hit, ModelSizeStats.PARSER, errorHandler); paramsBuilder.setModelSizeStats(modelSizeStats == null ? null : modelSizeStats.build()); - } else if (ModelSnapshot.TYPE.getPreferredName().equals(type)) { + } else if (hitId.startsWith(ModelSnapshot.documentIdPrefix(jobId))) { ModelSnapshot.Builder modelSnapshot = parseSearchHit(hit, ModelSnapshot.PARSER, errorHandler); paramsBuilder.setModelSnapshot(modelSnapshot == null ? null : modelSnapshot.build()); - } else if (Quantiles.TYPE.getPreferredName().equals(type)) { + } else if (Quantiles.documentId(jobId).equals(hit.getId())) { paramsBuilder.setQuantiles(parseSearchHit(hit, Quantiles.PARSER, errorHandler)); - } else if (MlFilter.TYPE.getPreferredName().equals(type)) { + } else if (hitId.startsWith(MlFilter.DOCUMENT_ID_PREFIX)) { paramsBuilder.addFilter(parseSearchHit(hit, MlFilter.PARSER, errorHandler).build()); } else { - errorHandler.accept(new IllegalStateException("Unexpected type [" + type + "]")); + errorHandler.accept(new IllegalStateException("Unexpected type [" + hit.getType() + "]")); } } @@ -644,7 +647,6 @@ public class JobProvider { SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions())); - searchRequest.types(Result.TYPE.getPreferredName()); searchRequest.source(new SearchSourceBuilder() .from(from) .size(size) @@ -703,7 +705,6 @@ public class JobProvider { SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions())); - searchRequest.types(Result.TYPE.getPreferredName()); FieldSortBuilder sb = query.getSortField() == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC) : new FieldSortBuilder(query.getSortField()).order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC); searchRequest.source(new SearchSourceBuilder().query(qb).from(query.getFrom()).size(query.getSize()).sort(sb)); @@ -744,8 +745,7 @@ public class JobProvider { return; } String resultsIndex = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); - SearchRequestBuilder search = createDocIdSearch(resultsIndex, ModelSnapshot.TYPE.getPreferredName(), - ModelSnapshot.documentId(jobId, modelSnapshotId)); + SearchRequestBuilder search = createDocIdSearch(resultsIndex, ModelSnapshot.documentId(jobId, modelSnapshotId)); searchSingleResult(jobId, ModelSnapshot.TYPE.getPreferredName(), search, ModelSnapshot.PARSER, result -> handler.accept(result.result == null ? null : new Result(result.index, result.result.build())), errorHandler, () -> null); @@ -814,7 +814,7 @@ public class JobProvider { .order(sortDescending ? SortOrder.DESC : SortOrder.ASC); String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); - LOGGER.trace("ES API CALL: search all of type {} from index {} sort ascending {} with filter after sort from {} size {}", + LOGGER.trace("ES API CALL: search all {}s from index {} sort ascending {} with filter after sort from {} size {}", ModelSnapshot.TYPE, indexName, sortField, from, size); SearchRequest searchRequest = new SearchRequest(indexName); @@ -909,7 +909,6 @@ public class JobProvider { searchResponse = client.prepareSearch(indexName) .setIndicesOptions(addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS)) - .setTypes(Result.TYPE.getPreferredName()) .setQuery(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), ModelPlot.RESULT_TYPE_VALUE)) .setFrom(from).setSize(size) .get(); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java index 7a4945a9a2e..034c879ca05 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java @@ -17,12 +17,12 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.BucketNormalizable; import org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable; import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; -import org.elasticsearch.xpack.ml.job.results.Result; import java.io.IOException; import java.util.List; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings.DOC_TYPE; /** @@ -66,7 +66,7 @@ public class JobRenormalizedResultsPersister extends AbstractComponent { public void updateResult(String id, String index, ToXContent resultDoc) { try (XContentBuilder content = toXContentBuilder(resultDoc)) { - bulkRequest.add(new IndexRequest(index, Result.TYPE.getPreferredName(), id).source(content)); + bulkRequest.add(new IndexRequest(index, DOC_TYPE, id).source(content)); } catch (IOException e) { logger.error("Error serialising result", e); } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java index 034fc674b8d..c9afeeca19d 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Objects; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings.DOC_TYPE; /** * Persists result types, Quantiles etc to Elasticsearch
    @@ -60,7 +61,6 @@ public class JobResultsPersister extends AbstractComponent { private final Client client; - public JobResultsPersister(Settings settings, Client client) { super(settings); this.client = client; @@ -100,8 +100,7 @@ public class JobResultsPersister extends AbstractComponent { logger.trace("[{}] ES API CALL: index result type {} to index {} at epoch {}", jobId, Bucket.RESULT_TYPE_VALUE, indexName, bucketWithoutRecords.getEpoch()); - bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), - bucketWithoutRecords.getId()).source(content)); + bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, bucketWithoutRecords.getId()).source(content)); persistBucketInfluencersStandalone(jobId, bucketWithoutRecords.getBucketInfluencers()); } catch (IOException e) { @@ -120,7 +119,7 @@ public class JobResultsPersister extends AbstractComponent { String id = bucketInfluencer.getId(); logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", jobId, BucketInfluencer.RESULT_TYPE_VALUE, indexName, id); - bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), id).source(content)); + bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, id).source(content)); } } } @@ -139,7 +138,7 @@ public class JobResultsPersister extends AbstractComponent { try (XContentBuilder content = toXContentBuilder(record)) { logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", jobId, AnomalyRecord.RESULT_TYPE_VALUE, indexName, record.getId()); - bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), record.getId()).source(content)); + bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, record.getId()).source(content)); } } } catch (IOException e) { @@ -162,7 +161,7 @@ public class JobResultsPersister extends AbstractComponent { try (XContentBuilder content = toXContentBuilder(influencer)) { logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", jobId, Influencer.RESULT_TYPE_VALUE, indexName, influencer.getId()); - bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), influencer.getId()).source(content)); + bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, influencer.getId()).source(content)); } } } catch (IOException e) { @@ -184,7 +183,7 @@ public class JobResultsPersister extends AbstractComponent { jobId, PerPartitionMaxProbabilities.RESULT_TYPE_VALUE, indexName, partitionProbabilities.getTimestamp(), partitionProbabilities.getId()); bulkRequest.add( - new IndexRequest(indexName, Result.TYPE.getPreferredName(), partitionProbabilities.getId()).source(builder)); + new IndexRequest(indexName, DOC_TYPE, partitionProbabilities.getId()).source(builder)); } catch (IOException e) { logger.error(new ParameterizedMessage("[{}] error serialising bucket per partition max normalized scores", @@ -349,14 +348,14 @@ public class JobResultsPersister extends AbstractComponent { private final String jobId; private final ToXContent object; - private final String type; + private final String description; private final String id; private WriteRequest.RefreshPolicy refreshPolicy; - Persistable(String jobId, ToXContent object, String type, String id) { + Persistable(String jobId, ToXContent object, String description, String id) { this.jobId = jobId; this.object = object; - this.type = type; + this.description = description; this.id = id; this.refreshPolicy = WriteRequest.RefreshPolicy.NONE; } @@ -373,12 +372,15 @@ public class JobResultsPersister extends AbstractComponent { void persist(String indexName, ActionListener listener) { logCall(indexName); + // TODO no_release: this is a temporary hack until we also switch state index to have doc type in which case + // we can remove this line and use DOC_TYPE directly in the index request + String type = AnomalyDetectorsIndex.jobStateIndexName().equals(indexName) ? description : DOC_TYPE; try (XContentBuilder content = toXContentBuilder(object)) { IndexRequest indexRequest = new IndexRequest(indexName, type, id).source(content).setRefreshPolicy(refreshPolicy); client.index(indexRequest, listener); } catch (IOException e) { - logger.error(new ParameterizedMessage("[{}] Error writing {}", new Object[]{jobId, type}), e); + logger.error(new ParameterizedMessage("[{}] Error writing {}", new Object[]{jobId, description}), e); IndexResponse.Builder notCreatedResponse = new IndexResponse.Builder(); notCreatedResponse.setCreated(false); listener.onResponse(notCreatedResponse.build()); @@ -387,9 +389,9 @@ public class JobResultsPersister extends AbstractComponent { private void logCall(String indexName) { if (id != null) { - logger.trace("[{}] ES API CALL: index type {} to index {} with ID {}", jobId, type, indexName, id); + logger.trace("[{}] ES API CALL: index {} to index {} with ID {}", jobId, description, indexName, id); } else { - logger.trace("[{}] ES API CALL: index type {} to index {} with auto-generated ID", jobId, type, indexName); + logger.trace("[{}] ES API CALL: index {} to index {} with auto-generated ID", jobId, description, indexName); } } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java index 616baf6a25e..182a602f7b7 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java @@ -143,7 +143,11 @@ public class ModelSizeStats extends ToXContentToBytes implements Writeable { } public String getId() { - return jobId + "_model_size_stats_" + logTime.getTime(); + return documentIdPrefix(jobId) + logTime.getTime(); + } + + public static String documentIdPrefix(String jobId) { + return jobId + "_model_size_stats_"; } @Override diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java index 88656a5fac4..6c27cc2ec9c 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java @@ -274,12 +274,16 @@ public class ModelSnapshot extends ToXContentToBytes implements Writeable { return stateDocumentIds; } + public static String documentIdPrefix(String jobId) { + return jobId + "_model_snapshot_"; + } + public static String documentId(ModelSnapshot snapshot) { return documentId(snapshot.getJobId(), snapshot.getSnapshotId()); } public static String documentId(String jobId, String snapshotId) { - return jobId + "_model_snapshot_" + snapshotId; + return documentIdPrefix(jobId) + snapshotId; } public static ModelSnapshot fromJson(BytesReference bytesReference) { diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java index 14c119e7469..07c0adb6437 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java @@ -67,12 +67,12 @@ public class ExpiredModelSnapshotsRemover extends AbstractExpiredJobDataRemover SearchRequest searchRequest = new SearchRequest(); searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId())); - searchRequest.types(ModelSnapshot.TYPE.getPreferredName()); QueryBuilder activeSnapshotFilter = QueryBuilders.termQuery( ModelSnapshot.SNAPSHOT_ID.getPreferredName(), job.getModelSnapshotId()); QueryBuilder retainFilter = QueryBuilders.termQuery(ModelSnapshot.RETAIN.getPreferredName(), true); QueryBuilder query = createQuery(job.getId(), cutoffEpochMs) + .filter(QueryBuilders.existsQuery(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName())) .mustNot(activeSnapshotFilter) .mustNot(retainFilter); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java index 5af97be032e..86f4b694f0a 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java @@ -86,9 +86,10 @@ public class ExpiredResultsRemover extends AbstractExpiredJobDataRemover { request.setSlices(5); searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId())); - searchRequest.types(Result.TYPE.getPreferredName()); QueryBuilder excludeFilter = QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), ModelSizeStats.RESULT_TYPE_VALUE); - QueryBuilder query = createQuery(job.getId(), cutoffEpochMs).mustNot(excludeFilter); + QueryBuilder query = createQuery(job.getId(), cutoffEpochMs) + .filter(QueryBuilders.existsQuery(Result.RESULT_TYPE.getPreferredName())) + .mustNot(excludeFilter); searchRequest.source(new SearchSourceBuilder().query(query)); return request; } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistryTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistryTests.java index 48cbff69c70..f0928852c3a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistryTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistryTests.java @@ -27,12 +27,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder; import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; -import org.elasticsearch.xpack.ml.job.results.CategoryDefinition; -import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.Before; @@ -221,7 +217,7 @@ public class MachineLearningTemplateRegistryTests extends ESTestCase { assertNotNull(request); assertEquals(templateRegistry.mlNotificationIndexSettings().build(), request.settings()); assertEquals(1, request.mappings().size()); - assertThat(request.mappings().containsKey(MapperService.DEFAULT_MAPPING), is(true)); + assertThat(request.mappings().containsKey("doc"), is(true)); assertEquals(Collections.singletonList(MlMetaIndex.INDEX_NAME), request.patterns()); assertEquals(new Integer(Version.CURRENT.id), request.version()); }); @@ -262,12 +258,8 @@ public class MachineLearningTemplateRegistryTests extends ESTestCase { PutIndexTemplateRequest request = captor.getValue(); assertNotNull(request); assertEquals(templateRegistry.mlResultsIndexSettings().build(), request.settings()); - assertTrue(request.mappings().containsKey("_default_")); - assertTrue(request.mappings().containsKey(Result.TYPE.getPreferredName())); - assertTrue(request.mappings().containsKey(CategoryDefinition.TYPE.getPreferredName())); - assertTrue(request.mappings().containsKey(DataCounts.TYPE.getPreferredName())); - assertTrue(request.mappings().containsKey(ModelSnapshot.TYPE.getPreferredName())); - assertEquals(5, request.mappings().size()); + assertTrue(request.mappings().containsKey("doc")); + assertEquals(1, request.mappings().size()); assertEquals(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"), request.patterns()); assertEquals(new Integer(Version.CURRENT.id), request.version()); }); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java index ad716ba32ee..1a1d11489f9 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java @@ -150,9 +150,9 @@ public class AutodetectResultProcessorIT extends XPackSingleNodeTestCase { assertEquals(1, persistedDefinition.count()); assertEquals(categoryDefinition, persistedDefinition.results().get(0)); - QueryPage persistedmodelPlot = jobProvider.modelPlot(JOB_ID, 0, 100); - assertEquals(1, persistedmodelPlot.count()); - assertEquals(modelPlot, persistedmodelPlot.results().get(0)); + QueryPage persistedModelPlot = jobProvider.modelPlot(JOB_ID, 0, 100); + assertEquals(1, persistedModelPlot.count()); + assertEquals(modelPlot, persistedModelPlot.results().get(0)); ModelSizeStats persistedModelSizeStats = getModelSizeStats(); assertEquals(modelSizeStats, persistedModelSizeStats); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java index dfcb4cd6d0f..1a945ebc8e2 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java @@ -113,7 +113,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase { // Update snapshot timestamp to force it out of snapshot retention window String snapshotUpdate = "{ \"timestamp\": " + oneDayAgo + "}"; - UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), "model_snapshot", snapshotDocId); + UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), "doc", snapshotDocId); updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON); client().execute(UpdateAction.INSTANCE, updateSnapshotRequest).get(); } @@ -136,7 +136,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase { retainAllSnapshots("snapshots-retention-with-retain"); - long totalModelSizeStatsBeforeDelete = client().prepareSearch("*").setTypes("result") + long totalModelSizeStatsBeforeDelete = client().prepareSearch("*") .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats")) .get().getHits().totalHits; long totalNotificationsCountBeforeDelete = client().prepareSearch(".ml-notifications").get().getHits().totalHits; @@ -175,7 +175,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase { assertThat(getRecords("results-and-snapshots-retention").size(), equalTo(0)); assertThat(getModelSnapshots("results-and-snapshots-retention").size(), equalTo(1)); - long totalModelSizeStatsAfterDelete = client().prepareSearch("*").setTypes("result") + long totalModelSizeStatsAfterDelete = client().prepareSearch("*") .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats")) .get().getHits().totalHits; long totalNotificationsCountAfterDelete = client().prepareSearch(".ml-notifications").get().getHits().totalHits; @@ -199,8 +199,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase { private void retainAllSnapshots(String jobId) throws Exception { List modelSnapshots = getModelSnapshots(jobId); for (ModelSnapshot modelSnapshot : modelSnapshots) { - UpdateModelSnapshotAction.Request request = new UpdateModelSnapshotAction.Request( - jobId, modelSnapshot.getSnapshotId()); + UpdateModelSnapshotAction.Request request = new UpdateModelSnapshotAction.Request(jobId, modelSnapshot.getSnapshotId()); request.setRetain(true); client().execute(UpdateModelSnapshotAction.INSTANCE, request).get(); } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index f0d1979b0f9..349c37ff742 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -171,7 +171,6 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase { // are what we expect them to be: private static DataCounts getDataCountsFromIndex(String jobId) { SearchResponse searchResponse = client().prepareSearch() - .setTypes(DataCounts.TYPE.getPreferredName()) .setQuery(QueryBuilders.idsQuery().addIds(DataCounts.documentId(jobId))) .get(); if (searchResponse.getHits().getTotalHits() != 1) { diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index 0aee46e889a..000576256ba 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterName; @@ -15,9 +14,6 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.env.Environment; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ml.MlMetadata; @@ -28,15 +24,11 @@ import org.elasticsearch.xpack.ml.job.config.DataDescription; import org.elasticsearch.xpack.ml.job.config.Detector; import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; -import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshotTests; -import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.notifications.Auditor; import org.junit.Before; import org.mockito.ArgumentCaptor; import org.mockito.Matchers; -import java.io.IOException; import java.util.Collections; import java.util.Date; @@ -163,28 +155,6 @@ public class JobManagerTests extends ESTestCase { }); } - public void testUpdateModelSnapshot() { - ArgumentCaptor indexRequestCaptor = ArgumentCaptor.forClass(IndexRequest.class); - doAnswer(invocationOnMock -> null).when(client).index(indexRequestCaptor.capture(), any()); - - ModelSnapshot modelSnapshot = ModelSnapshotTests.createRandomized(); - JobManager jobManager = createJobManager(); - - jobManager.updateModelSnapshot(new Result("snapshot-index", modelSnapshot), response -> {}, error -> {}); - - IndexRequest indexRequest = indexRequestCaptor.getValue(); - assertThat(indexRequest.index(), equalTo("snapshot-index")); - - // Assert snapshot was correctly serialised in the request by parsing it back and comparing to original - try (XContentParser parser = XContentFactory.xContent(indexRequest.source()).createParser(NamedXContentRegistry.EMPTY, - indexRequest.source())) { - ModelSnapshot requestSnapshot = ModelSnapshot.PARSER.apply(parser, null).build(); - assertThat(requestSnapshot, equalTo(modelSnapshot)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - private Job.Builder createJob() { Detector.Builder d1 = new Detector.Builder("info_content", "domain"); d1.setOverFieldName("client"); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java index 8a270c021f0..20e11a1c62e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java @@ -13,6 +13,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.equalTo; + public class MlFilterTests extends AbstractSerializingTestCase { @Override @@ -46,4 +48,7 @@ public class MlFilterTests extends AbstractSerializingTestCase { assertEquals(MlFilter.ITEMS.getPreferredName() + " must not be null", ex.getMessage()); } + public void testDocumentId() { + assertThat(MlFilter.documentId("foo"), equalTo("filter_foo")); + } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java index ad8f8a81525..19921d38bc9 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java @@ -33,30 +33,8 @@ import java.util.Set; public class ElasticsearchMappingsTests extends ESTestCase { - private void parseJson(JsonParser parser, Set expected) throws IOException { - try { - JsonToken token = parser.nextToken(); - while (token != null && token != JsonToken.END_OBJECT) { - switch (token) { - case START_OBJECT: - parseJson(parser, expected); - break; - case FIELD_NAME: - String fieldName = parser.getCurrentName(); - expected.add(fieldName); - break; - default: - break; - } - token = parser.nextToken(); - } - } catch (JsonParseException e) { - fail("Cannot parse JSON: " + e); - } - } - public void testReservedFields() - throws IOException, ClassNotFoundException, IllegalAccessException, IllegalArgumentException, InvocationTargetException { + public void testReservedFields() throws Exception { Set overridden = new HashSet<>(); // These are not reserved because they're Elasticsearch keywords, not @@ -78,30 +56,7 @@ public class ElasticsearchMappingsTests extends ESTestCase { overridden.add(ModelSnapshot.TYPE.getPreferredName()); overridden.add(Quantiles.TYPE.getPreferredName()); - Set expected = new HashSet<>(); - - // Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here. - - XContentBuilder builder = ElasticsearchMappings.resultsMapping(); - BufferedInputStream inputStream = - new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - JsonParser parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - - builder = ElasticsearchMappings.categoryDefinitionMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - - builder = ElasticsearchMappings.dataCountsMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); - - builder = ElasticsearchMappings.modelSnapshotMapping(); - inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); - parser = new JsonFactory().createParser(inputStream); - parseJson(parser, expected); + Set expected = collectResultsDocFieldNames(); expected.removeAll(overridden); @@ -147,4 +102,41 @@ public class ElasticsearchMappingsTests extends ESTestCase { instanceMapping = (Map) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName()); assertNull(instanceMapping); } + + private Set collectResultsDocFieldNames() throws IOException { + // Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here. + + XContentBuilder builder = ElasticsearchMappings.docMapping(); + BufferedInputStream inputStream = + new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8))); + JsonParser parser = new JsonFactory().createParser(inputStream); + Set fieldNames = new HashSet<>(); + boolean isAfterPropertiesStart = false; + try { + JsonToken token = parser.nextToken(); + while (token != null) { + switch (token) { + case START_OBJECT: + break; + case FIELD_NAME: + String fieldName = parser.getCurrentName(); + if (isAfterPropertiesStart) { + fieldNames.add(fieldName); + } else { + if (ElasticsearchMappings.PROPERTIES.equals(fieldName)) { + isAfterPropertiesStart = true; + } + } + break; + default: + break; + } + token = parser.nextToken(); + } + } catch (JsonParseException e) { + fail("Cannot parse JSON: " + e); + } + + return fieldNames; + } } diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml b/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml index d99248e2fe8..8ed548c8a46 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml @@ -28,7 +28,7 @@ setup: - do: index: index: .ml-anomalies-delete-model-snapshot - type: model_snapshot + type: doc id: "delete-model-snapshot_model_snapshot_inactive-snapshot" body: > { @@ -64,7 +64,7 @@ setup: - do: index: index: .ml-anomalies-delete-model-snapshot - type: model_snapshot + type: doc id: "delete-model-snapshot_model_snapshot_active-snapshot" body: > { diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml index 09d8bc8f07b..f2571e937e7 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml @@ -5,7 +5,7 @@ setup: index: index: .ml-meta type: doc - id: imposter-filter + id: filter_imposter-filter body: > { "filter_id": "imposter", diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml b/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml index c7dd2a0cd76..468f0229d08 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml @@ -19,7 +19,7 @@ - do: index: index: .ml-anomalies-shared - type: result + type: doc id: "new_doc" body: > { @@ -33,6 +33,6 @@ - do: indices.get_field_mapping: index: .ml-anomalies-shared - type: result + type: doc fields: new_field - - match: {\.ml-anomalies-shared.mappings.result.new_field.mapping.new_field.type: keyword} + - match: {\.ml-anomalies-shared.mappings.doc.new_field.mapping.new_field.type: keyword} diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml b/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml index 34fbc529550..497ea99e072 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml @@ -80,7 +80,7 @@ setup: - do: get: index: .ml-anomalies-post-data-job - type: data_counts + type: doc id: post-data-job_data_counts - match: { _source.processed_record_count: 2 } diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml b/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml index f040150d58d..49d08a3d741 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml @@ -28,7 +28,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: model_snapshot + type: doc id: "revert-model-snapshot_model_snapshot_first" body: > { @@ -53,7 +53,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: model_snapshot + type: doc id: "revert-model-snapshot_model_snapshot_second" body: > { @@ -78,7 +78,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1464825600000_1" body: > { @@ -91,7 +91,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1464782400000_1" body: > { @@ -104,7 +104,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1462060800000_1" body: > { @@ -117,7 +117,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1464825600000_1_1" body: > { @@ -130,7 +130,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1462060800000_1_2" body: > { @@ -143,7 +143,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1464825600000_1_3" body: { "job_id": "revert-model-snapshot", @@ -158,7 +158,7 @@ setup: - do: index: index: .ml-anomalies-revert-model-snapshot - type: result + type: doc id: "revert-model-snapshot_1462060800000_1_4" body: { diff --git a/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml b/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml index 104c27b0c6a..8dcc4e4dcc9 100644 --- a/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml +++ b/plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml @@ -16,7 +16,7 @@ setup: - do: index: index: .ml-anomalies-update-model-snapshot - type: model_snapshot + type: doc id: "update-model-snapshot_model_snapshot_snapshot-1" body: > { @@ -30,7 +30,7 @@ setup: - do: index: index: .ml-anomalies-update-model-snapshot - type: model_snapshot + type: doc id: "update-model-snapshot_model_snapshot_snapshot-2" body: > {