honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[ML] Single doc type for results index (elastic/x-pack-elasticsearch#1528) 

This commit changes all results to use the single doc type.
All searches were adjusted to work without the need to specify
type which ensures BWC with 5.4 results.

Additional work is needed to put the new doc mapping in indices
created with 5.4 but it will be done in separate PR.

Relates elastic/x-pack-elasticsearch#668

Original commit: elastic/x-pack-elasticsearch@041c88ac2d
This commit is contained in:
Dimitris Athanasiou 2017-05-24 13:24:32 +01:00 committed by GitHub
parent 3d057991e0
commit 71fe599592
31 changed files with 498 additions and 546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
plugin/src/main/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistry.java
View File

@ -30,12 +30,8 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
@ -220,8 +216,9 @@ public class MachineLearningTemplateRegistry extends AbstractComponent implemen
templateRequest.patterns(Collections.singletonList(MlMetaIndex.INDEX_NAME)); templateRequest.patterns(Collections.singletonList(MlMetaIndex.INDEX_NAME));
templateRequest.settings(mlNotificationIndexSettings()); templateRequest.settings(mlNotificationIndexSettings());
templateRequest.version(Version.CURRENT.id); templateRequest.version(Version.CURRENT.id);
try (XContentBuilder defaultMapping = ElasticsearchMappings.defaultMapping()) {
templateRequest.mapping(MapperService.DEFAULT_MAPPING, defaultMapping); try (XContentBuilder docMapping = MlMetaIndex.docMapping()) {
templateRequest.mapping(MlMetaIndex.TYPE, docMapping);
} catch (IOException e) { } catch (IOException e) {
String msg = "Error creating template mappings for the " + MlMetaIndex.INDEX_NAME + " index"; String msg = "Error creating template mappings for the " + MlMetaIndex.INDEX_NAME + " index";
logger.error(msg, e); logger.error(msg, e);
@ -255,20 +252,12 @@ public class MachineLearningTemplateRegistry extends AbstractComponent implemen
} }
void putJobResultsIndexTemplate(BiConsumer<Boolean, Exception> listener) { void putJobResultsIndexTemplate(BiConsumer<Boolean, Exception> listener) {
try (XContentBuilder defaultMapping = ElasticsearchMappings.defaultMapping(); try (XContentBuilder docMapping = ElasticsearchMappings.docMapping()) {
XContentBuilder resultsMapping = ElasticsearchMappings.resultsMapping();
XContentBuilder categoryDefinitionMapping = ElasticsearchMappings.categoryDefinitionMapping();
XContentBuilder dataCountsMapping = ElasticsearchMappings.dataCountsMapping();
XContentBuilder modelSnapshotMapping = ElasticsearchMappings.modelSnapshotMapping()) {
PutIndexTemplateRequest templateRequest = new PutIndexTemplateRequest(AnomalyDetectorsIndex.jobResultsIndexPrefix()); PutIndexTemplateRequest templateRequest = new PutIndexTemplateRequest(AnomalyDetectorsIndex.jobResultsIndexPrefix());
templateRequest.patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*")); templateRequest.patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"));
templateRequest.settings(mlResultsIndexSettings()); templateRequest.settings(mlResultsIndexSettings());
templateRequest.mapping(MapperService.DEFAULT_MAPPING, defaultMapping); templateRequest.mapping(ElasticsearchMappings.DOC_TYPE, docMapping);
templateRequest.mapping(Result.TYPE.getPreferredName(), resultsMapping);
templateRequest.mapping(CategoryDefinition.TYPE.getPreferredName(), categoryDefinitionMapping);
templateRequest.mapping(DataCounts.TYPE.getPreferredName(), dataCountsMapping);
templateRequest.mapping(ModelSnapshot.TYPE.getPreferredName(), modelSnapshotMapping);
templateRequest.version(Version.CURRENT.id); templateRequest.version(Version.CURRENT.id);
client.admin().indices().putTemplate(templateRequest, client.admin().indices().putTemplate(templateRequest,

17
plugin/src/main/java/org/elasticsearch/xpack/ml/MlMetaIndex.java
View File

@ -5,6 +5,13 @@
*/ */
package org.elasticsearch.xpack.ml; package org.elasticsearch.xpack.ml;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
public final class MlMetaIndex { public final class MlMetaIndex {
/** /**
* Where to store the ml info in Elasticsearch - must match what's * Where to store the ml info in Elasticsearch - must match what's
@ -15,4 +22,14 @@ public final class MlMetaIndex {
public static final String TYPE = "doc"; public static final String TYPE = "doc";
private MlMetaIndex() {} private MlMetaIndex() {}
public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(TYPE);
ElasticsearchMappings.addDefaultMapping(builder);
builder.endObject();
builder.endObject();
return builder;
}
} }

3
plugin/src/main/java/org/elasticsearch/xpack/ml/action/DeleteFilterAction.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.xpack.ml.MlMetaIndex;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.job.config.Detector; import org.elasticsearch.xpack.ml.job.config.Detector;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.config.MlFilter;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException; import java.io.IOException;
@ -178,7 +179,7 @@ public class DeleteFilterAction extends Action<DeleteFilterAction.Request, Delet
+ currentlyUsedBy); + currentlyUsedBy);
} }
DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId); DeleteRequest deleteRequest = new DeleteRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, MlFilter.documentId(filterId));
BulkRequest bulkRequest = new BulkRequest(); BulkRequest bulkRequest = new BulkRequest();
bulkRequest.add(deleteRequest); bulkRequest.add(deleteRequest);
transportAction.execute(bulkRequest, new ActionListener<BulkResponse>() { transportAction.execute(bulkRequest, new ActionListener<BulkResponse>() {

2
plugin/src/main/java/org/elasticsearch/xpack/ml/action/GetFiltersAction.java
View File

@ -245,7 +245,7 @@ public class GetFiltersAction extends Action<GetFiltersAction.Request, GetFilter
} }
private void getFilter(String filterId, ActionListener<Response> listener) { private void getFilter(String filterId, ActionListener<Response> listener) {
GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId); GetRequest getRequest = new GetRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, MlFilter.documentId(filterId));
transportGetAction.execute(getRequest, new ActionListener<GetResponse>() { transportGetAction.execute(getRequest, new ActionListener<GetResponse>() {
@Override @Override
public void onResponse(GetResponse getDocResponse) { public void onResponse(GetResponse getDocResponse) {

5
plugin/src/main/java/org/elasticsearch/xpack/ml/action/PutFilterAction.java
View File

@ -174,8 +174,7 @@ public class PutFilterAction extends Action<PutFilterAction.Request, PutFilterAc
@Override @Override
protected void doExecute(Request request, ActionListener<Response> listener) { protected void doExecute(Request request, ActionListener<Response> listener) {
MlFilter filter = request.getFilter(); MlFilter filter = request.getFilter();
final String filterId = filter.getId(); IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filter.documentId());
IndexRequest indexRequest = new IndexRequest(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId);
try (XContentBuilder builder = XContentFactory.jsonBuilder()) { try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
Payload.XContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlFilter.INCLUDE_TYPE_KEY, "true")); Payload.XContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(MlFilter.INCLUDE_TYPE_KEY, "true"));
indexRequest.source(filter.toXContent(builder, params)); indexRequest.source(filter.toXContent(builder, params));
@ -192,7 +191,7 @@ public class PutFilterAction extends Action<PutFilterAction.Request, PutFilterAc
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
listener.onFailure(new ResourceNotFoundException("Could not create filter with ID [" + filterId + "]", e)); listener.onFailure(new ResourceNotFoundException("Could not create filter with ID [" + filter.getId() + "]", e));
} }
}); });
} }

40
plugin/src/main/java/org/elasticsearch/xpack/ml/action/UpdateModelSnapshotAction.java
View File

@ -12,6 +12,10 @@ import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.bulk.TransportBulkAction;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
@ -26,13 +30,14 @@ import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.StatusToXContentObject;
import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.ml.job.JobManager;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.messages.Messages; import org.elasticsearch.xpack.ml.job.messages.Messages;
import org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.results.Result; import org.elasticsearch.xpack.ml.job.results.Result;
@ -40,6 +45,7 @@ import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import java.io.IOException; import java.io.IOException;
import java.util.Objects; import java.util.Objects;
import java.util.function.Consumer;
public class UpdateModelSnapshotAction extends Action<UpdateModelSnapshotAction.Request, public class UpdateModelSnapshotAction extends Action<UpdateModelSnapshotAction.Request,
UpdateModelSnapshotAction.Response, UpdateModelSnapshotAction.RequestBuilder> { UpdateModelSnapshotAction.Response, UpdateModelSnapshotAction.RequestBuilder> {
@ -257,15 +263,15 @@ public class UpdateModelSnapshotAction extends Action<UpdateModelSnapshotAction.
public static class TransportAction extends HandledTransportAction<Request, Response> { public static class TransportAction extends HandledTransportAction<Request, Response> {
private final JobManager jobManager;
private final JobProvider jobProvider; private final JobProvider jobProvider;
private final TransportBulkAction transportBulkAction;
@Inject @Inject
public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters, public TransportAction(Settings settings, TransportService transportService, ThreadPool threadPool, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, JobManager jobManager, JobProvider jobProvider) { IndexNameExpressionResolver indexNameExpressionResolver, JobProvider jobProvider, TransportBulkAction transportBulkAction) {
super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new); super(settings, NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver, Request::new);
this.jobManager = jobManager;
this.jobProvider = jobProvider; this.jobProvider = jobProvider;
this.transportBulkAction = transportBulkAction;
} }
@Override @Override
@ -277,7 +283,7 @@ public class UpdateModelSnapshotAction extends Action<UpdateModelSnapshotAction.
Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getSnapshotId(), request.getJobId()))); Messages.REST_NO_SUCH_MODEL_SNAPSHOT, request.getSnapshotId(), request.getJobId())));
} else { } else {
Result<ModelSnapshot> updatedSnapshot = applyUpdate(request, modelSnapshot); Result<ModelSnapshot> updatedSnapshot = applyUpdate(request, modelSnapshot);
jobManager.updateModelSnapshot(updatedSnapshot, b -> { indexModelSnapshot(updatedSnapshot, b -> {
// The quantiles can be large, and totally dominate the output - // The quantiles can be large, and totally dominate the output -
// it's clearer to remove them // it's clearer to remove them
listener.onResponse(new Response(new ModelSnapshot.Builder(updatedSnapshot.result).setQuantiles(null).build())); listener.onResponse(new Response(new ModelSnapshot.Builder(updatedSnapshot.result).setQuantiles(null).build()));
@ -296,5 +302,29 @@ public class UpdateModelSnapshotAction extends Action<UpdateModelSnapshotAction.
} }
return new Result(target.index, updatedSnapshotBuilder.build()); return new Result(target.index, updatedSnapshotBuilder.build());
} }
private void indexModelSnapshot(Result<ModelSnapshot> modelSnapshot, Consumer<Boolean> handler, Consumer<Exception> errorHandler) {
IndexRequest indexRequest = new IndexRequest(modelSnapshot.index, ElasticsearchMappings.DOC_TYPE,
ModelSnapshot.documentId(modelSnapshot.result));
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS);
indexRequest.source(builder);
} catch (IOException e) {
errorHandler.accept(e);
return;
}
BulkRequest bulkRequest = new BulkRequest().add(indexRequest);
transportBulkAction.execute(bulkRequest, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse indexResponse) {
handler.accept(true);
}
@Override
public void onFailure(Exception e) {
errorHandler.accept(e);
}
});
}
} }
} }

24
plugin/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job;
import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
@ -21,9 +20,6 @@ import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedConsumer;
import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
import org.elasticsearch.xpack.ml.action.DeleteJobAction; import org.elasticsearch.xpack.ml.action.DeleteJobAction;
import org.elasticsearch.xpack.ml.action.PutJobAction; import org.elasticsearch.xpack.ml.action.PutJobAction;
@ -38,12 +34,10 @@ import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister;
import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask; import org.elasticsearch.xpack.ml.job.persistence.JobStorageDeletionTask;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSizeStats;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.elasticsearch.xpack.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.xpack.persistent.PersistentTasksCustomMetaData;
import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
import java.util.List; import java.util.List;
@ -404,24 +398,6 @@ public class JobManager extends AbstractComponent {
}); });
} }
/**
* Update a persisted model snapshot metadata document to match the
* argument supplied.
*
* @param modelSnapshot the updated model snapshot object to be stored
*/
public void updateModelSnapshot(Result<ModelSnapshot> modelSnapshot, Consumer<Boolean> handler, Consumer<Exception> errorHandler) {
IndexRequest indexRequest = new IndexRequest(modelSnapshot.index, ModelSnapshot.TYPE.getPreferredName(),
ModelSnapshot.documentId(modelSnapshot.result));
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
modelSnapshot.result.toXContent(builder, ToXContent.EMPTY_PARAMS);
indexRequest.source(builder);
} catch (IOException e) {
errorHandler.accept(e);
}
client.index(indexRequest, ActionListener.wrap(r -> handler.accept(true), errorHandler));
}
private static MlMetadata.Builder createMlMetadataBuilder(ClusterState currentState) { private static MlMetadata.Builder createMlMetadataBuilder(ClusterState currentState) {
MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE); MlMetadata currentMlMetadata = currentState.metaData().custom(MlMetadata.TYPE);
return new MlMetadata.Builder(currentMlMetadata); return new MlMetadata.Builder(currentMlMetadata);

10
plugin/src/main/java/org/elasticsearch/xpack/ml/job/config/MlFilter.java
View File

@ -23,6 +23,8 @@ import java.util.Objects;
public class MlFilter extends ToXContentToBytes implements Writeable { public class MlFilter extends ToXContentToBytes implements Writeable {
public static final String DOCUMENT_ID_PREFIX = "filter_";
public static final String INCLUDE_TYPE_KEY = "include_type"; public static final String INCLUDE_TYPE_KEY = "include_type";
public static final String FILTER_TYPE = "filter"; public static final String FILTER_TYPE = "filter";
@ -99,6 +101,14 @@ public class MlFilter extends ToXContentToBytes implements Writeable {
return Objects.hash(id, items); return Objects.hash(id, items);
} }
public String documentId() {
return documentId(id);
}
public static String documentId(String filterId) {
return DOCUMENT_ID_PREFIX + filterId;
}
public static class Builder { public static class Builder {
private String id; private String id;

593
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappings.java
View File

@ -6,7 +6,6 @@
package org.elasticsearch.xpack.ml.job.persistence; package org.elasticsearch.xpack.ml.job.persistence;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts; import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
@ -52,6 +51,9 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
* using whitespace. * using whitespace.
*/ */
public class ElasticsearchMappings { public class ElasticsearchMappings {
public static final String DOC_TYPE = "doc";
/** /**
* String constants used in mappings * String constants used in mappings
*/ */
@ -97,25 +99,56 @@ public class ElasticsearchMappings {
* so that the per-job term fields will not be automatically added * so that the per-job term fields will not be automatically added
* as fields of type 'text' to the index mappings of newly rolled indices. * as fields of type 'text' to the index mappings of newly rolled indices.
* *
* @return The default mapping
* @throws IOException On write error * @throws IOException On write error
*/ */
public static XContentBuilder defaultMapping() throws IOException { public static void addDefaultMapping(XContentBuilder builder) throws IOException {
return jsonBuilder() builder.startArray("dynamic_templates")
.startObject() .startObject()
.startObject(MapperService.DEFAULT_MAPPING) .startObject("strings_as_keywords")
.startArray("dynamic_templates") .field("match", "*")
.startObject() .startObject("mapping")
.startObject("strings_as_keywords") .field(TYPE, KEYWORD)
.field("match", "*")
.startObject("mapping")
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject() .endObject()
.endArray() .endObject()
.endObject() .endObject()
.endObject(); .endArray();
}
public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
// Add result all field for easy searches in kibana
builder.startObject(ALL_FIELD_VALUES)
.field(TYPE, TEXT)
.field(ANALYZER, WHITESPACE)
.endObject();
builder.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.field(COPY_TO, ALL_FIELD_VALUES)
.endObject();
builder.startObject(Result.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject();
addResultsMapping(builder);
addCategoryDefinitionMapping(builder);
addDataCountsMapping(builder);
addModelSnapshotMapping(builder);
// end properties
builder.endObject();
// end mapping
builder.endObject();
// end doc
builder.endObject();
return builder;
} }
/** /**
@ -142,145 +175,120 @@ public class ElasticsearchMappings {
* <li>Influencer.influencer_field_value</li> * <li>Influencer.influencer_field_value</li>
* </ul> * </ul>
* *
* @return The mapping
* @throws IOException On write error * @throws IOException On write error
*/ */
public static XContentBuilder resultsMapping() throws IOException { private static void addResultsMapping(XContentBuilder builder) throws IOException {
XContentBuilder builder = jsonBuilder() builder.startObject(Result.RESULT_TYPE.getPreferredName())
.startObject() .field(TYPE, KEYWORD)
.startObject(Result.TYPE.getPreferredName()) .endObject()
.startObject(PROPERTIES) .startObject(Bucket.ANOMALY_SCORE.getPreferredName())
.startObject(ALL_FIELD_VALUES) .field(TYPE, DOUBLE)
.field(TYPE, TEXT) .endObject()
.field(ANALYZER, WHITESPACE) .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName())
.endObject() .field(TYPE, DOUBLE)
.startObject(Result.RESULT_TYPE.getPreferredName()) .endObject()
.field(TYPE, KEYWORD) .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName())
.endObject() .field(TYPE, DOUBLE)
.startObject(Job.ID.getPreferredName()) .endObject()
.field(TYPE, KEYWORD) .startObject(Result.IS_INTERIM.getPreferredName())
.field(COPY_TO, ALL_FIELD_VALUES) .field(TYPE, BOOLEAN)
.endObject() .endObject()
.startObject(Result.TIMESTAMP.getPreferredName()) .startObject(Bucket.RECORD_COUNT.getPreferredName())
.field(TYPE, DATE) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(Bucket.ANOMALY_SCORE.getPreferredName()) .startObject(Bucket.EVENT_COUNT.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) .startObject(Bucket.BUCKET_SPAN.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName()) .startObject(Bucket.PROCESSING_TIME_MS.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(Result.IS_INTERIM.getPreferredName()) .startObject(Bucket.PARTITION_SCORES.getPreferredName())
.field(TYPE, BOOLEAN) .field(TYPE, NESTED)
.endObject() .startObject(PROPERTIES)
.startObject(Bucket.RECORD_COUNT.getPreferredName()) .startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName())
.field(TYPE, LONG) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(Bucket.EVENT_COUNT.getPreferredName()) .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName())
.field(TYPE, LONG) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(Bucket.BUCKET_SPAN.getPreferredName()) .startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName())
.field(TYPE, LONG) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(Bucket.PROCESSING_TIME_MS.getPreferredName()) .startObject(AnomalyRecord.PROBABILITY.getPreferredName())
.field(TYPE, LONG) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(Bucket.PARTITION_SCORES.getPreferredName()) .endObject()
.field(TYPE, NESTED) .endObject()
.startObject(PROPERTIES)
.startObject(AnomalyRecord.PARTITION_FIELD_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Bucket.INITIAL_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.startObject(AnomalyRecord.PROBABILITY.getPreferredName())
.field(TYPE, DOUBLE)
.endObject()
.endObject()
.endObject()
.startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName()) .startObject(Bucket.BUCKET_INFLUENCERS.getPreferredName())
.field(TYPE, NESTED) .field(TYPE, NESTED)
.startObject(PROPERTIES) .startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName()) .startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(Result.RESULT_TYPE.getPreferredName()) .startObject(Result.RESULT_TYPE.getPreferredName())
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName()) .startObject(BucketInfluencer.INFLUENCER_FIELD_NAME.getPreferredName())
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName()) .startObject(BucketInfluencer.INITIAL_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName()) .startObject(BucketInfluencer.ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName()) .startObject(BucketInfluencer.RAW_ANOMALY_SCORE.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(BucketInfluencer.PROBABILITY.getPreferredName()) .startObject(BucketInfluencer.PROBABILITY.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(Result.TIMESTAMP.getPreferredName()) .startObject(Result.TIMESTAMP.getPreferredName())
.field(TYPE, DATE) .field(TYPE, DATE)
.endObject() .endObject()
.startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName()) .startObject(BucketInfluencer.BUCKET_SPAN.getPreferredName())
.field(TYPE, LONG) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(Result.IS_INTERIM.getPreferredName()) .startObject(Result.IS_INTERIM.getPreferredName())
.field(TYPE, BOOLEAN) .field(TYPE, BOOLEAN)
.endObject() .endObject()
.endObject() .endObject()
.endObject() .endObject()
// per-partition max probabilities mapping // per-partition max probabilities mapping
.startObject(PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName()) .startObject(PerPartitionMaxProbabilities.PER_PARTITION_MAX_PROBABILITIES.getPreferredName())
.field(TYPE, NESTED) .field(TYPE, NESTED)
.startObject(PROPERTIES) .startObject(PROPERTIES)
.startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName()) .startObject(AnomalyRecord.PARTITION_FIELD_VALUE.getPreferredName())
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(PerPartitionMaxProbabilities.MAX_RECORD_SCORE.getPreferredName()) .startObject(PerPartitionMaxProbabilities.MAX_RECORD_SCORE.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.endObject() .endObject()
.endObject() .endObject()
// Model Plot Output // Model Plot Output
.startObject(ModelPlot.MODEL_FEATURE.getPreferredName()) .startObject(ModelPlot.MODEL_FEATURE.getPreferredName())
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(ModelPlot.MODEL_LOWER.getPreferredName()) .startObject(ModelPlot.MODEL_LOWER.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(ModelPlot.MODEL_UPPER.getPreferredName()) .startObject(ModelPlot.MODEL_UPPER.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
.startObject(ModelPlot.MODEL_MEDIAN.getPreferredName()) .startObject(ModelPlot.MODEL_MEDIAN.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject(); .endObject();
addAnomalyRecordFieldsToMapping(builder); addAnomalyRecordFieldsToMapping(builder);
addInfluencerFieldsToMapping(builder); addInfluencerFieldsToMapping(builder);
addModelSizeStatsFieldsToMapping(builder); addModelSizeStatsFieldsToMapping(builder);
// End result properties
builder.endObject();
// End result
builder.endObject();
// End mapping
builder.endObject();
return builder;
} }
static XContentBuilder termFieldsMapping(String type, Collection<String> termFields) { static XContentBuilder termFieldsMapping(String type, Collection<String> termFields) {
@ -312,11 +320,9 @@ public class ElasticsearchMappings {
/** /**
* AnomalyRecord fields to be added under the 'properties' section of the mapping * AnomalyRecord fields to be added under the 'properties' section of the mapping
* @param builder Add properties to this builder * @param builder Add properties to this builder
* @return builder
* @throws IOException On write error * @throws IOException On write error
*/ */
private static XContentBuilder addAnomalyRecordFieldsToMapping(XContentBuilder builder) private static void addAnomalyRecordFieldsToMapping(XContentBuilder builder) throws IOException {
throws IOException {
builder.startObject(AnomalyRecord.DETECTOR_INDEX.getPreferredName()) builder.startObject(AnomalyRecord.DETECTOR_INDEX.getPreferredName())
.field(TYPE, INTEGER) .field(TYPE, INTEGER)
.endObject() .endObject()
@ -426,11 +432,9 @@ public class ElasticsearchMappings {
.endObject() .endObject()
.endObject() .endObject()
.endObject(); .endObject();
return builder;
} }
private static XContentBuilder addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException { private static void addInfluencerFieldsToMapping(XContentBuilder builder) throws IOException {
builder.startObject(Influencer.INFLUENCER_SCORE.getPreferredName()) builder.startObject(Influencer.INFLUENCER_SCORE.getPreferredName())
.field(TYPE, DOUBLE) .field(TYPE, DOUBLE)
.endObject() .endObject()
@ -444,8 +448,6 @@ public class ElasticsearchMappings {
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.field(COPY_TO, ALL_FIELD_VALUES) .field(COPY_TO, ALL_FIELD_VALUES)
.endObject(); .endObject();
return builder;
} }
/** /**
@ -453,68 +455,57 @@ public class ElasticsearchMappings {
* The type is disabled so {@link DataCounts} aren't searchable and * The type is disabled so {@link DataCounts} aren't searchable and
* the '_all' field is disabled * the '_all' field is disabled
* *
* @return The builder
* @throws IOException On builder write error * @throws IOException On builder write error
*/ */
public static XContentBuilder dataCountsMapping() throws IOException { private static void addDataCountsMapping(XContentBuilder builder) throws IOException {
return jsonBuilder() builder.startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName())
.startObject() .field(TYPE, LONG)
.startObject(DataCounts.TYPE.getPreferredName()) .endObject()
.startObject(PROPERTIES) .startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName())
.startObject(Job.ID.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, KEYWORD) .endObject()
.endObject() .startObject(DataCounts.INPUT_BYTES.getPreferredName())
.startObject(DataCounts.PROCESSED_RECORD_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName())
.startObject(DataCounts.PROCESSED_FIELD_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName())
.startObject(DataCounts.INPUT_BYTES.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName())
.startObject(DataCounts.INPUT_RECORD_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName())
.startObject(DataCounts.INPUT_FIELD_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName())
.startObject(DataCounts.INVALID_DATE_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.EMPTY_BUCKET_COUNT.getPreferredName())
.startObject(DataCounts.MISSING_FIELD_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.SPARSE_BUCKET_COUNT.getPreferredName())
.startObject(DataCounts.OUT_OF_ORDER_TIME_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.BUCKET_COUNT.getPreferredName())
.startObject(DataCounts.EMPTY_BUCKET_COUNT.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.EARLIEST_RECORD_TIME.getPreferredName())
.startObject(DataCounts.SPARSE_BUCKET_COUNT.getPreferredName()) .field(TYPE, DATE)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName())
.startObject(DataCounts.BUCKET_COUNT.getPreferredName()) .field(TYPE, DATE)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(DataCounts.LATEST_EMPTY_BUCKET_TIME.getPreferredName())
.startObject(DataCounts.EARLIEST_RECORD_TIME.getPreferredName()) .field(TYPE, DATE)
.field(TYPE, DATE) .endObject()
.endObject() .startObject(DataCounts.LATEST_SPARSE_BUCKET_TIME.getPreferredName())
.startObject(DataCounts.LATEST_RECORD_TIME.getPreferredName()) .field(TYPE, DATE)
.field(TYPE, DATE) .endObject()
.endObject() .startObject(DataCounts.LAST_DATA_TIME.getPreferredName())
.startObject(DataCounts.LATEST_EMPTY_BUCKET_TIME.getPreferredName()) .field(TYPE, DATE)
.field(TYPE, DATE) .endObject();
.endObject()
.startObject(DataCounts.LATEST_SPARSE_BUCKET_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(DataCounts.LAST_DATA_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject()
.endObject()
.endObject()
.endObject();
} }
/** /**
@ -554,35 +545,24 @@ public class ElasticsearchMappings {
* Create the Elasticsearch mapping for {@linkplain CategoryDefinition}. * Create the Elasticsearch mapping for {@linkplain CategoryDefinition}.
* The '_all' field is disabled as the document isn't meant to be searched. * The '_all' field is disabled as the document isn't meant to be searched.
* *
* @return The builder
* @throws IOException On builder error * @throws IOException On builder error
*/ */
public static XContentBuilder categoryDefinitionMapping() throws IOException { private static void addCategoryDefinitionMapping(XContentBuilder builder) throws IOException {
return jsonBuilder() builder.startObject(CategoryDefinition.CATEGORY_ID.getPreferredName())
.startObject() .field(TYPE, LONG)
.startObject(CategoryDefinition.TYPE.getPreferredName()) .endObject()
.startObject(PROPERTIES) .startObject(CategoryDefinition.TERMS.getPreferredName())
.startObject(CategoryDefinition.CATEGORY_ID.getPreferredName()) .field(TYPE, TEXT)
.field(TYPE, LONG) .endObject()
.endObject() .startObject(CategoryDefinition.REGEX.getPreferredName())
.startObject(Job.ID.getPreferredName()) .field(TYPE, KEYWORD)
.field(TYPE, KEYWORD) .endObject()
.endObject() .startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName())
.startObject(CategoryDefinition.TERMS.getPreferredName()) .field(TYPE, LONG)
.field(TYPE, TEXT) .endObject()
.endObject() .startObject(CategoryDefinition.EXAMPLES.getPreferredName())
.startObject(CategoryDefinition.REGEX.getPreferredName()) .field(TYPE, TEXT)
.field(TYPE, KEYWORD) .endObject();
.endObject()
.startObject(CategoryDefinition.MAX_MATCHING_LENGTH.getPreferredName())
.field(TYPE, LONG)
.endObject()
.startObject(CategoryDefinition.EXAMPLES.getPreferredName())
.field(TYPE, TEXT)
.endObject()
.endObject()
.endObject()
.endObject();
} }
/** /**
@ -605,91 +585,76 @@ public class ElasticsearchMappings {
* Create the Elasticsearch mapping for {@linkplain ModelSnapshot}. * Create the Elasticsearch mapping for {@linkplain ModelSnapshot}.
* The '_all' field is disabled but the type is searchable * The '_all' field is disabled but the type is searchable
*/ */
public static XContentBuilder modelSnapshotMapping() throws IOException { private static void addModelSnapshotMapping(XContentBuilder builder) throws IOException {
XContentBuilder builder = jsonBuilder() builder.startObject(ModelSnapshot.DESCRIPTION.getPreferredName())
.startObject() .field(TYPE, TEXT)
.startObject(ModelSnapshot.TYPE.getPreferredName()) .endObject()
.startObject(PROPERTIES) .startObject(ModelSnapshot.SNAPSHOT_ID.getPreferredName())
.startObject(Job.ID.getPreferredName()) .field(TYPE, KEYWORD)
.field(TYPE, KEYWORD) .endObject()
.endObject() .startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName())
.startObject(ModelSnapshot.TIMESTAMP.getPreferredName()) .field(TYPE, INTEGER)
.field(TYPE, DATE) .endObject()
.endObject() .startObject(ModelSnapshot.RETAIN.getPreferredName())
.startObject(ModelSnapshot.DESCRIPTION.getPreferredName()) .field(TYPE, BOOLEAN)
.field(TYPE, TEXT) .endObject()
.endObject() .startObject(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName())
.startObject(ModelSnapshot.SNAPSHOT_ID.getPreferredName()) .startObject(PROPERTIES)
.field(TYPE, KEYWORD) .startObject(Job.ID.getPreferredName())
.endObject() .field(TYPE, KEYWORD)
.startObject(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()) .endObject()
.field(TYPE, INTEGER) .startObject(Result.RESULT_TYPE.getPreferredName())
.endObject() .field(TYPE, KEYWORD)
.startObject(ModelSnapshot.RETAIN.getPreferredName()) .endObject()
.field(TYPE, BOOLEAN) .startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName())
.endObject() .field(TYPE, DATE)
.startObject(ModelSizeStats.RESULT_TYPE_FIELD.getPreferredName()) .endObject();
.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Result.RESULT_TYPE.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(ModelSizeStats.TIMESTAMP_FIELD.getPreferredName())
.field(TYPE, DATE)
.endObject();
addModelSizeStatsFieldsToMapping(builder); addModelSizeStatsFieldsToMapping(builder);
builder.endObject() // end model size stats properties
.endObject() builder.endObject();
.startObject(Quantiles.TYPE.getPreferredName()) // end model size stats mapping
.field(ENABLED, false) builder.endObject();
.endObject()
.startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject()
.endObject()
.endObject()
.endObject();
return builder; builder.startObject(Quantiles.TYPE.getPreferredName())
.field(ENABLED, false)
.endObject()
.startObject(ModelSnapshot.LATEST_RECORD_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(ModelSnapshot.LATEST_RESULT_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject();
} }
/** /**
* {@link ModelSizeStats} fields to be added under the 'properties' section of the mapping * {@link ModelSizeStats} fields to be added under the 'properties' section of the mapping
* @param builder Add properties to this builder * @param builder Add properties to this builder
* @return builder
* @throws IOException On write error * @throws IOException On write error
*/ */
private static XContentBuilder addModelSizeStatsFieldsToMapping(XContentBuilder builder) throws IOException { private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) throws IOException {
builder.startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName()) builder.startObject(ModelSizeStats.MODEL_BYTES_FIELD.getPreferredName())
.field(TYPE, LONG) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName()) .startObject(ModelSizeStats.TOTAL_BY_FIELD_COUNT_FIELD.getPreferredName())
.field(TYPE, LONG) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName()) .startObject(ModelSizeStats.TOTAL_OVER_FIELD_COUNT_FIELD.getPreferredName())
.field(TYPE, LONG) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName()) .startObject(ModelSizeStats.TOTAL_PARTITION_FIELD_COUNT_FIELD.getPreferredName())
.field(TYPE, LONG) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName()) .startObject(ModelSizeStats.BUCKET_ALLOCATION_FAILURES_COUNT_FIELD.getPreferredName())
.field(TYPE, LONG) .field(TYPE, LONG)
.endObject() .endObject()
.startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName()) .startObject(ModelSizeStats.MEMORY_STATUS_FIELD.getPreferredName())
.field(TYPE, KEYWORD) .field(TYPE, KEYWORD)
.endObject() .endObject()
.startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName()) .startObject(ModelSizeStats.LOG_TIME_FIELD.getPreferredName())
.field(TYPE, DATE) .field(TYPE, DATE)
.endObject(); .endObject();
return builder;
} }
public static XContentBuilder auditMessageMapping() throws IOException { public static XContentBuilder auditMessageMapping() throws IOException {

2
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataCountsPersister.java
View File

@ -47,7 +47,7 @@ public class JobDataCountsPersister extends AbstractComponent {
*/ */
public void persistDataCounts(String jobId, DataCounts counts, ActionListener<Boolean> listener) { public void persistDataCounts(String jobId, DataCounts counts, ActionListener<Boolean> listener) {
try (XContentBuilder content = serialiseCounts(counts)) { try (XContentBuilder content = serialiseCounts(counts)) {
client.prepareIndex(AnomalyDetectorsIndex.resultsWriteAlias(jobId), DataCounts.TYPE.getPreferredName(), client.prepareIndex(AnomalyDetectorsIndex.resultsWriteAlias(jobId), ElasticsearchMappings.DOC_TYPE,
DataCounts.documentId(jobId)) DataCounts.documentId(jobId))
.setSource(content).execute(new ActionListener<IndexResponse>() { .setSource(content).execute(new ActionListener<IndexResponse>() {
@Override @Override

18
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobDataDeleter.java
View File

@ -10,9 +10,6 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.WriteRequest;
@ -21,7 +18,9 @@ import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.index.query.ConstantScoreQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState;
@ -61,7 +60,7 @@ public class JobDataDeleter {
} }
bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsAliasedName(modelSnapshot.getJobId()), bulkRequestBuilder.add(client.prepareDelete(AnomalyDetectorsIndex.jobResultsAliasedName(modelSnapshot.getJobId()),
ModelSnapshot.TYPE.getPreferredName(), ModelSnapshot.documentId(modelSnapshot))); ElasticsearchMappings.DOC_TYPE, ModelSnapshot.documentId(modelSnapshot)));
} }
bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
@ -82,11 +81,11 @@ public class JobDataDeleter {
DeleteByQueryHolder deleteByQueryHolder = new DeleteByQueryHolder(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)); DeleteByQueryHolder deleteByQueryHolder = new DeleteByQueryHolder(AnomalyDetectorsIndex.jobResultsAliasedName(jobId));
deleteByQueryHolder.dbqRequest.setRefresh(true); deleteByQueryHolder.dbqRequest.setRefresh(true);
RangeQueryBuilder timeRange = QueryBuilders.rangeQuery(Result.TIMESTAMP.getPreferredName()); QueryBuilder query = QueryBuilders.boolQuery()
timeRange.gte(cutoffEpochMs); .filter(QueryBuilders.existsQuery(Result.RESULT_TYPE.getPreferredName()))
.filter(QueryBuilders.rangeQuery(Result.TIMESTAMP.getPreferredName()).gte(cutoffEpochMs));
deleteByQueryHolder.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); deleteByQueryHolder.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
deleteByQueryHolder.searchRequest.types(Result.TYPE.getPreferredName()); deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(query));
deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(timeRange));
client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryHolder.dbqRequest, new ActionListener<BulkByScrollResponse>() { client.execute(DeleteByQueryAction.INSTANCE, deleteByQueryHolder.dbqRequest, new ActionListener<BulkByScrollResponse>() {
@Override @Override
public void onResponse(BulkByScrollResponse bulkByScrollResponse) { public void onResponse(BulkByScrollResponse bulkByScrollResponse) {
@ -108,7 +107,6 @@ public class JobDataDeleter {
deleteByQueryHolder.dbqRequest.setRefresh(false); deleteByQueryHolder.dbqRequest.setRefresh(false);
deleteByQueryHolder.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); deleteByQueryHolder.searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
deleteByQueryHolder.searchRequest.types(Result.TYPE.getPreferredName());
QueryBuilder qb = QueryBuilders.termQuery(Result.IS_INTERIM.getPreferredName(), true); QueryBuilder qb = QueryBuilders.termQuery(Result.IS_INTERIM.getPreferredName(), true);
deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(new ConstantScoreQueryBuilder(qb))); deleteByQueryHolder.searchRequest.source(new SearchSourceBuilder().query(new ConstantScoreQueryBuilder(qb)));

71
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobProvider.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService;
@ -138,8 +139,9 @@ public class JobProvider {
if (!state.getMetaData().hasIndex(indexName)) { if (!state.getMetaData().hasIndex(indexName)) {
LOGGER.trace("ES API CALL: create index {}", indexName); LOGGER.trace("ES API CALL: create index {}", indexName);
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
String type = Result.TYPE.getPreferredName(); try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(ElasticsearchMappings.DOC_TYPE, termFields)) {
createIndexRequest.mapping(type, ElasticsearchMappings.termFieldsMapping(type, termFields)); createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, termFieldsMapping);
}
client.admin().indices().create(createIndexRequest, client.admin().indices().create(createIndexRequest,
ActionListener.wrap( ActionListener.wrap(
r -> createAliasListener.onResponse(r.isAcknowledged()), r -> createAliasListener.onResponse(r.isAcknowledged()),
@ -209,19 +211,21 @@ public class JobProvider {
} }
private void updateIndexMappingWithTermFields(String indexName, Collection<String> termFields, ActionListener<Boolean> listener) { private void updateIndexMappingWithTermFields(String indexName, Collection<String> termFields, ActionListener<Boolean> listener) {
client.admin().indices().preparePutMapping(indexName).setType(Result.TYPE.getPreferredName()) try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(null, termFields)) {
.setSource(ElasticsearchMappings.termFieldsMapping(null, termFields)) client.admin().indices().preparePutMapping(indexName).setType(ElasticsearchMappings.DOC_TYPE)
.execute(new ActionListener<PutMappingResponse>() { .setSource(termFieldsMapping)
@Override .execute(new ActionListener<PutMappingResponse>() {
public void onResponse(PutMappingResponse putMappingResponse) { @Override
listener.onResponse(putMappingResponse.isAcknowledged()); public void onResponse(PutMappingResponse putMappingResponse) {
} listener.onResponse(putMappingResponse.isAcknowledged());
}
@Override @Override
public void onFailure(Exception e) { public void onFailure(Exception e) {
listener.onFailure(e); listener.onFailure(e);
} }
}); });
}
} }
/** /**
@ -251,12 +255,11 @@ public class JobProvider {
MultiSearchRequestBuilder msearch = client.prepareMultiSearch() MultiSearchRequestBuilder msearch = client.prepareMultiSearch()
.add(createLatestDataCountsSearch(resultsIndex, jobId)) .add(createLatestDataCountsSearch(resultsIndex, jobId))
.add(createLatestModelSizeStatsSearch(resultsIndex)) .add(createLatestModelSizeStatsSearch(resultsIndex))
.add(createDocIdSearch(resultsIndex, ModelSnapshot.TYPE.getPreferredName(), .add(createDocIdSearch(resultsIndex, ModelSnapshot.documentId(jobId, job.getModelSnapshotId())))
ModelSnapshot.documentId(jobId, job.getModelSnapshotId()))) .add(createDocIdSearch(stateIndex, Quantiles.documentId(jobId)));
.add(createDocIdSearch(stateIndex, Quantiles.TYPE.getPreferredName(), Quantiles.documentId(jobId)));
for (String filterId : job.getAnalysisConfig().extractReferencedFilters()) { for (String filterId : job.getAnalysisConfig().extractReferencedFilters()) {
msearch.add(createDocIdSearch(MlMetaIndex.INDEX_NAME, MlMetaIndex.TYPE, filterId)); msearch.add(createDocIdSearch(MlMetaIndex.INDEX_NAME, filterId));
} }
msearch.execute(ActionListener.wrap( msearch.execute(ActionListener.wrap(
@ -284,7 +287,7 @@ public class JobProvider {
SearchRequest searchRequest = msearch.request().requests().get(i); SearchRequest searchRequest = msearch.request().requests().get(i);
LOGGER.debug("Found 0 hits for [{}/{}]", searchRequest.indices(), searchRequest.types()); LOGGER.debug("Found 0 hits for [{}/{}]", searchRequest.indices(), searchRequest.types());
} else if (hitsCount == 1) { } else if (hitsCount == 1) {
parseAutodetectParamSearchHit(paramsBuilder, hits.getAt(0), errorHandler); parseAutodetectParamSearchHit(jobId, paramsBuilder, hits.getAt(0), errorHandler);
} else if (hitsCount > 1) { } else if (hitsCount > 1) {
errorHandler.accept(new IllegalStateException("Expected hits count to be 0 or 1, but got [" errorHandler.accept(new IllegalStateException("Expected hits count to be 0 or 1, but got ["
+ hitsCount + "]")); + hitsCount + "]"));
@ -298,30 +301,30 @@ public class JobProvider {
)); ));
} }
private SearchRequestBuilder createDocIdSearch(String index, String type, String id) { private SearchRequestBuilder createDocIdSearch(String index, String id) {
return client.prepareSearch(index).setSize(1) return client.prepareSearch(index).setSize(1)
.setIndicesOptions(IndicesOptions.lenientExpandOpen()) .setIndicesOptions(IndicesOptions.lenientExpandOpen())
.setQuery(QueryBuilders.idsQuery(type).addIds(id)) .setQuery(QueryBuilders.idsQuery().addIds(id))
.setRouting(id); .setRouting(id);
} }
private void parseAutodetectParamSearchHit(AutodetectParams.Builder paramsBuilder, private void parseAutodetectParamSearchHit(String jobId, AutodetectParams.Builder paramsBuilder, SearchHit hit,
SearchHit hit, Consumer<Exception> errorHandler) { Consumer<Exception> errorHandler) {
String type = hit.getType(); String hitId = hit.getId();
if (DataCounts.TYPE.getPreferredName().equals(type)) { if (DataCounts.documentId(jobId).equals(hitId)) {
paramsBuilder.setDataCounts(parseSearchHit(hit, DataCounts.PARSER, errorHandler)); paramsBuilder.setDataCounts(parseSearchHit(hit, DataCounts.PARSER, errorHandler));
} else if (Result.TYPE.getPreferredName().equals(type)) { } else if (hitId.startsWith(ModelSizeStats.documentIdPrefix(jobId))) {
ModelSizeStats.Builder modelSizeStats = parseSearchHit(hit, ModelSizeStats.PARSER, errorHandler); ModelSizeStats.Builder modelSizeStats = parseSearchHit(hit, ModelSizeStats.PARSER, errorHandler);
paramsBuilder.setModelSizeStats(modelSizeStats == null ? null : modelSizeStats.build()); paramsBuilder.setModelSizeStats(modelSizeStats == null ? null : modelSizeStats.build());
} else if (ModelSnapshot.TYPE.getPreferredName().equals(type)) { } else if (hitId.startsWith(ModelSnapshot.documentIdPrefix(jobId))) {
ModelSnapshot.Builder modelSnapshot = parseSearchHit(hit, ModelSnapshot.PARSER, errorHandler); ModelSnapshot.Builder modelSnapshot = parseSearchHit(hit, ModelSnapshot.PARSER, errorHandler);
paramsBuilder.setModelSnapshot(modelSnapshot == null ? null : modelSnapshot.build()); paramsBuilder.setModelSnapshot(modelSnapshot == null ? null : modelSnapshot.build());
} else if (Quantiles.TYPE.getPreferredName().equals(type)) { } else if (Quantiles.documentId(jobId).equals(hit.getId())) {
paramsBuilder.setQuantiles(parseSearchHit(hit, Quantiles.PARSER, errorHandler)); paramsBuilder.setQuantiles(parseSearchHit(hit, Quantiles.PARSER, errorHandler));
} else if (MlFilter.TYPE.getPreferredName().equals(type)) { } else if (hitId.startsWith(MlFilter.DOCUMENT_ID_PREFIX)) {
paramsBuilder.addFilter(parseSearchHit(hit, MlFilter.PARSER, errorHandler).build()); paramsBuilder.addFilter(parseSearchHit(hit, MlFilter.PARSER, errorHandler).build());
} else { } else {
errorHandler.accept(new IllegalStateException("Unexpected type [" + type + "]")); errorHandler.accept(new IllegalStateException("Unexpected type [" + hit.getType() + "]"));
} }
} }
@ -644,7 +647,6 @@ public class JobProvider {
SearchRequest searchRequest = new SearchRequest(indexName); SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions())); searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions()));
searchRequest.types(Result.TYPE.getPreferredName());
searchRequest.source(new SearchSourceBuilder() searchRequest.source(new SearchSourceBuilder()
.from(from) .from(from)
.size(size) .size(size)
@ -703,7 +705,6 @@ public class JobProvider {
SearchRequest searchRequest = new SearchRequest(indexName); SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions())); searchRequest.indicesOptions(addIgnoreUnavailable(searchRequest.indicesOptions()));
searchRequest.types(Result.TYPE.getPreferredName());
FieldSortBuilder sb = query.getSortField() == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC) FieldSortBuilder sb = query.getSortField() == null ? SortBuilders.fieldSort(ElasticsearchMappings.ES_DOC)
: new FieldSortBuilder(query.getSortField()).order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC); : new FieldSortBuilder(query.getSortField()).order(query.isSortDescending() ? SortOrder.DESC : SortOrder.ASC);
searchRequest.source(new SearchSourceBuilder().query(qb).from(query.getFrom()).size(query.getSize()).sort(sb)); searchRequest.source(new SearchSourceBuilder().query(qb).from(query.getFrom()).size(query.getSize()).sort(sb));
@ -744,8 +745,7 @@ public class JobProvider {
return; return;
} }
String resultsIndex = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); String resultsIndex = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
SearchRequestBuilder search = createDocIdSearch(resultsIndex, ModelSnapshot.TYPE.getPreferredName(), SearchRequestBuilder search = createDocIdSearch(resultsIndex, ModelSnapshot.documentId(jobId, modelSnapshotId));
ModelSnapshot.documentId(jobId, modelSnapshotId));
searchSingleResult(jobId, ModelSnapshot.TYPE.getPreferredName(), search, ModelSnapshot.PARSER, searchSingleResult(jobId, ModelSnapshot.TYPE.getPreferredName(), search, ModelSnapshot.PARSER,
result -> handler.accept(result.result == null ? null : new Result(result.index, result.result.build())), result -> handler.accept(result.result == null ? null : new Result(result.index, result.result.build())),
errorHandler, () -> null); errorHandler, () -> null);
@ -814,7 +814,7 @@ public class JobProvider {
.order(sortDescending ? SortOrder.DESC : SortOrder.ASC); .order(sortDescending ? SortOrder.DESC : SortOrder.ASC);
String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId); String indexName = AnomalyDetectorsIndex.jobResultsAliasedName(jobId);
LOGGER.trace("ES API CALL: search all of type {} from index {} sort ascending {} with filter after sort from {} size {}", LOGGER.trace("ES API CALL: search all {}s from index {} sort ascending {} with filter after sort from {} size {}",
ModelSnapshot.TYPE, indexName, sortField, from, size); ModelSnapshot.TYPE, indexName, sortField, from, size);
SearchRequest searchRequest = new SearchRequest(indexName); SearchRequest searchRequest = new SearchRequest(indexName);
@ -909,7 +909,6 @@ public class JobProvider {
searchResponse = client.prepareSearch(indexName) searchResponse = client.prepareSearch(indexName)
.setIndicesOptions(addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS)) .setIndicesOptions(addIgnoreUnavailable(SearchRequest.DEFAULT_INDICES_OPTIONS))
.setTypes(Result.TYPE.getPreferredName())
.setQuery(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), ModelPlot.RESULT_TYPE_VALUE)) .setQuery(new TermsQueryBuilder(Result.RESULT_TYPE.getPreferredName(), ModelPlot.RESULT_TYPE_VALUE))
.setFrom(from).setSize(size) .setFrom(from).setSize(size)
.get(); .get();

4
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobRenormalizedResultsPersister.java
View File

@ -17,12 +17,12 @@ import org.elasticsearch.xpack.ml.job.process.normalizer.BucketNormalizable;
import org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable; import org.elasticsearch.xpack.ml.job.process.normalizer.Normalizable;
import org.elasticsearch.xpack.ml.job.results.Bucket; import org.elasticsearch.xpack.ml.job.results.Bucket;
import org.elasticsearch.xpack.ml.job.results.BucketInfluencer; import org.elasticsearch.xpack.ml.job.results.BucketInfluencer;
import org.elasticsearch.xpack.ml.job.results.Result;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings.DOC_TYPE;
/** /**
@ -66,7 +66,7 @@ public class JobRenormalizedResultsPersister extends AbstractComponent {
public void updateResult(String id, String index, ToXContent resultDoc) { public void updateResult(String id, String index, ToXContent resultDoc) {
try (XContentBuilder content = toXContentBuilder(resultDoc)) { try (XContentBuilder content = toXContentBuilder(resultDoc)) {
bulkRequest.add(new IndexRequest(index, Result.TYPE.getPreferredName(), id).source(content)); bulkRequest.add(new IndexRequest(index, DOC_TYPE, id).source(content));
} catch (IOException e) { } catch (IOException e) {
logger.error("Error serialising result", e); logger.error("Error serialising result", e);
} }

28
plugin/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsPersister.java
View File

@ -39,6 +39,7 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.xpack.ml.job.persistence.ElasticsearchMappings.DOC_TYPE;
/** /**
* Persists result types, Quantiles etc to Elasticsearch<br> * Persists result types, Quantiles etc to Elasticsearch<br>
@ -60,7 +61,6 @@ public class JobResultsPersister extends AbstractComponent {
private final Client client; private final Client client;
public JobResultsPersister(Settings settings, Client client) { public JobResultsPersister(Settings settings, Client client) {
super(settings); super(settings);
this.client = client; this.client = client;
@ -100,8 +100,7 @@ public class JobResultsPersister extends AbstractComponent {
logger.trace("[{}] ES API CALL: index result type {} to index {} at epoch {}", logger.trace("[{}] ES API CALL: index result type {} to index {} at epoch {}",
jobId, Bucket.RESULT_TYPE_VALUE, indexName, bucketWithoutRecords.getEpoch()); jobId, Bucket.RESULT_TYPE_VALUE, indexName, bucketWithoutRecords.getEpoch());
bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, bucketWithoutRecords.getId()).source(content));
bucketWithoutRecords.getId()).source(content));
persistBucketInfluencersStandalone(jobId, bucketWithoutRecords.getBucketInfluencers()); persistBucketInfluencersStandalone(jobId, bucketWithoutRecords.getBucketInfluencers());
} catch (IOException e) { } catch (IOException e) {
@ -120,7 +119,7 @@ public class JobResultsPersister extends AbstractComponent {
String id = bucketInfluencer.getId(); String id = bucketInfluencer.getId();
logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}",
jobId, BucketInfluencer.RESULT_TYPE_VALUE, indexName, id); jobId, BucketInfluencer.RESULT_TYPE_VALUE, indexName, id);
bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), id).source(content)); bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, id).source(content));
} }
} }
} }
@ -139,7 +138,7 @@ public class JobResultsPersister extends AbstractComponent {
try (XContentBuilder content = toXContentBuilder(record)) { try (XContentBuilder content = toXContentBuilder(record)) {
logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}",
jobId, AnomalyRecord.RESULT_TYPE_VALUE, indexName, record.getId()); jobId, AnomalyRecord.RESULT_TYPE_VALUE, indexName, record.getId());
bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), record.getId()).source(content)); bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, record.getId()).source(content));
} }
} }
} catch (IOException e) { } catch (IOException e) {
@ -162,7 +161,7 @@ public class JobResultsPersister extends AbstractComponent {
try (XContentBuilder content = toXContentBuilder(influencer)) { try (XContentBuilder content = toXContentBuilder(influencer)) {
logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}", logger.trace("[{}] ES BULK ACTION: index result type {} to index {} with ID {}",
jobId, Influencer.RESULT_TYPE_VALUE, indexName, influencer.getId()); jobId, Influencer.RESULT_TYPE_VALUE, indexName, influencer.getId());
bulkRequest.add(new IndexRequest(indexName, Result.TYPE.getPreferredName(), influencer.getId()).source(content)); bulkRequest.add(new IndexRequest(indexName, DOC_TYPE, influencer.getId()).source(content));
} }
} }
} catch (IOException e) { } catch (IOException e) {
@ -184,7 +183,7 @@ public class JobResultsPersister extends AbstractComponent {
jobId, PerPartitionMaxProbabilities.RESULT_TYPE_VALUE, indexName, partitionProbabilities.getTimestamp(), jobId, PerPartitionMaxProbabilities.RESULT_TYPE_VALUE, indexName, partitionProbabilities.getTimestamp(),
partitionProbabilities.getId()); partitionProbabilities.getId());
bulkRequest.add( bulkRequest.add(
new IndexRequest(indexName, Result.TYPE.getPreferredName(), partitionProbabilities.getId()).source(builder)); new IndexRequest(indexName, DOC_TYPE, partitionProbabilities.getId()).source(builder));
} catch (IOException e) { } catch (IOException e) {
logger.error(new ParameterizedMessage("[{}] error serialising bucket per partition max normalized scores", logger.error(new ParameterizedMessage("[{}] error serialising bucket per partition max normalized scores",
@ -349,14 +348,14 @@ public class JobResultsPersister extends AbstractComponent {
private final String jobId; private final String jobId;
private final ToXContent object; private final ToXContent object;
private final String type; private final String description;
private final String id; private final String id;
private WriteRequest.RefreshPolicy refreshPolicy; private WriteRequest.RefreshPolicy refreshPolicy;
Persistable(String jobId, ToXContent object, String type, String id) { Persistable(String jobId, ToXContent object, String description, String id) {
this.jobId = jobId; this.jobId = jobId;
this.object = object; this.object = object;
this.type = type; this.description = description;
this.id = id; this.id = id;
this.refreshPolicy = WriteRequest.RefreshPolicy.NONE; this.refreshPolicy = WriteRequest.RefreshPolicy.NONE;
} }
@ -373,12 +372,15 @@ public class JobResultsPersister extends AbstractComponent {
void persist(String indexName, ActionListener<IndexResponse> listener) { void persist(String indexName, ActionListener<IndexResponse> listener) {
logCall(indexName); logCall(indexName);
// TODO no_release: this is a temporary hack until we also switch state index to have doc type in which case
// we can remove this line and use DOC_TYPE directly in the index request
String type = AnomalyDetectorsIndex.jobStateIndexName().equals(indexName) ? description : DOC_TYPE;
try (XContentBuilder content = toXContentBuilder(object)) { try (XContentBuilder content = toXContentBuilder(object)) {
IndexRequest indexRequest = new IndexRequest(indexName, type, id).source(content).setRefreshPolicy(refreshPolicy); IndexRequest indexRequest = new IndexRequest(indexName, type, id).source(content).setRefreshPolicy(refreshPolicy);
client.index(indexRequest, listener); client.index(indexRequest, listener);
} catch (IOException e) { } catch (IOException e) {
logger.error(new ParameterizedMessage("[{}] Error writing {}", new Object[]{jobId, type}), e); logger.error(new ParameterizedMessage("[{}] Error writing {}", new Object[]{jobId, description}), e);
IndexResponse.Builder notCreatedResponse = new IndexResponse.Builder(); IndexResponse.Builder notCreatedResponse = new IndexResponse.Builder();
notCreatedResponse.setCreated(false); notCreatedResponse.setCreated(false);
listener.onResponse(notCreatedResponse.build()); listener.onResponse(notCreatedResponse.build());
@ -387,9 +389,9 @@ public class JobResultsPersister extends AbstractComponent {
private void logCall(String indexName) { private void logCall(String indexName) {
if (id != null) { if (id != null) {
logger.trace("[{}] ES API CALL: index type {} to index {} with ID {}", jobId, type, indexName, id); logger.trace("[{}] ES API CALL: index {} to index {} with ID {}", jobId, description, indexName, id);
} else { } else {
logger.trace("[{}] ES API CALL: index type {} to index {} with auto-generated ID", jobId, type, indexName); logger.trace("[{}] ES API CALL: index {} to index {} with auto-generated ID", jobId, description, indexName);
} }
} }
} }

6
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSizeStats.java
View File

@ -143,7 +143,11 @@ public class ModelSizeStats extends ToXContentToBytes implements Writeable {
} }
public String getId() { public String getId() {
return jobId + "_model_size_stats_" + logTime.getTime(); return documentIdPrefix(jobId) + logTime.getTime();
}
public static String documentIdPrefix(String jobId) {
return jobId + "_model_size_stats_";
} }
@Override @Override

6
plugin/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/state/ModelSnapshot.java
View File

@ -274,12 +274,16 @@ public class ModelSnapshot extends ToXContentToBytes implements Writeable {
return stateDocumentIds; return stateDocumentIds;
} }
public static String documentIdPrefix(String jobId) {
return jobId + "_model_snapshot_";
}
public static String documentId(ModelSnapshot snapshot) { public static String documentId(ModelSnapshot snapshot) {
return documentId(snapshot.getJobId(), snapshot.getSnapshotId()); return documentId(snapshot.getJobId(), snapshot.getSnapshotId());
} }
public static String documentId(String jobId, String snapshotId) { public static String documentId(String jobId, String snapshotId) {
return jobId + "_model_snapshot_" + snapshotId; return documentIdPrefix(jobId) + snapshotId;
} }
public static ModelSnapshot fromJson(BytesReference bytesReference) { public static ModelSnapshot fromJson(BytesReference bytesReference) {

2
plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredModelSnapshotsRemover.java
View File

@ -67,12 +67,12 @@ public class ExpiredModelSnapshotsRemover extends AbstractExpiredJobDataRemover
SearchRequest searchRequest = new SearchRequest(); SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId())); searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()));
searchRequest.types(ModelSnapshot.TYPE.getPreferredName());
QueryBuilder activeSnapshotFilter = QueryBuilders.termQuery( QueryBuilder activeSnapshotFilter = QueryBuilders.termQuery(
ModelSnapshot.SNAPSHOT_ID.getPreferredName(), job.getModelSnapshotId()); ModelSnapshot.SNAPSHOT_ID.getPreferredName(), job.getModelSnapshotId());
QueryBuilder retainFilter = QueryBuilders.termQuery(ModelSnapshot.RETAIN.getPreferredName(), true); QueryBuilder retainFilter = QueryBuilders.termQuery(ModelSnapshot.RETAIN.getPreferredName(), true);
QueryBuilder query = createQuery(job.getId(), cutoffEpochMs) QueryBuilder query = createQuery(job.getId(), cutoffEpochMs)
.filter(QueryBuilders.existsQuery(ModelSnapshot.SNAPSHOT_DOC_COUNT.getPreferredName()))
.mustNot(activeSnapshotFilter) .mustNot(activeSnapshotFilter)
.mustNot(retainFilter); .mustNot(retainFilter);

5
plugin/src/main/java/org/elasticsearch/xpack/ml/job/retention/ExpiredResultsRemover.java
View File

@ -86,9 +86,10 @@ public class ExpiredResultsRemover extends AbstractExpiredJobDataRemover {
request.setSlices(5); request.setSlices(5);
searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId())); searchRequest.indices(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()));
searchRequest.types(Result.TYPE.getPreferredName());
QueryBuilder excludeFilter = QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), ModelSizeStats.RESULT_TYPE_VALUE); QueryBuilder excludeFilter = QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), ModelSizeStats.RESULT_TYPE_VALUE);
QueryBuilder query = createQuery(job.getId(), cutoffEpochMs).mustNot(excludeFilter); QueryBuilder query = createQuery(job.getId(), cutoffEpochMs)
.filter(QueryBuilders.existsQuery(Result.RESULT_TYPE.getPreferredName()))
.mustNot(excludeFilter);
searchRequest.source(new SearchSourceBuilder().query(query)); searchRequest.source(new SearchSourceBuilder().query(query));
return request; return request;
} }

14
plugin/src/test/java/org/elasticsearch/xpack/ml/MachineLearningTemplateRegistryTests.java
View File

@ -27,12 +27,8 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder; import org.elasticsearch.xpack.ml.job.persistence.MockClientBuilder;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.CategorizerState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.DataCounts;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState; import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelState;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.notifications.AuditMessage; import org.elasticsearch.xpack.ml.notifications.AuditMessage;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.junit.Before; import org.junit.Before;
@ -221,7 +217,7 @@ public class MachineLearningTemplateRegistryTests extends ESTestCase {
assertNotNull(request); assertNotNull(request);
assertEquals(templateRegistry.mlNotificationIndexSettings().build(), request.settings()); assertEquals(templateRegistry.mlNotificationIndexSettings().build(), request.settings());
assertEquals(1, request.mappings().size()); assertEquals(1, request.mappings().size());
assertThat(request.mappings().containsKey(MapperService.DEFAULT_MAPPING), is(true)); assertThat(request.mappings().containsKey("doc"), is(true));
assertEquals(Collections.singletonList(MlMetaIndex.INDEX_NAME), request.patterns()); assertEquals(Collections.singletonList(MlMetaIndex.INDEX_NAME), request.patterns());
assertEquals(new Integer(Version.CURRENT.id), request.version()); assertEquals(new Integer(Version.CURRENT.id), request.version());
}); });
@ -262,12 +258,8 @@ public class MachineLearningTemplateRegistryTests extends ESTestCase {
PutIndexTemplateRequest request = captor.getValue(); PutIndexTemplateRequest request = captor.getValue();
assertNotNull(request); assertNotNull(request);
assertEquals(templateRegistry.mlResultsIndexSettings().build(), request.settings()); assertEquals(templateRegistry.mlResultsIndexSettings().build(), request.settings());
assertTrue(request.mappings().containsKey("_default_")); assertTrue(request.mappings().containsKey("doc"));
assertTrue(request.mappings().containsKey(Result.TYPE.getPreferredName())); assertEquals(1, request.mappings().size());
assertTrue(request.mappings().containsKey(CategoryDefinition.TYPE.getPreferredName()));
assertTrue(request.mappings().containsKey(DataCounts.TYPE.getPreferredName()));
assertTrue(request.mappings().containsKey(ModelSnapshot.TYPE.getPreferredName()));
assertEquals(5, request.mappings().size());
assertEquals(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"), request.patterns()); assertEquals(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"), request.patterns());
assertEquals(new Integer(Version.CURRENT.id), request.version()); assertEquals(new Integer(Version.CURRENT.id), request.version());
}); });

6
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectResultProcessorIT.java
View File

@ -150,9 +150,9 @@ public class AutodetectResultProcessorIT extends XPackSingleNodeTestCase {
assertEquals(1, persistedDefinition.count()); assertEquals(1, persistedDefinition.count());
assertEquals(categoryDefinition, persistedDefinition.results().get(0)); assertEquals(categoryDefinition, persistedDefinition.results().get(0));
QueryPage<ModelPlot> persistedmodelPlot = jobProvider.modelPlot(JOB_ID, 0, 100); QueryPage<ModelPlot> persistedModelPlot = jobProvider.modelPlot(JOB_ID, 0, 100);
assertEquals(1, persistedmodelPlot.count()); assertEquals(1, persistedModelPlot.count());
assertEquals(modelPlot, persistedmodelPlot.results().get(0)); assertEquals(modelPlot, persistedModelPlot.results().get(0));
ModelSizeStats persistedModelSizeStats = getModelSizeStats(); ModelSizeStats persistedModelSizeStats = getModelSizeStats();
assertEquals(modelSizeStats, persistedModelSizeStats); assertEquals(modelSizeStats, persistedModelSizeStats);

9
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
View File

@ -113,7 +113,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
// Update snapshot timestamp to force it out of snapshot retention window // Update snapshot timestamp to force it out of snapshot retention window
String snapshotUpdate = "{ \"timestamp\": " + oneDayAgo + "}"; String snapshotUpdate = "{ \"timestamp\": " + oneDayAgo + "}";
UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), "model_snapshot", snapshotDocId); UpdateRequest updateSnapshotRequest = new UpdateRequest(".ml-anomalies-" + job.getId(), "doc", snapshotDocId);
updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON); updateSnapshotRequest.doc(snapshotUpdate.getBytes(StandardCharsets.UTF_8), XContentType.JSON);
client().execute(UpdateAction.INSTANCE, updateSnapshotRequest).get(); client().execute(UpdateAction.INSTANCE, updateSnapshotRequest).get();
} }
@ -136,7 +136,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
retainAllSnapshots("snapshots-retention-with-retain"); retainAllSnapshots("snapshots-retention-with-retain");
long totalModelSizeStatsBeforeDelete = client().prepareSearch("*").setTypes("result") long totalModelSizeStatsBeforeDelete = client().prepareSearch("*")
.setQuery(QueryBuilders.termQuery("result_type", "model_size_stats")) .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
.get().getHits().totalHits; .get().getHits().totalHits;
long totalNotificationsCountBeforeDelete = client().prepareSearch(".ml-notifications").get().getHits().totalHits; long totalNotificationsCountBeforeDelete = client().prepareSearch(".ml-notifications").get().getHits().totalHits;
@ -175,7 +175,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
assertThat(getRecords("results-and-snapshots-retention").size(), equalTo(0)); assertThat(getRecords("results-and-snapshots-retention").size(), equalTo(0));
assertThat(getModelSnapshots("results-and-snapshots-retention").size(), equalTo(1)); assertThat(getModelSnapshots("results-and-snapshots-retention").size(), equalTo(1));
long totalModelSizeStatsAfterDelete = client().prepareSearch("*").setTypes("result") long totalModelSizeStatsAfterDelete = client().prepareSearch("*")
.setQuery(QueryBuilders.termQuery("result_type", "model_size_stats")) .setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
.get().getHits().totalHits; .get().getHits().totalHits;
long totalNotificationsCountAfterDelete = client().prepareSearch(".ml-notifications").get().getHits().totalHits; long totalNotificationsCountAfterDelete = client().prepareSearch(".ml-notifications").get().getHits().totalHits;
@ -199,8 +199,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
private void retainAllSnapshots(String jobId) throws Exception { private void retainAllSnapshots(String jobId) throws Exception {
List<ModelSnapshot> modelSnapshots = getModelSnapshots(jobId); List<ModelSnapshot> modelSnapshots = getModelSnapshots(jobId);
for (ModelSnapshot modelSnapshot : modelSnapshots) { for (ModelSnapshot modelSnapshot : modelSnapshots) {
UpdateModelSnapshotAction.Request request = new UpdateModelSnapshotAction.Request( UpdateModelSnapshotAction.Request request = new UpdateModelSnapshotAction.Request(jobId, modelSnapshot.getSnapshotId());
jobId, modelSnapshot.getSnapshotId());
request.setRetain(true); request.setRetain(true);
client().execute(UpdateModelSnapshotAction.INSTANCE, request).get(); client().execute(UpdateModelSnapshotAction.INSTANCE, request).get();
} }

1
plugin/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java
View File

@ -171,7 +171,6 @@ public class MlDistributedFailureIT extends BaseMlIntegTestCase {
// are what we expect them to be: // are what we expect them to be:
private static DataCounts getDataCountsFromIndex(String jobId) { private static DataCounts getDataCountsFromIndex(String jobId) {
SearchResponse searchResponse = client().prepareSearch() SearchResponse searchResponse = client().prepareSearch()
.setTypes(DataCounts.TYPE.getPreferredName())
.setQuery(QueryBuilders.idsQuery().addIds(DataCounts.documentId(jobId))) .setQuery(QueryBuilders.idsQuery().addIds(DataCounts.documentId(jobId)))
.get(); .get();
if (searchResponse.getHits().getTotalHits() != 1) { if (searchResponse.getHits().getTotalHits() != 1) {

30
plugin/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java
View File

@ -7,7 +7,6 @@ package org.elasticsearch.xpack.ml.job;
import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterName;
@ -15,9 +14,6 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.env.Environment; import org.elasticsearch.env.Environment;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ml.MlMetadata; import org.elasticsearch.xpack.ml.MlMetadata;
@ -28,15 +24,11 @@ import org.elasticsearch.xpack.ml.job.config.DataDescription;
import org.elasticsearch.xpack.ml.job.config.Detector; import org.elasticsearch.xpack.ml.job.config.Detector;
import org.elasticsearch.xpack.ml.job.config.Job; import org.elasticsearch.xpack.ml.job.config.Job;
import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.job.persistence.JobProvider;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.ml.job.process.autodetect.state.ModelSnapshotTests;
import org.elasticsearch.xpack.ml.job.results.Result;
import org.elasticsearch.xpack.ml.notifications.Auditor; import org.elasticsearch.xpack.ml.notifications.Auditor;
import org.junit.Before; import org.junit.Before;
import org.mockito.ArgumentCaptor; import org.mockito.ArgumentCaptor;
import org.mockito.Matchers; import org.mockito.Matchers;
import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.Date; import java.util.Date;
@ -163,28 +155,6 @@ public class JobManagerTests extends ESTestCase {
}); });
} }
public void testUpdateModelSnapshot() {
ArgumentCaptor<IndexRequest> indexRequestCaptor = ArgumentCaptor.forClass(IndexRequest.class);
doAnswer(invocationOnMock -> null).when(client).index(indexRequestCaptor.capture(), any());
ModelSnapshot modelSnapshot = ModelSnapshotTests.createRandomized();
JobManager jobManager = createJobManager();
jobManager.updateModelSnapshot(new Result("snapshot-index", modelSnapshot), response -> {}, error -> {});
IndexRequest indexRequest = indexRequestCaptor.getValue();
assertThat(indexRequest.index(), equalTo("snapshot-index"));
// Assert snapshot was correctly serialised in the request by parsing it back and comparing to original
try (XContentParser parser = XContentFactory.xContent(indexRequest.source()).createParser(NamedXContentRegistry.EMPTY,
indexRequest.source())) {
ModelSnapshot requestSnapshot = ModelSnapshot.PARSER.apply(parser, null).build();
assertThat(requestSnapshot, equalTo(modelSnapshot));
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Job.Builder createJob() { private Job.Builder createJob() {
Detector.Builder d1 = new Detector.Builder("info_content", "domain"); Detector.Builder d1 = new Detector.Builder("info_content", "domain");
d1.setOverFieldName("client"); d1.setOverFieldName("client");

5
plugin/src/test/java/org/elasticsearch/xpack/ml/job/config/MlFilterTests.java
View File

@ -13,6 +13,8 @@ import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import static org.hamcrest.Matchers.equalTo;
public class MlFilterTests extends AbstractSerializingTestCase<MlFilter> { public class MlFilterTests extends AbstractSerializingTestCase<MlFilter> {
@Override @Override
@ -46,4 +48,7 @@ public class MlFilterTests extends AbstractSerializingTestCase<MlFilter> {
assertEquals(MlFilter.ITEMS.getPreferredName() + " must not be null", ex.getMessage()); assertEquals(MlFilter.ITEMS.getPreferredName() + " must not be null", ex.getMessage());
} }
public void testDocumentId() {
assertThat(MlFilter.documentId("foo"), equalTo("filter_foo"));
}
} }

86
plugin/src/test/java/org/elasticsearch/xpack/ml/job/persistence/ElasticsearchMappingsTests.java
View File

@ -33,30 +33,8 @@ import java.util.Set;
public class ElasticsearchMappingsTests extends ESTestCase { public class ElasticsearchMappingsTests extends ESTestCase {
private void parseJson(JsonParser parser, Set<String> expected) throws IOException {
try {
JsonToken token = parser.nextToken();
while (token != null && token != JsonToken.END_OBJECT) {
switch (token) {
case START_OBJECT:
parseJson(parser, expected);
break;
case FIELD_NAME:
String fieldName = parser.getCurrentName();
expected.add(fieldName);
break;
default:
break;
}
token = parser.nextToken();
}
} catch (JsonParseException e) {
fail("Cannot parse JSON: " + e);
}
}
public void testReservedFields() public void testReservedFields() throws Exception {
throws IOException, ClassNotFoundException, IllegalAccessException, IllegalArgumentException, InvocationTargetException {
Set<String> overridden = new HashSet<>(); Set<String> overridden = new HashSet<>();
// These are not reserved because they're Elasticsearch keywords, not // These are not reserved because they're Elasticsearch keywords, not
@ -78,30 +56,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
overridden.add(ModelSnapshot.TYPE.getPreferredName()); overridden.add(ModelSnapshot.TYPE.getPreferredName());
overridden.add(Quantiles.TYPE.getPreferredName()); overridden.add(Quantiles.TYPE.getPreferredName());
Set<String> expected = new HashSet<>(); Set<String> expected = collectResultsDocFieldNames();
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
XContentBuilder builder = ElasticsearchMappings.resultsMapping();
BufferedInputStream inputStream =
new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
JsonParser parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.categoryDefinitionMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.dataCountsMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
builder = ElasticsearchMappings.modelSnapshotMapping();
inputStream = new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
parser = new JsonFactory().createParser(inputStream);
parseJson(parser, expected);
expected.removeAll(overridden); expected.removeAll(overridden);
@ -147,4 +102,41 @@ public class ElasticsearchMappingsTests extends ESTestCase {
instanceMapping = (Map<String, Object>) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName()); instanceMapping = (Map<String, Object>) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName());
assertNull(instanceMapping); assertNull(instanceMapping);
} }
private Set<String> collectResultsDocFieldNames() throws IOException {
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
XContentBuilder builder = ElasticsearchMappings.docMapping();
BufferedInputStream inputStream =
new BufferedInputStream(new ByteArrayInputStream(builder.string().getBytes(StandardCharsets.UTF_8)));
JsonParser parser = new JsonFactory().createParser(inputStream);
Set<String> fieldNames = new HashSet<>();
boolean isAfterPropertiesStart = false;
try {
JsonToken token = parser.nextToken();
while (token != null) {
switch (token) {
case START_OBJECT:
break;
case FIELD_NAME:
String fieldName = parser.getCurrentName();
if (isAfterPropertiesStart) {
fieldNames.add(fieldName);
} else {
if (ElasticsearchMappings.PROPERTIES.equals(fieldName)) {
isAfterPropertiesStart = true;
}
}
break;
default:
break;
}
token = parser.nextToken();
}
} catch (JsonParseException e) {
fail("Cannot parse JSON: " + e);
}
return fieldNames;
}
} }

4
plugin/src/test/resources/rest-api-spec/test/ml/delete_model_snapshot.yml
View File

@ -28,7 +28,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-delete-model-snapshot index: .ml-anomalies-delete-model-snapshot
type: model_snapshot type: doc
id: "delete-model-snapshot_model_snapshot_inactive-snapshot" id: "delete-model-snapshot_model_snapshot_inactive-snapshot"
body: > body: >
{ {
@ -64,7 +64,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-delete-model-snapshot index: .ml-anomalies-delete-model-snapshot
type: model_snapshot type: doc
id: "delete-model-snapshot_model_snapshot_active-snapshot" id: "delete-model-snapshot_model_snapshot_active-snapshot"
body: > body: >
{ {

2
plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml
View File

@ -5,7 +5,7 @@ setup:
index: index:
index: .ml-meta index: .ml-meta
type: doc type: doc
id: imposter-filter id: filter_imposter-filter
body: > body: >
{ {
"filter_id": "imposter", "filter_id": "imposter",

6
plugin/src/test/resources/rest-api-spec/test/ml/ml_anomalies_default_mappings.yml
View File

@ -19,7 +19,7 @@
- do: - do:
index: index:
index: .ml-anomalies-shared index: .ml-anomalies-shared
type: result type: doc
id: "new_doc" id: "new_doc"
body: > body: >
{ {
@ -33,6 +33,6 @@
- do: - do:
indices.get_field_mapping: indices.get_field_mapping:
index: .ml-anomalies-shared index: .ml-anomalies-shared
type: result type: doc
fields: new_field fields: new_field
- match: {\.ml-anomalies-shared.mappings.result.new_field.mapping.new_field.type: keyword} - match: {\.ml-anomalies-shared.mappings.doc.new_field.mapping.new_field.type: keyword}

2
plugin/src/test/resources/rest-api-spec/test/ml/post_data.yml
View File

@ -80,7 +80,7 @@ setup:
- do: - do:
get: get:
index: .ml-anomalies-post-data-job index: .ml-anomalies-post-data-job
type: data_counts type: doc
id: post-data-job_data_counts id: post-data-job_data_counts
- match: { _source.processed_record_count: 2 } - match: { _source.processed_record_count: 2 }

18
plugin/src/test/resources/rest-api-spec/test/ml/revert_model_snapshot.yml
View File

@ -28,7 +28,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: model_snapshot type: doc
id: "revert-model-snapshot_model_snapshot_first" id: "revert-model-snapshot_model_snapshot_first"
body: > body: >
{ {
@ -53,7 +53,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: model_snapshot type: doc
id: "revert-model-snapshot_model_snapshot_second" id: "revert-model-snapshot_model_snapshot_second"
body: > body: >
{ {
@ -78,7 +78,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1464825600000_1" id: "revert-model-snapshot_1464825600000_1"
body: > body: >
{ {
@ -91,7 +91,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1464782400000_1" id: "revert-model-snapshot_1464782400000_1"
body: > body: >
{ {
@ -104,7 +104,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1462060800000_1" id: "revert-model-snapshot_1462060800000_1"
body: > body: >
{ {
@ -117,7 +117,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1464825600000_1_1" id: "revert-model-snapshot_1464825600000_1_1"
body: > body: >
{ {
@ -130,7 +130,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1462060800000_1_2" id: "revert-model-snapshot_1462060800000_1_2"
body: > body: >
{ {
@ -143,7 +143,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1464825600000_1_3" id: "revert-model-snapshot_1464825600000_1_3"
body: { body: {
"job_id": "revert-model-snapshot", "job_id": "revert-model-snapshot",
@ -158,7 +158,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-revert-model-snapshot index: .ml-anomalies-revert-model-snapshot
type: result type: doc
id: "revert-model-snapshot_1462060800000_1_4" id: "revert-model-snapshot_1462060800000_1_4"
body: body:
{ {

4
plugin/src/test/resources/rest-api-spec/test/ml/update_model_snapshot.yml
View File

@ -16,7 +16,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-update-model-snapshot index: .ml-anomalies-update-model-snapshot
type: model_snapshot type: doc
id: "update-model-snapshot_model_snapshot_snapshot-1" id: "update-model-snapshot_model_snapshot_snapshot-1"
body: > body: >
{ {
@ -30,7 +30,7 @@ setup:
- do: - do:
index: index:
index: .ml-anomalies-update-model-snapshot index: .ml-anomalies-update-model-snapshot
type: model_snapshot type: doc
id: "update-model-snapshot_model_snapshot_snapshot-2" id: "update-model-snapshot_model_snapshot_snapshot-2"
body: > body: >
{ {