diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 2dc06d77ca4..a0ffa7f6c2a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -18,9 +18,11 @@ import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Collections; import java.util.SortedMap; import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; @@ -46,6 +48,15 @@ public class AnnotationIndex { boolean isHiddenAttributeAvailable = state.nodes().getMinNodeVersion().onOrAfter(HIDDEN_INTRODUCED_VERSION); + final ActionListener checkMappingsListener = ActionListener.wrap(success -> { + ElasticsearchMappings.addDocMappingIfMissing( + WRITE_ALIAS_NAME, + AnnotationIndex::annotationsMapping, + client, + state, + finalListener); + }, finalListener::onFailure); + final ActionListener createAliasListener = ActionListener.wrap(success -> { IndicesAliasesRequest.AliasActions addReadAliasAction = IndicesAliasesRequest.AliasActions.add().index(INDEX_NAME).alias(READ_ALIAS_NAME); @@ -61,7 +72,8 @@ public class AnnotationIndex { .addAliasAction(addWriteAliasAction) .request(); executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, request, - ActionListener.wrap(r -> finalListener.onResponse(r.isAcknowledged()), finalListener::onFailure), + ActionListener.wrap( + r -> checkMappingsListener.onResponse(r.isAcknowledged()), finalListener::onFailure), client.admin().indices()::aliases); }, finalListener::onFailure); @@ -107,6 +119,10 @@ public class AnnotationIndex { createAliasListener.onResponse(true); return; } + + // Check the mappings + checkMappingsListener.onResponse(false); + return; } // Nothing to do, but respond to the listener @@ -114,7 +130,11 @@ public class AnnotationIndex { } private static String annotationsMapping() { - return TemplateUtils.loadTemplate( - "/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return annotationsMapping(SINGLE_MAPPING_NAME); + } + + private static String annotationsMapping(String mappingType) { + return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json", + Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType)); } } diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json index 17ebf089de8..b3385054111 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json @@ -1,5 +1,5 @@ { - "_doc": { + "${xpack.ml.mapping_type}": { "_meta" : { "version" : "${xpack.ml.version}" }, diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index a0e57cee60b..cd8ba103b90 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -35,6 +35,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.action.util.QueryPage; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.GetFiltersAction; +import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; @@ -448,8 +449,20 @@ public class AutodetectProcessManager implements ClusterStateListener { ); // Try adding the results doc mapping - this updates to the latest version if an old mapping is present - ElasticsearchMappings.addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobId), - AnomalyDetectorsIndex::resultsMapping, client, clusterState, resultsMappingUpdateHandler); + ActionListener annotationsIndexUpdateHandler = ActionListener.wrap( + ack -> ElasticsearchMappings.addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobId), + AnomalyDetectorsIndex::resultsMapping, client, clusterState, resultsMappingUpdateHandler), + e -> { + // Due to a bug in 7.9.0 it's possible that the annotations index already has incorrect mappings + // and it would cause more harm than good to block jobs from opening in subsequent releases + logger.warn(new ParameterizedMessage("[{}] ML annotations index could not be updated with latest mappings", jobId), e); + ElasticsearchMappings.addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobId), + AnomalyDetectorsIndex::resultsMapping, client, clusterState, resultsMappingUpdateHandler); + } + ); + + // Create the annotations index if necessary - this also updates the mappings if an old mapping is present + AnnotationIndex.createAnnotationsIndexIfNecessary(client, clusterState, annotationsIndexUpdateHandler); } private boolean createProcessAndSetRunning(ProcessContext processContext, diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java index 1d096d37da9..90449129cf2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManagerTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; @@ -183,6 +184,18 @@ public class AutodetectProcessManagerTests extends ESTestCase { .build()) .putAlias(AliasMetadata.builder(AnomalyDetectorsIndex.jobStateIndexWriteAlias()).build()) .build()) + .fPut( + AnnotationIndex.INDEX_NAME, + IndexMetadata.builder(AnnotationIndex.INDEX_NAME) + .settings( + Settings.builder() + .put(SETTING_NUMBER_OF_SHARDS, 1) + .put(SETTING_NUMBER_OF_REPLICAS, 0) + .put(SETTING_VERSION_CREATED, Version.CURRENT) + .build()) + .putAlias(AliasMetadata.builder(AnnotationIndex.READ_ALIAS_NAME).build()) + .putAlias(AliasMetadata.builder(AnnotationIndex.WRITE_ALIAS_NAME).build()) + .build()) .build()) .build(); DiscoveryNodes nodes = mock(DiscoveryNodes.class); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml index 801738d1ae1..252d2bf1bd5 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml @@ -147,3 +147,14 @@ setup: job_id: old-cluster-function-shortcut-expansion - match: { count: 1 } - match: { jobs.0.analysis_config.detectors.0.function: "non_zero_count" } + +--- +"Test annotation index mappings": + + - do: + indices.get_mapping: + index: .ml-annotations-write + + - match: { \.ml-annotations-6.mappings.properties.type.type: "keyword" } + - match: { \.ml-annotations-6.mappings.properties.event.type: "keyword" } + - match: { \.ml-annotations-6.mappings.properties.detector_index.type: "integer" }