honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-03-25 01:19:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

[7.x][ML] Refactor ML mappings and templates into JSON resources (#51… (#52353)

Browse Source 
ML mappings and index templates have so far been created
programmatically. While this had its merits due to static typing,
there is consensus it would be clear to maintain those in json files.
In addition, we are going to adding ILM policies to these indices
and the component for a plugin to register ILM policies is
`IndexTemplateRegistry`. It expects the templates to be in resource
json files.

For the above reasons this commit refactors ML mappings and index
templates into json resource files that are registered via
`MlIndexTemplateRegistry`.

Backport of #51765
This commit is contained in:
Dimitris Athanasiou 2020-02-14 17:16:06 +02:00 committed by GitHub
parent cabc1769e2
commit ad56802ac6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
51 changed files with 1705 additions and 2107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java Normal file
View File

@ -0,0 +1,28 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ml;
import org.elasticsearch.Version;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import java.util.Collections;
public class MlConfigIndex {
private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
private MlConfigIndex() {}
public static String mapping() {
return mapping(MapperService.SINGLE_MAPPING_NAME);
}
public static String mapping(String mappingType) {
return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/config_index_mappings.json",
Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType));
}
}

39
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java
View File

@ -5,16 +5,6 @@
*/
package org.elasticsearch.xpack.core.ml;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.calendars.Calendar;
import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import java.io.IOException;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
public final class MlMetaIndex {
/**
* Where to store the ml info in Elasticsearch - must match what's
@ -22,33 +12,6 @@ public final class MlMetaIndex {
*/
public static final String INDEX_NAME = ".ml-meta";
private MlMetaIndex() {}
public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Calendar.ID.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Calendar.JOB_IDS.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Calendar.DESCRIPTION.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(ScheduledEvent.START_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(ScheduledEvent.END_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.endObject()
.endObject()
.endObject();
return builder;
}
private MlMetaIndex() {}
}

100
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.core.ml.annotations;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
@ -15,19 +16,13 @@ import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.MachineLearningField;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import java.io.IOException;
import java.util.SortedMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -40,6 +35,8 @@ public class AnnotationIndex {
public static final String INDEX_NAME = ".ml-annotations-6";
public static final String INDEX_PATTERN = ".ml-annotations*";
private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
/**
* Create the .ml-annotations index with correct mappings if it does not already
* exist. This index is read and written by the UI results views, so needs to
@ -64,39 +61,26 @@ public class AnnotationIndex {
// Create the annotations index if it doesn't exist already.
if (mlLookup.containsKey(INDEX_NAME) == false) {
final TimeValue delayedNodeTimeOutSetting;
// Whether we are using native process is a good way to detect whether we are in dev / test mode:
if (MachineLearningField.AUTODETECT_PROCESS.get(settings)) {
delayedNodeTimeOutSetting = UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(settings);
} else {
delayedNodeTimeOutSetting = TimeValue.ZERO;
}
CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), delayedNodeTimeOutSetting));
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping(), XContentType.JSON);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1"));
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(
r -> createAliasListener.onResponse(r.isAcknowledged()),
e -> {
// Possible that the index was created while the request was executing,
// so we need to handle that possibility
if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
// Create the alias
createAliasListener.onResponse(true);
} else {
finalListener.onFailure(e);
}
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(
r -> createAliasListener.onResponse(r.isAcknowledged()),
e -> {
// Possible that the index was created while the request was executing,
// so we need to handle that possibility
if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
// Create the alias
createAliasListener.onResponse(true);
} else {
finalListener.onFailure(e);
}
), client.admin().indices()::create);
} catch (IOException e) {
finalListener.onFailure(e);
}
}
), client.admin().indices()::create);
return;
}
@ -111,42 +95,8 @@ public class AnnotationIndex {
finalListener.onResponse(false);
}
public static XContentBuilder annotationsMapping() throws IOException {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Annotation.ANNOTATION.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.TEXT)
.endObject()
.startObject(Annotation.CREATE_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Annotation.CREATE_USERNAME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Annotation.TIMESTAMP.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Annotation.END_TIMESTAMP.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Job.ID.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Annotation.MODIFIED_TIME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.DATE)
.endObject()
.startObject(Annotation.MODIFIED_USERNAME.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.startObject(Annotation.TYPE.getPreferredName())
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD)
.endObject()
.endObject()
.endObject()
.endObject();
return builder;
public static String annotationsMapping() {
return TemplateUtils.loadTemplate("/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json",
Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE);
}
}

14
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.core.ml.job.persistence;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@ -16,7 +17,9 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import java.util.Arrays;
import java.util.Collections;
@ -31,6 +34,9 @@ public final class AnomalyDetectorsIndex {
public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000;
private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version";
private static final String RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/anomalydetection/";
private AnomalyDetectorsIndex() {
}
@ -144,4 +150,12 @@ public final class AnomalyDetectorsIndex {
}
}
public static String resultsMapping() {
return resultsMapping(MapperService.SINGLE_MAPPING_NAME);
}
public static String resultsMapping(String mappingType) {
return TemplateUtils.loadTemplate(RESOURCE_PATH + "results_index_mappings.json",
Version.CURRENT.toString(), RESULTS_MAPPINGS_VERSION_VARIABLE, Collections.singletonMap("xpack.ml.mapping_type", mappingType));
}
}

1129
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
View File

File diff suppressed because it is too large Load Diff

1
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java
View File

@ -234,6 +234,7 @@ public final class ReservedFieldNames {
Job.MODEL_SNAPSHOT_ID.getPreferredName(),
Job.MODEL_SNAPSHOT_MIN_VERSION.getPreferredName(),
Job.RESULTS_INDEX_NAME.getPreferredName(),
Job.ALLOW_LAZY_OPEN.getPreferredName(),
AnalysisConfig.BUCKET_SPAN.getPreferredName(),
AnalysisConfig.CATEGORIZATION_FIELD_NAME.getPreferredName(),

4
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditorField.java → x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java
View File

@ -5,9 +5,9 @@
*/
package org.elasticsearch.xpack.core.ml.notifications;
public final class AuditorField {
public final class NotificationsIndex {
public static final String NOTIFICATIONS_INDEX = ".ml-notifications-000001";
private AuditorField() {}
private NotificationsIndex() {}
}

3
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java
View File

@ -17,12 +17,11 @@ import org.elasticsearch.xpack.core.template.TemplateUtils;
import java.io.IOException;
import java.time.Instant;
import java.util.Locale;
import java.util.regex.Pattern;
public final class MonitoringTemplateUtils {
private static final String TEMPLATE_FILE = "/monitoring-%s.json";
private static final String TEMPLATE_VERSION_PROPERTY = Pattern.quote("${monitoring.template.version}");
private static final String TEMPLATE_VERSION_PROPERTY = "monitoring.template.version";
/**
* The last version of X-Pack that updated the templates and pipelines.

30
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java
View File

@ -7,6 +7,9 @@
package org.elasticsearch.xpack.core.template;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.regex.Pattern;
/**
@ -18,6 +21,7 @@ public class IndexTemplateConfig {
private final String fileName;
private final int version;
private final String versionProperty;
private final Map<String, String> variables;
/**
* Describes a template to be loaded from a resource file. Includes handling for substituting a version property into the template.
@ -38,10 +42,33 @@ public class IndexTemplateConfig {
* @param versionProperty The property that will be replaced with the {@code version} string as described above.
*/
public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty) {
this(templateName, fileName, version, versionProperty, Collections.emptyMap());
}
/**
* Describes a template to be loaded from a resource file. Includes handling for substituting a version property into the template.
*
* The {@code versionProperty} parameter will be used to substitute the value of {@code version} into the template. For example,
* this template:
* {@code {"myTemplateVersion": "${my.version.property}"}}
* With {@code version = "42"; versionProperty = "my.version.property"} will result in {@code {"myTemplateVersion": "42"}}.
*
* @param templateName The name that will be used for the index template. Literal, include the version in this string if
* it should be used.
* @param fileName The filename the template should be loaded from. Literal, should include leading {@literal /} and
* extension if necessary.
* @param version The version of the template. Substituted for {@code versionProperty} as described above.
* @param versionProperty The property that will be replaced with the {@code version} string as described above.
* @param variables A map of additional variable substitutions. The map's keys are the variable names.
* The corresponding values will replace the variable names.
*/
public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty, Map<String, String> variables)
{
this.templateName = templateName;
this.fileName = fileName;
this.version = version;
this.versionProperty = versionProperty;
this.variables = Objects.requireNonNull(variables);
}
public String getFileName() {
@ -61,8 +88,7 @@ public class IndexTemplateConfig {
* @return The template as a UTF-8 byte array.
*/
public byte[] loadBytes() {
final String versionPattern = Pattern.quote("${" + versionProperty + "}");
String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionPattern);
String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionProperty, variables);
assert template != null && template.length() > 0;
assert Pattern.compile("\"version\"\\s*:\\s*" + version).matcher(template).find()
: "index template must have a version property set to the given version property";

15
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java
View File

@ -119,6 +119,12 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
return;
}
// This registry requires to run on a master node.
// If not a master node, exit.
if (requiresMasterNode() && state.nodes().isLocalNodeElectedMaster() == false) {
return;
}
// if this node is newer than the master node, we probably need to add the template, which might be newer than the
// template the master node has, so we need potentially add new templates despite being not the master node
DiscoveryNode localNode = event.state().getNodes().getLocalNode();
@ -130,6 +136,15 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
}
}
/**
* Whether the registry should only apply changes when running on the master node.
* This is useful for plugins where certain actions are performed on master nodes
* and the templates should match the respective version.
*/
protected boolean requiresMasterNode() {
return false;
}
private void addTemplatesIfMissing(ClusterState state) {
final List<IndexTemplateConfig> indexTemplates = getTemplateConfigs();
for (IndexTemplateConfig newTemplate : indexTemplates) {

51
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
View File

@ -11,11 +11,11 @@ import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.compress.NotXContentException;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -23,8 +23,10 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@ -57,12 +59,18 @@ public class TemplateUtils {
* Loads a built-in template and returns its source.
*/
public static String loadTemplate(String resource, String version, String versionProperty) {
try {
BytesReference source = load(resource);
final String filteredJson = filter(source, version, versionProperty);
validate(new BytesArray(filteredJson));
return loadTemplate(resource, version, versionProperty, Collections.emptyMap());
}
return filteredJson;
/**
* Loads a built-in template and returns its source after replacing given variables.
*/
public static String loadTemplate(String resource, String version, String versionProperty, Map<String, String> variables) {
try {
String source = load(resource);
source = replaceVariables(source, version, versionProperty, variables);
validate(source);
return source;
} catch (Exception e) {
throw new IllegalArgumentException("Unable to load template [" + resource + "]", e);
}
@ -71,34 +79,43 @@ public class TemplateUtils {
/**
* Loads a resource from the classpath and returns it as a {@link BytesReference}
*/
public static BytesReference load(String name) throws IOException {
return Streams.readFully(TemplateUtils.class.getResourceAsStream(name));
public static String load(String name) throws IOException {
return Streams.readFully(TemplateUtils.class.getResourceAsStream(name)).utf8ToString();
}
/**
* Parses and validates that the source is not empty.
*/
public static void validate(BytesReference source) {
public static void validate(String source) {
if (source == null) {
throw new ElasticsearchParseException("Template must not be null");
}
if (Strings.isEmpty(source)) {
throw new ElasticsearchParseException("Template must not be empty");
}
try {
XContentHelper.convertToMap(source, false, XContentType.JSON).v2();
} catch (NotXContentException e) {
throw new ElasticsearchParseException("Template must not be empty");
XContentHelper.convertToMap(JsonXContent.jsonXContent, source, false);
} catch (Exception e) {
throw new ElasticsearchParseException("Invalid template", e);
}
}
private static String replaceVariables(String input, String version, String versionProperty, Map<String, String> variables) {
String template = replaceVariable(input, versionProperty, version);
for (Map.Entry<String, String> variable : variables.entrySet()) {
template = replaceVariable(template, variable.getKey(), variable.getValue());
}
return template;
}
/**
* Filters the source: replaces any template version property with the version number
* Replaces all occurences of given variable with the value
*/
public static String filter(BytesReference source, String version, String versionProperty) {
return Pattern.compile(versionProperty)
.matcher(source.utf8ToString())
.replaceAll(version);
public static String replaceVariable(String input, String variable, String value) {
return Pattern.compile("${" + variable + "}", Pattern.LITERAL)
.matcher(input)
.replaceAll(value);
}
/**

36
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/annotations_index_mappings.json Normal file
View File

@ -0,0 +1,36 @@
{
"_doc": {
"_meta" : {
"version" : "${xpack.ml.version}"
},
"properties" : {
"annotation" : {
"type" : "text"
},
"create_time" : {
"type" : "date"
},
"create_username" : {
"type" : "keyword"
},
"end_timestamp" : {
"type" : "date"
},
"job_id" : {
"type" : "keyword"
},
"modified_time" : {
"type" : "date"
},
"modified_username" : {
"type" : "keyword"
},
"timestamp" : {
"type" : "date"
},
"type" : {
"type" : "keyword"
}
}
}
}

478
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_mappings.json Normal file
View File

@ -0,0 +1,478 @@
{
"${xpack.ml.mapping_type}" : {
"_meta" : {
"version" : "${xpack.ml.version}"
},
"dynamic_templates" : [
{
"strings_as_keywords" : {
"match" : "*",
"mapping" : {
"type" : "keyword"
}
}
}
],
"properties" : {
"actual" : {
"type" : "double"
},
"all_field_values" : {
"type" : "text",
"analyzer" : "whitespace"
},
"anomaly_score" : {
"type" : "double"
},
"average_bucket_processing_time_ms" : {
"type" : "double"
},
"bucket_allocation_failures_count" : {
"type" : "long"
},
"bucket_count" : {
"type" : "long"
},
"bucket_influencers" : {
"type" : "nested",
"properties" : {
"anomaly_score" : {
"type" : "double"
},
"bucket_span" : {
"type" : "long"
},
"influencer_field_name" : {
"type" : "keyword"
},
"initial_anomaly_score" : {
"type" : "double"
},
"is_interim" : {
"type" : "boolean"
},
"job_id" : {
"type" : "keyword"
},
"probability" : {
"type" : "double"
},
"raw_anomaly_score" : {
"type" : "double"
},
"result_type" : {
"type" : "keyword"
},
"timestamp" : {
"type" : "date"
}
}
},
"bucket_span" : {
"type" : "long"
},
"by_field_name" : {
"type" : "keyword"
},
"by_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"category_id" : {
"type" : "long"
},
"causes" : {
"type" : "nested",
"properties" : {
"actual" : {
"type" : "double"
},
"by_field_name" : {
"type" : "keyword"
},
"by_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"correlated_by_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"field_name" : {
"type" : "keyword"
},
"function" : {
"type" : "keyword"
},
"function_description" : {
"type" : "keyword"
},
"geo_results" : {
"properties" : {
"actual_point" : {
"type" : "geo_point"
},
"typical_point" : {
"type" : "geo_point"
}
}
},
"over_field_name" : {
"type" : "keyword"
},
"over_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"partition_field_name" : {
"type" : "keyword"
},
"partition_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"probability" : {
"type" : "double"
},
"typical" : {
"type" : "double"
}
}
},
"description" : {
"type" : "text"
},
"detector_index" : {
"type" : "integer"
},
"earliest_record_timestamp" : {
"type" : "date"
},
"empty_bucket_count" : {
"type" : "long"
},
"event_count" : {
"type" : "long"
},
"examples" : {
"type" : "text"
},
"exponential_average_bucket_processing_time_ms" : {
"type" : "double"
},
"exponential_average_calculation_context" : {
"properties" : {
"incremental_metric_value_ms" : {
"type" : "double"
},
"latest_timestamp" : {
"type" : "date"
},
"previous_exponential_average_ms" : {
"type" : "double"
}
}
},
"field_name" : {
"type" : "keyword"
},
"forecast_create_timestamp" : {
"type" : "date"
},
"forecast_end_timestamp" : {
"type" : "date"
},
"forecast_expiry_timestamp" : {
"type" : "date"
},
"forecast_id" : {
"type" : "keyword"
},
"forecast_lower" : {
"type" : "double"
},
"forecast_memory_bytes" : {
"type" : "long"
},
"forecast_messages" : {
"type" : "keyword"
},
"forecast_prediction" : {
"type" : "double"
},
"forecast_progress" : {
"type" : "double"
},
"forecast_start_timestamp" : {
"type" : "date"
},
"forecast_status" : {
"type" : "keyword"
},
"forecast_upper" : {
"type" : "double"
},
"function" : {
"type" : "keyword"
},
"function_description" : {
"type" : "keyword"
},
"geo_results" : {
"properties" : {
"actual_point" : {
"type" : "geo_point"
},
"typical_point" : {
"type" : "geo_point"
}
}
},
"influencer_field_name" : {
"type" : "keyword"
},
"influencer_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"influencer_score" : {
"type" : "double"
},
"influencers" : {
"type" : "nested",
"properties" : {
"influencer_field_name" : {
"type" : "keyword"
},
"influencer_field_values" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
}
}
},
"initial_anomaly_score" : {
"type" : "double"
},
"initial_influencer_score" : {
"type" : "double"
},
"initial_record_score" : {
"type" : "double"
},
"input_bytes" : {
"type" : "long"
},
"input_field_count" : {
"type" : "long"
},
"input_record_count" : {
"type" : "long"
},
"invalid_date_count" : {
"type" : "long"
},
"is_interim" : {
"type" : "boolean"
},
"job_id" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"last_data_time" : {
"type" : "date"
},
"latest_empty_bucket_timestamp" : {
"type" : "date"
},
"latest_record_time_stamp" : {
"type" : "date"
},
"latest_record_timestamp" : {
"type" : "date"
},
"latest_result_time_stamp" : {
"type" : "date"
},
"latest_sparse_bucket_timestamp" : {
"type" : "date"
},
"log_time" : {
"type" : "date"
},
"max_matching_length" : {
"type" : "long"
},
"maximum_bucket_processing_time_ms" : {
"type" : "double"
},
"memory_status" : {
"type" : "keyword"
},
"min_version" : {
"type" : "keyword"
},
"minimum_bucket_processing_time_ms" : {
"type" : "double"
},
"missing_field_count" : {
"type" : "long"
},
"model_bytes" : {
"type" : "long"
},
"model_feature" : {
"type" : "keyword"
},
"model_lower" : {
"type" : "double"
},
"model_median" : {
"type" : "double"
},
"model_size_stats" : {
"properties" : {
"bucket_allocation_failures_count" : {
"type" : "long"
},
"job_id" : {
"type" : "keyword"
},
"log_time" : {
"type" : "date"
},
"memory_status" : {
"type" : "keyword"
},
"model_bytes" : {
"type" : "long"
},
"result_type" : {
"type" : "keyword"
},
"timestamp" : {
"type" : "date"
},
"total_by_field_count" : {
"type" : "long"
},
"total_over_field_count" : {
"type" : "long"
},
"total_partition_field_count" : {
"type" : "long"
}
}
},
"model_upper" : {
"type" : "double"
},
"multi_bucket_impact" : {
"type" : "double"
},
"out_of_order_timestamp_count" : {
"type" : "long"
},
"over_field_name" : {
"type" : "keyword"
},
"over_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"partition_field_name" : {
"type" : "keyword"
},
"partition_field_value" : {
"type" : "keyword",
"copy_to" : [
"all_field_values"
]
},
"probability" : {
"type" : "double"
},
"processed_field_count" : {
"type" : "long"
},
"processed_record_count" : {
"type" : "long"
},
"processing_time_ms" : {
"type" : "long"
},
"quantiles" : {
"type" : "object",
"enabled" : false
},
"raw_anomaly_score" : {
"type" : "double"
},
"record_score" : {
"type" : "double"
},
"regex" : {
"type" : "keyword"
},
"result_type" : {
"type" : "keyword"
},
"retain" : {
"type" : "boolean"
},
"scheduled_events" : {
"type" : "keyword"
},
"search_count" : {
"type" : "long"
},
"snapshot_doc_count" : {
"type" : "integer"
},
"snapshot_id" : {
"type" : "keyword"
},
"sparse_bucket_count" : {
"type" : "long"
},
"terms" : {
"type" : "text"
},
"timestamp" : {
"type" : "date"
},
"total_by_field_count" : {
"type" : "long"
},
"total_over_field_count" : {
"type" : "long"
},
"total_partition_field_count" : {
"type" : "long"
},
"total_search_time_ms" : {
"type" : "double"
},
"typical" : {
"type" : "double"
}
}
}
}

19
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/results_index_template.json Normal file
View File

@ -0,0 +1,19 @@
{
"order" : 0,
"version" : ${xpack.ml.version.id},
"index_patterns" : [
".ml-anomalies-*"
],
"settings" : {
"index" : {
"translog" : {
"durability" : "async"
},
"auto_expand_replicas" : "0-1",
"query" : {
"default_field" : "all_field_values"
}
}
},
"mappings": ${xpack.ml.anomalydetection.results.mappings}
}

21
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/anomalydetection/state_index_template.json Normal file
View File

@ -0,0 +1,21 @@
{
"order" : 0,
"version" : ${xpack.ml.version.id},
"index_patterns" : [
".ml-state*"
],
"settings" : {
"index" : {
"auto_expand_replicas" : "0-1"
}
},
"mappings" : {
"_doc": {
"_meta": {
"version": "${xpack.ml.version}"
},
"enabled": false
}
},
"aliases" : { }
}

364
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json Normal file
View File

@ -0,0 +1,364 @@
{
"${xpack.ml.mapping_type}" : {
"_meta" : {
"version" : "${xpack.ml.version}"
},
"dynamic_templates" : [
{
"strings_as_keywords" : {
"match" : "*",
"mapping" : {
"type" : "keyword"
}
}
}
],
"properties" : {
"aggregations" : {
"type" : "object",
"enabled" : false
},
"allow_lazy_open" : {
"type" : "keyword"
},
"analysis" : {
"properties" : {
"classification" : {
"properties" : {
"dependent_variable" : {
"type" : "keyword"
},
"eta" : {
"type" : "double"
},
"feature_bag_fraction" : {
"type" : "double"
},
"gamma" : {
"type" : "double"
},
"lambda" : {
"type" : "double"
},
"maximum_number_trees" : {
"type" : "integer"
},
"num_top_classes" : {
"type" : "integer"
},
"num_top_feature_importance_values" : {
"type" : "integer"
},
"prediction_field_name" : {
"type" : "keyword"
},
"training_percent" : {
"type" : "double"
}
}
},
"outlier_detection" : {
"properties" : {
"feature_influence_threshold" : {
"type" : "double"
},
"method" : {
"type" : "keyword"
},
"n_neighbors" : {
"type" : "integer"
}
}
},
"regression" : {
"properties" : {
"dependent_variable" : {
"type" : "keyword"
},
"eta" : {
"type" : "double"
},
"feature_bag_fraction" : {
"type" : "double"
},
"gamma" : {
"type" : "double"
},
"lambda" : {
"type" : "double"
},
"maximum_number_trees" : {
"type" : "integer"
},
"num_top_feature_importance_values" : {
"type" : "integer"
},
"prediction_field_name" : {
"type" : "keyword"
},
"training_percent" : {
"type" : "double"
}
}
}
}
},
"analysis_config" : {
"properties" : {
"bucket_span" : {
"type" : "keyword"
},
"categorization_analyzer" : {
"type" : "object",
"enabled" : false
},
"categorization_field_name" : {
"type" : "keyword"
},
"categorization_filters" : {
"type" : "keyword"
},
"detectors" : {
"properties" : {
"by_field_name" : {
"type" : "keyword"
},
"custom_rules" : {
"type" : "nested",
"properties" : {
"actions" : {
"type" : "keyword"
},
"conditions" : {
"type" : "nested",
"properties" : {
"applies_to" : {
"type" : "keyword"
},
"operator" : {
"type" : "keyword"
},
"value" : {
"type" : "double"
}
}
},
"scope" : {
"type" : "object",
"enabled" : false
}
}
},
"detector_description" : {
"type" : "text"
},
"detector_index" : {
"type" : "integer"
},
"exclude_frequent" : {
"type" : "keyword"
},
"field_name" : {
"type" : "keyword"
},
"function" : {
"type" : "keyword"
},
"over_field_name" : {
"type" : "keyword"
},
"partition_field_name" : {
"type" : "keyword"
},
"use_null" : {
"type" : "boolean"
}
}
},
"influencers" : {
"type" : "keyword"
},
"latency" : {
"type" : "keyword"
},
"multivariate_by_fields" : {
"type" : "boolean"
},
"summary_count_field_name" : {
"type" : "keyword"
}
}
},
"analysis_limits" : {
"properties" : {
"categorization_examples_limit" : {
"type" : "long"
},
"model_memory_limit" : {
"type" : "keyword"
}
}
},
"analyzed_fields" : {
"type" : "object",
"enabled" : false
},
"background_persist_interval" : {
"type" : "keyword"
},
"chunking_config" : {
"properties" : {
"mode" : {
"type" : "keyword"
},
"time_span" : {
"type" : "keyword"
}
}
},
"config_type" : {
"type" : "keyword"
},
"create_time" : {
"type" : "date"
},
"custom_settings" : {
"type" : "object",
"enabled" : false
},
"data_description" : {
"properties" : {
"field_delimiter" : {
"type" : "keyword"
},
"format" : {
"type" : "keyword"
},
"quote_character" : {
"type" : "keyword"
},
"time_field" : {
"type" : "keyword"
},
"time_format" : {
"type" : "keyword"
}
}
},
"datafeed_id" : {
"type" : "keyword"
},
"delayed_data_check_config" : {
"properties" : {
"check_window" : {
"type" : "keyword"
},
"enabled" : {
"type" : "boolean"
}
}
},
"description" : {
"type" : "text"
},
"dest" : {
"properties" : {
"index" : {
"type" : "keyword"
},
"results_field" : {
"type" : "keyword"
}
}
},
"finished_time" : {
"type" : "date"
},
"frequency" : {
"type" : "keyword"
},
"groups" : {
"type" : "keyword"
},
"headers" : {
"type" : "object",
"enabled" : false
},
"id" : {
"type" : "keyword"
},
"indices" : {
"type" : "keyword"
},
"job_id" : {
"type" : "keyword"
},
"job_type" : {
"type" : "keyword"
},
"job_version" : {
"type" : "keyword"
},
"model_plot_config" : {
"properties" : {
"enabled" : {
"type" : "boolean"
},
"terms" : {
"type" : "keyword"
}
}
},
"model_snapshot_id" : {
"type" : "keyword"
},
"model_snapshot_min_version" : {
"type" : "keyword"
},
"model_snapshot_retention_days" : {
"type" : "long"
},
"query" : {
"type" : "object",
"enabled" : false
},
"query_delay" : {
"type" : "keyword"
},
"renormalization_window_days" : {
"type" : "long"
},
"results_index_name" : {
"type" : "keyword"
},
"results_retention_days" : {
"type" : "long"
},
"script_fields" : {
"type" : "object",
"enabled" : false
},
"scroll_size" : {
"type" : "long"
},
"source" : {
"properties" : {
"_source" : {
"type" : "object",
"enabled" : false
},
"index" : {
"type" : "keyword"
},
"query" : {
"type" : "object",
"enabled" : false
}
}
},
"version" : {
"type" : "keyword"
}
}
}
}

15
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_template.json Normal file
View File

@ -0,0 +1,15 @@
{
"order" : 0,
"version" : ${xpack.ml.version.id},
"index_patterns" : [
".ml-config"
],
"settings" : {
"index" : {
"max_result_window" : "${xpack.ml.config.max_result_window}",
"number_of_shards" : "1",
"auto_expand_replicas" : "0-1"
}
},
"mappings": ${xpack.ml.config.mappings}
}

72
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/inference_index_template.json Normal file
View File

@ -0,0 +1,72 @@
{
"order" : 0,
"version" : ${xpack.ml.version.id},
"index_patterns" : [
".ml-inference-000001"
],
"settings" : {
"index" : {
"number_of_shards" : "1",
"auto_expand_replicas" : "0-1"
}
},
"mappings" : {
"_doc": {
"_meta": {
"version": "${xpack.ml.version}"
},
"dynamic": "false",
"properties": {
"doc_type": {
"type": "keyword"
},
"model_id": {
"type": "keyword"
},
"created_by": {
"type": "keyword"
},
"input": {
"enabled": false
},
"version": {
"type": "keyword"
},
"description": {
"type": "text"
},
"create_time": {
"type": "date"
},
"tags": {
"type": "keyword"
},
"metadata": {
"enabled": false
},
"estimated_operations": {
"type": "long"
},
"estimated_heap_memory_usage_bytes": {
"type": "long"
},
"doc_num": {
"type": "long"
},
"definition": {
"enabled": false
},
"compression_version": {
"type": "long"
},
"definition_length": {
"type": "long"
},
"total_definition_length": {
"type": "long"
}
}
}
},
"aliases" : { }
}

47
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/meta_index_template.json Normal file
View File

@ -0,0 +1,47 @@
{
"order" : 0,
"version" : ${xpack.ml.version.id},
"index_patterns" : [
".ml-meta"
],
"settings" : {
"index" : {
"number_of_shards" : "1",
"auto_expand_replicas" : "0-1"
}
},
"mappings" : {
"_doc": {
"_meta": {
"version": "${xpack.ml.version}"
},
"dynamic_templates": [
{
"strings_as_keywords": {
"match": "*",
"mapping": {
"type": "keyword"
}
}
}
],
"properties": {
"calendar_id": {
"type": "keyword"
},
"job_ids": {
"type": "keyword"
},
"description": {
"type": "keyword"
},
"start_time": {
"type": "date"
},
"end_time": {
"type": "date"
}
}
}
}
}

46
x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/notifications_index_template.json Normal file
View File

@ -0,0 +1,46 @@
{
"order" : 0,
"version" : ${xpack.ml.version.id},
"index_patterns" : [
".ml-notifications-000001"
],
"settings" : {
"index" : {
"number_of_shards" : "1",
"auto_expand_replicas" : "0-1"
}
},
"mappings" : {
"_doc": {
"_meta" : {
"version" : "${xpack.ml.version}"
},
"dynamic" : "false",
"properties" : {
"job_id": {
"type": "keyword"
},
"level": {
"type": "keyword"
},
"message": {
"type": "text",
"fields": {
"raw": {
"type": "keyword"
}
}
},
"timestamp": {
"type": "date"
},
"node_name": {
"type": "keyword"
},
"job_type": {
"type": "keyword"
}
}
}
}
}

57
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
View File

@ -20,15 +20,13 @@ import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.get.GetResult;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.VersionUtils;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats;
import org.elasticsearch.xpack.core.ml.job.config.Job;
@ -38,7 +36,6 @@ import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeSta
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames;
import org.elasticsearch.xpack.core.ml.job.results.Result;
@ -56,8 +53,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.hamcrest.Matchers.equalTo;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
@ -125,7 +120,6 @@ public class ElasticsearchMappingsTests extends ESTestCase {
compareFields(expected, ReservedFieldNames.RESERVED_CONFIG_FIELD_NAMES);
}
private void compareFields(Set<String> expected, Set<String> reserved) {
if (reserved.size() != expected.size()) {
Set<String> diff = new HashSet<>(reserved);
@ -146,32 +140,6 @@ public class ElasticsearchMappingsTests extends ESTestCase {
}
}
@SuppressWarnings("unchecked")
public void testTermFieldMapping() throws IOException {
XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(Arrays.asList("apple", "strawberry",
AnomalyRecord.BUCKET_SPAN.getPreferredName()));
XContentParser parser = createParser(builder);
Map<String, Object> mapping = (Map<String, Object>) parser.map().get(SINGLE_MAPPING_NAME);
Map<String, Object> properties = (Map<String, Object>) mapping.get(ElasticsearchMappings.PROPERTIES);
Map<String, Object> instanceMapping = (Map<String, Object>) properties.get("apple");
assertNotNull(instanceMapping);
String dataType = (String)instanceMapping.get(ElasticsearchMappings.TYPE);
assertEquals(ElasticsearchMappings.KEYWORD, dataType);
instanceMapping = (Map<String, Object>) properties.get("strawberry");
assertNotNull(instanceMapping);
dataType = (String)instanceMapping.get(ElasticsearchMappings.TYPE);
assertEquals(ElasticsearchMappings.KEYWORD, dataType);
// check no mapping for the reserved field
instanceMapping = (Map<String, Object>) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName());
assertNull(instanceMapping);
}
public void testMappingRequiresUpdateNoMapping() throws IOException {
ClusterState.Builder csBuilder = ClusterState.builder(new ClusterName("_name"));
ClusterState cs = csBuilder.build();
@ -240,7 +208,7 @@ public class ElasticsearchMappingsTests extends ESTestCase {
ClusterState clusterState = getClusterStateWithMappingsWithMetaData(Collections.singletonMap("index-name", "0.0"));
ElasticsearchMappings.addDocMappingIfMissing(
"index-name",
ElasticsearchMappingsTests::fakeMapping,
mappingType -> "{\"_doc\":{\"properties\":{\"some-field\":{\"type\":\"long\"}}}}",
client,
clusterState,
ActionListener.wrap(
@ -260,19 +228,6 @@ public class ElasticsearchMappingsTests extends ESTestCase {
assertThat(request.source(), equalTo("{\"_doc\":{\"properties\":{\"some-field\":{\"type\":\"long\"}}}}"));
}
private static XContentBuilder fakeMapping(String mappingType) throws IOException {
return jsonBuilder()
.startObject()
.startObject(mappingType)
.startObject(ElasticsearchMappings.PROPERTIES)
.startObject("some-field")
.field(ElasticsearchMappings.TYPE, ElasticsearchMappings.LONG)
.endObject()
.endObject()
.endObject()
.endObject();
}
private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object> namesAndVersions) throws IOException {
MetaData.Builder metaDataBuilder = MetaData.builder();
@ -311,17 +266,17 @@ public class ElasticsearchMappingsTests extends ESTestCase {
private Set<String> collectResultsDocFieldNames() throws IOException {
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
return collectFieldNames(ElasticsearchMappings.resultsMapping("_doc"));
return collectFieldNames(AnomalyDetectorsIndex.resultsMapping());
}
private Set<String> collectConfigDocFieldNames() throws IOException {
// Only the mappings for the config index should be added below. Do NOT add mappings for other indexes here.
return collectFieldNames(ElasticsearchMappings.configMapping());
return collectFieldNames(MlConfigIndex.mapping());
}
private Set<String> collectFieldNames(XContentBuilder mapping) throws IOException {
private Set<String> collectFieldNames(String mapping) throws IOException {
BufferedInputStream inputStream =
new BufferedInputStream(new ByteArrayInputStream(Strings.toString(mapping).getBytes(StandardCharsets.UTF_8)));
new BufferedInputStream(new ByteArrayInputStream(mapping.getBytes(StandardCharsets.UTF_8)));
JsonParser parser = new JsonFactory().createParser(inputStream);
Set<String> fieldNames = new HashSet<>();
boolean isAfterPropertiesStart = false;

6
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
View File

@ -108,7 +108,7 @@ import org.elasticsearch.xpack.core.ml.action.ValidateDetectorAction;
import org.elasticsearch.xpack.core.ml.action.ValidateJobConfigAction;
import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
import org.elasticsearch.xpack.core.security.action.DelegatePkiAuthenticationAction;
import org.elasticsearch.xpack.core.security.action.privilege.DeletePrivilegesAction;
@ -1135,7 +1135,7 @@ public class ReservedRolesStoreTests extends ESTestCase {
assertOnlyReadAllowed(role, MlMetaIndex.INDEX_NAME);
assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
assertOnlyReadAllowed(role, AuditorField.NOTIFICATIONS_INDEX);
assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX);
assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME);
assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);
@ -1222,7 +1222,7 @@ public class ReservedRolesStoreTests extends ESTestCase {
assertNoAccessAllowed(role, MlMetaIndex.INDEX_NAME);
assertNoAccessAllowed(role, AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
assertOnlyReadAllowed(role, AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
assertOnlyReadAllowed(role, AuditorField.NOTIFICATIONS_INDEX);
assertOnlyReadAllowed(role, NotificationsIndex.NOTIFICATIONS_INDEX);
assertReadWriteDocsButNotDeleteIndexAllowed(role, AnnotationIndex.INDEX_NAME);
assertNoAccessAllowed(role, RestrictedIndicesNames.RESTRICTED_NAMES);

75
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TemplateUtilsTests.java
View File

@ -8,15 +8,13 @@ package org.elasticsearch.xpack.core.template;
import org.apache.lucene.util.Constants;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import org.hamcrest.Matcher;
import java.io.IOException;
import java.util.HashMap;
import java.util.Locale;
import java.util.regex.Pattern;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
@ -25,12 +23,13 @@ import static org.hamcrest.core.Is.is;
public class TemplateUtilsTests extends ESTestCase {
private static final String TEST_TEMPLATE = "/monitoring-%s.json";
private static final String SIMPLE_TEST_TEMPLATE = "/monitoring-%s.json";
private static final String TEST_TEMPLATE_WITH_VARIABLES = "/template_with_variables-test.json";
public void testLoadTemplate() throws IOException {
public void testLoadTemplate() {
final int version = randomIntBetween(0, 10_000);
String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test");
String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), Pattern.quote("${monitoring.template.version}"));
String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test");
String source = TemplateUtils.loadTemplate(resource, String.valueOf(version), "monitoring.template.version");
assertThat(source, notNullValue());
assertThat(source.length(), greaterThan(0));
@ -46,9 +45,36 @@ public class TemplateUtilsTests extends ESTestCase {
"}\n"));
}
public void testLoadTemplate_GivenTemplateWithVariables() {
final int version = randomIntBetween(0, 10_000);
Map<String, String> variables = new HashMap<>();
variables.put("test.template.field_1", "test_field_1");
variables.put("test.template.field_2", "\"test_field_2\": {\"type\": \"long\"}");
String source = TemplateUtils.loadTemplate(TEST_TEMPLATE_WITH_VARIABLES, String.valueOf(version),
"test.template.version", variables);
assertThat(source, notNullValue());
assertThat(source.length(), greaterThan(0));
assertTemplate(source, equalTo("{\n" +
" \"index_patterns\": \".test-" + version + "\",\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
" \"_meta\": {\n" +
" \"template.version\": \"" + version + "\"\n" +
" },\n" +
" \"properties\": {\n" +
" \"test_field_1\": {\"type\": \"keyword\"},\n" +
" \"test_field_2\": {\"type\": \"long\"}\n" +
" }\n" +
" }\n" +
" }\n" +
"}\n"));
}
public void testLoad() throws IOException {
String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test");
BytesReference source = TemplateUtils.load(resource);
String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test");
String source = TemplateUtils.load(resource);
assertThat(source, notNullValue());
assertThat(source.length(), greaterThan(0));
}
@ -60,35 +86,34 @@ public class TemplateUtilsTests extends ESTestCase {
public void testValidateEmptySource() {
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
() -> TemplateUtils.validate(new BytesArray("")));
() -> TemplateUtils.validate(""));
assertThat(exception.getMessage(), is("Template must not be empty"));
}
public void testValidateInvalidSource() {
ElasticsearchParseException exception = expectThrows(ElasticsearchParseException.class,
() -> TemplateUtils.validate(new BytesArray("{\"foo\": \"bar")));
() -> TemplateUtils.validate("{\"foo\": \"bar"));
assertThat(exception.getMessage(), is("Invalid template"));
}
public void testValidate() throws IOException {
String resource = String.format(Locale.ROOT, TEST_TEMPLATE, "test");
String resource = String.format(Locale.ROOT, SIMPLE_TEST_TEMPLATE, "test");
TemplateUtils.validate(TemplateUtils.load(resource));
}
public void testFilter() {
assertTemplate(TemplateUtils.filter(new BytesArray("${monitoring.template.version}"), "0",
Pattern.quote("${monitoring.template.version}")), equalTo("0"));
assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}\"}"), "1",
Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-1\"}"));
assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"${monitoring.template.version}-test\"}"), "2",
Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"2-test\"}"));
assertTemplate(TemplateUtils.filter(new BytesArray("{\"template\": \"test-${monitoring.template.version}-test\"}"), "3",
Pattern.quote("${monitoring.template.version}")), equalTo("{\"template\": \"test-3-test\"}"));
public void testReplaceVariable() {
assertTemplate(TemplateUtils.replaceVariable("${monitoring.template.version}",
"monitoring.template.version", "0"), equalTo("0"));
assertTemplate(TemplateUtils.replaceVariable("{\"template\": \"test-${monitoring.template.version}\"}",
"monitoring.template.version", "1"), equalTo("{\"template\": \"test-1\"}"));
assertTemplate(TemplateUtils.replaceVariable("{\"template\": \"${monitoring.template.version}-test\"}",
"monitoring.template.version", "2"), equalTo("{\"template\": \"2-test\"}"));
assertTemplate(TemplateUtils.replaceVariable("{\"template\": \"test-${monitoring.template.version}-test\"}",
"monitoring.template.version", "3"), equalTo("{\"template\": \"test-3-test\"}"));
final int version = randomIntBetween(0, 100);
assertTemplate(TemplateUtils.filter(new BytesArray("{\"foo-${monitoring.template.version}\": " +
"\"bar-${monitoring.template.version}\"}"), String.valueOf(version),
Pattern.quote("${monitoring.template.version}")),
assertTemplate(TemplateUtils.replaceVariable("{\"foo-${monitoring.template.version}\": " +
"\"bar-${monitoring.template.version}\"}", "monitoring.template.version", String.valueOf(version)),
equalTo("{\"foo-" + version + "\": \"bar-" + version + "\"}"));
}

14
x-pack/plugin/core/src/test/resources/template_with_variables-test.json Normal file
View File

@ -0,0 +1,14 @@
{
"index_patterns": ".test-${test.template.version}",
"mappings": {
"doc": {
"_meta": {
"template.version": "${test.template.version}"
},
"properties": {
"${test.template.field_1}": {"type": "keyword"},
${test.template.field_2}
}
}
}
}

6
x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java
View File

@ -24,7 +24,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.function.UnaryOperator;
import java.util.regex.Pattern;
/**
* This class activates/deactivates the logstash modules depending if we're running a node client or transport client
@ -35,8 +34,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin {
private static final String LOGSTASH_TEMPLATE_FILE_NAME = "logstash-management";
private static final String LOGSTASH_INDEX_TEMPLATE_NAME = ".logstash-management";
private static final String OLD_LOGSTASH_INDEX_NAME = "logstash-index-template";
private static final String TEMPLATE_VERSION_PATTERN =
Pattern.quote("${logstash.template.version}");
private static final String TEMPLATE_VERSION_VARIABLE = "logstash.template.version";
private final boolean enabled;
private final boolean transportClientMode;
@ -66,7 +64,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin {
return templates -> {
templates.keySet().removeIf(OLD_LOGSTASH_INDEX_NAME::equals);
TemplateUtils.loadTemplateIntoMap("/" + LOGSTASH_TEMPLATE_FILE_NAME + ".json", templates, LOGSTASH_INDEX_TEMPLATE_NAME,
Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN, LogManager.getLogger(Logstash.class));
Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE, LogManager.getLogger(Logstash.class));
//internal representation of typeless templates requires the default "_doc" type, which is also required for internal templates
assert templates.get(LOGSTASH_INDEX_TEMPLATE_NAME).mappings().get(MapperService.SINGLE_MAPPING_NAME) != null;
return templates;

6
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java
View File

@ -16,7 +16,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.SecuritySettingsSourceField;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.rollup.job.RollupJob;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.junit.After;
@ -749,7 +749,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
// There should be a notification saying that there was a problem extracting data
client().performRequest(new Request("POST", "/_refresh"));
Response notificationsResponse = client().performRequest(
new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
"action [indices:data/read/search] is unauthorized for user [ml_admin_plus_data]\""));
@ -956,7 +956,7 @@ public class DatafeedJobsRestIT extends ESRestTestCase {
// There should be a notification saying that there was a problem extracting data
client().performRequest(new Request("POST", "/_refresh"));
Response notificationsResponse = client().performRequest(
new Request("GET", AuditorField.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
new Request("GET", NotificationsIndex.NOTIFICATIONS_INDEX + "/_search?size=1000&q=job_id:" + jobId));
String notificationsResponseAsString = EntityUtils.toString(notificationsResponse.getEntity());
assertThat(notificationsResponseAsString, containsString("\"message\":\"Datafeed is encountering errors extracting data: " +
"action [indices:admin/xpack/rollup/search] is unauthorized for user [ml_admin_plus_data]\""));

6
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DeleteExpiredDataIT.java
View File

@ -29,7 +29,7 @@ import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.core.ml.job.results.Bucket;
import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.junit.After;
import org.junit.Before;
@ -185,7 +185,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
.setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
.get().getHits().getTotalHits().value;
long totalNotificationsCountBeforeDelete =
client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value;
client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value;
assertThat(totalModelSizeStatsBeforeDelete, greaterThan(0L));
assertThat(totalNotificationsCountBeforeDelete, greaterThan(0L));
@ -233,7 +233,7 @@ public class DeleteExpiredDataIT extends MlNativeAutodetectIntegTestCase {
.setQuery(QueryBuilders.termQuery("result_type", "model_size_stats"))
.get().getHits().getTotalHits().value;
long totalNotificationsCountAfterDelete =
client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value;
client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX).get().getHits().getTotalHits().value;
assertThat(totalModelSizeStatsAfterDelete, equalTo(totalModelSizeStatsBeforeDelete));
assertThat(totalNotificationsCountAfterDelete, greaterThanOrEqualTo(totalNotificationsCountBeforeDelete));

4
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java
View File

@ -23,7 +23,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Operator;
import org.elasticsearch.xpack.core.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.core.ml.job.config.RuleScope;
import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.junit.After;
import java.io.IOException;
@ -188,7 +188,7 @@ public class DetectionRulesIT extends MlNativeAutodetectIntegTestCase {
// Wait until the notification that the filter was updated is indexed
assertBusy(() -> {
SearchResponse searchResponse =
client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX)
client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX)
.setSize(1)
.addSort("timestamp", SortOrder.DESC)
.setQuery(QueryBuilders.boolQuery()

8
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeDataFrameAnalyticsIntegTestCase.java
View File

@ -39,7 +39,7 @@ import org.elasticsearch.xpack.core.ml.dataframe.evaluation.Evaluation;
import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.ml.utils.PhaseProgress;
import org.elasticsearch.xpack.core.ml.utils.QueryProvider;
import org.elasticsearch.xpack.ml.dataframe.StoredProgress;
@ -232,7 +232,7 @@ abstract class MlNativeDataFrameAnalyticsIntegTestCase extends MlNativeIntegTest
// Make sure we wrote to the audit
// Since calls to write the AbstractAuditor are sent and forgot (async) we could have returned from the start,
// finished the job (as this is a very short analytics job), all without the audit being fully written.
assertBusy(() -> assertTrue(indexExists(AuditorField.NOTIFICATIONS_INDEX)));
assertBusy(() -> assertTrue(indexExists(NotificationsIndex.NOTIFICATIONS_INDEX)));
@SuppressWarnings("unchecked")
Matcher<String>[] itemMatchers = Arrays.stream(expectedAuditMessagePrefixes).map(Matchers::startsWith).toArray(Matcher[]::new);
assertBusy(() -> {
@ -244,12 +244,12 @@ abstract class MlNativeDataFrameAnalyticsIntegTestCase extends MlNativeIntegTest
}
private static List<String> fetchAllAuditMessages(String dataFrameAnalyticsId) {
RefreshRequest refreshRequest = new RefreshRequest(AuditorField.NOTIFICATIONS_INDEX);
RefreshRequest refreshRequest = new RefreshRequest(NotificationsIndex.NOTIFICATIONS_INDEX);
RefreshResponse refreshResponse = client().execute(RefreshAction.INSTANCE, refreshRequest).actionGet();
assertThat(refreshResponse.getStatus().getStatus(), anyOf(equalTo(200), equalTo(201)));
SearchRequest searchRequest = new SearchRequestBuilder(client(), SearchAction.INSTANCE)
.setIndices(AuditorField.NOTIFICATIONS_INDEX)
.setIndices(NotificationsIndex.NOTIFICATIONS_INDEX)
.addSort("timestamp", SortOrder.ASC)
.setQuery(QueryBuilders.termQuery("job_id", dataFrameAnalyticsId))
.request();

6
x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java
View File

@ -21,7 +21,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.config.JobUpdate;
import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
import org.elasticsearch.xpack.core.ml.job.results.Bucket;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.junit.After;
import java.io.IOException;
@ -225,7 +225,7 @@ public class ScheduledEventsIT extends MlNativeAutodetectIntegTestCase {
// Wait until the notification that the process was updated is indexed
assertBusy(() -> {
SearchResponse searchResponse =
client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX)
client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX)
.setSize(1)
.addSort("timestamp", SortOrder.DESC)
.setQuery(QueryBuilders.boolQuery()
@ -301,7 +301,7 @@ public class ScheduledEventsIT extends MlNativeAutodetectIntegTestCase {
// Wait until the notification that the job was updated is indexed
assertBusy(() -> {
SearchResponse searchResponse =
client().prepareSearch(AuditorField.NOTIFICATIONS_INDEX)
client().prepareSearch(NotificationsIndex.NOTIFICATIONS_INDEX)
.setSize(1)
.addSort("timestamp", SortOrder.DESC)
.setQuery(QueryBuilders.boolQuery()

114
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java
View File

@ -9,21 +9,18 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.OriginSettingClient;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
@ -37,10 +34,8 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.analysis.TokenizerFactory;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider;
@ -135,8 +130,7 @@ import org.elasticsearch.xpack.core.ml.inference.MlInferenceNamedXContentProvide
import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.template.TemplateUtils;
import org.elasticsearch.xpack.ml.action.TransportCloseJobAction;
import org.elasticsearch.xpack.ml.action.TransportDeleteCalendarAction;
@ -215,7 +209,6 @@ import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult;
import org.elasticsearch.xpack.ml.dataframe.process.results.MemoryUsageEstimationResult;
import org.elasticsearch.xpack.ml.inference.ingest.InferenceProcessor;
import org.elasticsearch.xpack.ml.inference.loadingservice.ModelLoadingService;
import org.elasticsearch.xpack.ml.inference.persistence.InferenceInternalIndex;
import org.elasticsearch.xpack.ml.inference.persistence.TrainedModelProvider;
import org.elasticsearch.xpack.ml.job.JobManager;
import org.elasticsearch.xpack.ml.job.JobManagerHolder;
@ -321,7 +314,6 @@ import java.util.function.Supplier;
import java.util.function.UnaryOperator;
import static java.util.Collections.emptyList;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
public class MachineLearning extends Plugin implements SystemIndexPlugin, AnalysisPlugin, IngestPlugin, PersistentTaskPlugin {
public static final String NAME = "ml";
@ -525,6 +517,8 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin, Analys
return Collections.singletonList(new JobManagerHolder());
}
new MlIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
AnomalyDetectionAuditor anomalyDetectionAuditor = new AnomalyDetectionAuditor(client, clusterService.getNodeName());
DataFrameAnalyticsAuditor dataFrameAnalyticsAuditor = new DataFrameAnalyticsAuditor(client, clusterService.getNodeName());
InferenceAuditor inferenceAuditor = new InferenceAuditor(client, clusterService.getNodeName());
@ -898,112 +892,14 @@ public class MachineLearning extends Plugin implements SystemIndexPlugin, Analys
@Override
public UnaryOperator<Map<String, IndexTemplateMetaData>> getIndexTemplateMetaDataUpgrader() {
return templates -> {
try (XContentBuilder auditMapping = ElasticsearchMappings.auditMessageMapping()) {
IndexTemplateMetaData notificationMessageTemplate =
IndexTemplateMetaData.builder(AuditorField.NOTIFICATIONS_INDEX)
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(auditMapping))
.patterns(Collections.singletonList(AuditorField.NOTIFICATIONS_INDEX))
.version(Version.CURRENT.id)
.settings(Settings.builder()
// Our indexes are small and one shard puts the
// least possible burden on Elasticsearch
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"))
.build();
templates.put(AuditorField.NOTIFICATIONS_INDEX, notificationMessageTemplate);
} catch (IOException e) {
logger.warn("Error loading the template for the notification message index", e);
}
try (XContentBuilder docMapping = MlMetaIndex.docMapping()) {
IndexTemplateMetaData metaTemplate =
IndexTemplateMetaData.builder(MlMetaIndex.INDEX_NAME)
.patterns(Collections.singletonList(MlMetaIndex.INDEX_NAME))
.settings(Settings.builder()
// Our indexes are small and one shard puts the
// least possible burden on Elasticsearch
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"))
.version(Version.CURRENT.id)
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(docMapping))
.build();
templates.put(MlMetaIndex.INDEX_NAME, metaTemplate);
} catch (IOException e) {
logger.warn("Error loading the template for the " + MlMetaIndex.INDEX_NAME + " index", e);
}
try (XContentBuilder configMapping = ElasticsearchMappings.configMapping()) {
IndexTemplateMetaData configTemplate =
IndexTemplateMetaData.builder(AnomalyDetectorsIndex.configIndexName())
.patterns(Collections.singletonList(AnomalyDetectorsIndex.configIndexName()))
.settings(Settings.builder()
// Our indexes are small and one shard puts the
// least possible burden on Elasticsearch
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW))
.version(Version.CURRENT.id)
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(configMapping))
.build();
templates.put(AnomalyDetectorsIndex.configIndexName(), configTemplate);
} catch (IOException e) {
logger.warn("Error loading the template for the " + AnomalyDetectorsIndex.configIndexName() + " index", e);
}
try (XContentBuilder stateMapping = ElasticsearchMappings.stateMapping()) {
IndexTemplateMetaData stateTemplate =
IndexTemplateMetaData.builder(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX)
.patterns(Collections.singletonList(AnomalyDetectorsIndex.jobStateIndexPattern()))
// TODO review these settings
.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(stateMapping))
.version(Version.CURRENT.id)
.build();
templates.put(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX, stateTemplate);
} catch (IOException e) {
logger.error("Error loading the template for the " + AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX + " index", e);
}
try (XContentBuilder docMapping = ElasticsearchMappings.resultsMapping(SINGLE_MAPPING_NAME)) {
IndexTemplateMetaData jobResultsTemplate =
IndexTemplateMetaData.builder(AnomalyDetectorsIndex.jobResultsIndexPrefix())
.patterns(Collections.singletonList(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*"))
.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
// Sacrifice durability for performance: in the event of power
// failure we can lose the last 5 seconds of changes, but it's
// much faster
.put(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.getKey(), "async")
// set the default all search field
.put(IndexSettings.DEFAULT_FIELD_SETTING.getKey(), ElasticsearchMappings.ALL_FIELD_VALUES))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(docMapping))
.version(Version.CURRENT.id)
.build();
templates.put(AnomalyDetectorsIndex.jobResultsIndexPrefix(), jobResultsTemplate);
} catch (IOException e) {
logger.error("Error loading the template for the " + AnomalyDetectorsIndex.jobResultsIndexPrefix() + " indices", e);
}
try {
templates.put(InferenceIndexConstants.LATEST_INDEX_NAME, InferenceInternalIndex.getIndexTemplateMetaData());
} catch (IOException e) {
logger.error("Error loading the template for the " + InferenceIndexConstants.LATEST_INDEX_NAME + " index", e);
}
return templates;
};
return UnaryOperator.identity();
}
public static boolean allTemplatesInstalled(ClusterState clusterState) {
boolean allPresent = true;
List<String> templateNames =
Arrays.asList(
AuditorField.NOTIFICATIONS_INDEX,
NotificationsIndex.NOTIFICATIONS_INDEX,
MlMetaIndex.INDEX_NAME,
AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
AnomalyDetectorsIndex.jobResultsIndexPrefix(),

5
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlConfigMigrator.java
View File

@ -32,9 +32,11 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.action.OpenJobAction;
@ -43,7 +45,6 @@ import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
import org.elasticsearch.xpack.core.ml.utils.ToXContentParams;
import org.elasticsearch.xpack.ml.datafeed.persistence.DatafeedConfigProvider;
@ -494,7 +495,7 @@ public class MlConfigMigrator {
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW)
);
createIndexRequest.mapping(SINGLE_MAPPING_NAME, ElasticsearchMappings.configMapping());
createIndexRequest.mapping(SINGLE_MAPPING_NAME, MlConfigIndex.mapping(), XContentType.JSON);
} catch (Exception e) {
logger.error("error writing the .ml-config mappings", e);
listener.onFailure(e);

114
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MlIndexTemplateRegistry.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml;
import org.elasticsearch.Version;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
import org.elasticsearch.xpack.core.template.IndexTemplateRegistry;
import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class MlIndexTemplateRegistry extends IndexTemplateRegistry {
private static final String ROOT_RESOURCE_PATH = "/org/elasticsearch/xpack/core/ml/";
private static final String ANOMALY_DETECTION_PATH = ROOT_RESOURCE_PATH + "anomalydetection/";
private static final String VERSION_PATTERN = "xpack.ml.version";
private static final String VERSION_ID_PATTERN = "xpack.ml.version.id";
private static final IndexTemplateConfig ANOMALY_DETECTION_RESULTS_TEMPLATE = anomalyDetectionResultsTemplate();
private static final IndexTemplateConfig ANOMALY_DETECTION_STATE_TEMPLATE = new IndexTemplateConfig(
AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,ANOMALY_DETECTION_PATH + "state_index_template.json",
Version.CURRENT.id, VERSION_PATTERN,
Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
private static final IndexTemplateConfig META_TEMPLATE = new IndexTemplateConfig(MlMetaIndex.INDEX_NAME,
ROOT_RESOURCE_PATH + "meta_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
private static final IndexTemplateConfig NOTIFICATIONS_TEMPLATE = new IndexTemplateConfig(NotificationsIndex.NOTIFICATIONS_INDEX,
ROOT_RESOURCE_PATH + "notifications_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
private static final IndexTemplateConfig CONFIG_TEMPLATE = configTemplate();
private static final IndexTemplateConfig INFERENCE_TEMPLATE = new IndexTemplateConfig(InferenceIndexConstants.LATEST_INDEX_NAME,
ROOT_RESOURCE_PATH + "inference_index_template.json", Version.CURRENT.id, VERSION_PATTERN,
Collections.singletonMap(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id)));
private static IndexTemplateConfig configTemplate() {
Map<String, String> variables = new HashMap<>();
variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
variables.put("xpack.ml.config.max_result_window",
String.valueOf(AnomalyDetectorsIndex.CONFIG_INDEX_MAX_RESULTS_WINDOW));
variables.put("xpack.ml.config.mappings", MlConfigIndex.mapping());
return new IndexTemplateConfig(AnomalyDetectorsIndex.configIndexName(),
ROOT_RESOURCE_PATH + "config_index_template.json",
Version.CURRENT.id, VERSION_PATTERN,
variables);
}
private static IndexTemplateConfig anomalyDetectionResultsTemplate() {
Map<String, String> variables = new HashMap<>();
variables.put(VERSION_ID_PATTERN, String.valueOf(Version.CURRENT.id));
variables.put("xpack.ml.anomalydetection.results.mappings", AnomalyDetectorsIndex.resultsMapping());
return new IndexTemplateConfig(AnomalyDetectorsIndex.jobResultsIndexPrefix(),
ANOMALY_DETECTION_PATH + "results_index_template.json",
Version.CURRENT.id, VERSION_PATTERN,
variables);
}
public MlIndexTemplateRegistry(Settings nodeSettings, ClusterService clusterService, ThreadPool threadPool, Client client,
NamedXContentRegistry xContentRegistry) {
super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
}
@Override
protected boolean requiresMasterNode() {
return true;
}
@Override
protected List<IndexTemplateConfig> getTemplateConfigs() {
return Arrays.asList(
ANOMALY_DETECTION_RESULTS_TEMPLATE,
ANOMALY_DETECTION_STATE_TEMPLATE,
CONFIG_TEMPLATE,
INFERENCE_TEMPLATE,
META_TEMPLATE,
NOTIFICATIONS_TEMPLATE
);
}
@Override
protected List<LifecyclePolicyConfig> getPolicyConfigs() {
return Collections.emptyList();
}
@Override
protected String getOrigin() {
return ClientHelper.ML_ORIGIN;
}
}

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java
View File

@ -35,6 +35,7 @@ import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.common.validation.SourceDestValidator;
import org.elasticsearch.xpack.core.ml.MachineLearningField;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.action.PutDataFrameAnalyticsAction;
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
import org.elasticsearch.xpack.core.ml.job.messages.Messages;
@ -208,7 +209,7 @@ public class TransportPutDataFrameAnalyticsAction
}
ElasticsearchMappings.addDocMappingIfMissing(
AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings::configMapping,
MlConfigIndex::mapping,
client,
clusterState,
ActionListener.wrap(

3
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDatafeedAction.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.core.XPackSettings;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig;
@ -210,7 +211,7 @@ public class TransportPutDatafeedAction extends TransportMasterNodeAction<PutDat
}
ElasticsearchMappings.addDocMappingIfMissing(
AnomalyDetectorsIndex.configIndexName(),
ElasticsearchMappings::configMapping,
MlConfigIndex::mapping,
client,
clusterState,
ActionListener.wrap(mappingsUpdated, listener::onFailure));

134
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/persistence/InferenceInternalIndex.java
View File

@ -1,134 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ml.inference.persistence;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig;
import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants;
import java.io.IOException;
import java.util.Collections;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants.LATEST_INDEX_NAME;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.DATE;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.DYNAMIC;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.ENABLED;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.KEYWORD;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.LONG;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.PROPERTIES;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.TEXT;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.TYPE;
import static org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings.addMetaInformation;
/**
* Changelog of internal index versions
*
* Please list changes, increase the version in {@link InferenceInternalIndex} if you are 1st in this release cycle
*
* version 1 (7.5): initial
*/
public final class InferenceInternalIndex {
private InferenceInternalIndex() {}
public static XContentBuilder mappings() throws IOException {
return configMapping(SINGLE_MAPPING_NAME);
}
public static IndexTemplateMetaData getIndexTemplateMetaData() throws IOException {
IndexTemplateMetaData inferenceTemplate = IndexTemplateMetaData.builder(LATEST_INDEX_NAME)
.patterns(Collections.singletonList(LATEST_INDEX_NAME))
.version(Version.CURRENT.id)
.settings(Settings.builder()
// the configurations are expected to be small
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1"))
.putMapping(SINGLE_MAPPING_NAME, Strings.toString(mappings()))
.build();
return inferenceTemplate;
}
public static XContentBuilder configMapping(String mappingType) throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(mappingType);
addMetaInformation(builder);
// do not allow anything outside of the defined schema
builder.field(DYNAMIC, "false");
builder.startObject(PROPERTIES);
// Add the doc_type field
builder.startObject(InferenceIndexConstants.DOC_TYPE.getPreferredName())
.field(TYPE, KEYWORD)
.endObject();
addInferenceDocFields(builder);
addDefinitionDocFields(builder);
return builder.endObject()
.endObject()
.endObject();
}
private static void addInferenceDocFields(XContentBuilder builder) throws IOException {
builder.startObject(TrainedModelConfig.MODEL_ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(TrainedModelConfig.CREATED_BY.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(TrainedModelConfig.INPUT.getPreferredName())
.field(ENABLED, false)
.endObject()
.startObject(TrainedModelConfig.VERSION.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(TrainedModelConfig.DESCRIPTION.getPreferredName())
.field(TYPE, TEXT)
.endObject()
.startObject(TrainedModelConfig.CREATE_TIME.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(TrainedModelConfig.TAGS.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(TrainedModelConfig.METADATA.getPreferredName())
.field(ENABLED, false)
.endObject()
.startObject(TrainedModelConfig.ESTIMATED_OPERATIONS.getPreferredName())
.field(TYPE, LONG)
.endObject()
.startObject(TrainedModelConfig.ESTIMATED_HEAP_MEMORY_USAGE_BYTES.getPreferredName())
.field(TYPE, LONG)
.endObject();
}
private static void addDefinitionDocFields(XContentBuilder builder) throws IOException {
builder.startObject(TrainedModelDefinitionDoc.DOC_NUM.getPreferredName())
.field(TYPE, LONG)
.endObject()
.startObject(TrainedModelDefinitionDoc.DEFINITION.getPreferredName())
.field(ENABLED, false)
.endObject()
.startObject(TrainedModelDefinitionDoc.COMPRESSION_VERSION.getPreferredName())
.field(TYPE, LONG)
.endObject()
.startObject(TrainedModelDefinitionDoc.DEFINITION_LENGTH.getPreferredName())
.field(TYPE, LONG)
.endObject()
.startObject(TrainedModelDefinitionDoc.TOTAL_DEFINITION_LENGTH.getPreferredName())
.field(TYPE, LONG)
.endObject();
}
}

5
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java
View File

@ -30,13 +30,14 @@ import org.elasticsearch.env.Environment;
import org.elasticsearch.index.analysis.AnalysisRegistry;
import org.elasticsearch.persistent.PersistentTasksCustomMetaData;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.MachineLearningField;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.MlTasks;
import org.elasticsearch.xpack.core.ml.action.PutJobAction;
import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction;
import org.elasticsearch.xpack.core.ml.action.UpdateJobAction;
import org.elasticsearch.xpack.core.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisLimits;
import org.elasticsearch.xpack.core.ml.job.config.CategorizationAnalyzerConfig;
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
@ -294,7 +295,7 @@ public class JobManager {
return;
}
ElasticsearchMappings.addDocMappingIfMissing(
AnomalyDetectorsIndex.configIndexName(), ElasticsearchMappings::configMapping, client, state, putJobListener);
AnomalyDetectorsIndex.configIndexName(), MlConfigIndex::mapping, client, state, putJobListener);
},
putJobListener::onFailure
);

77
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProvider.java
View File

@ -56,6 +56,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.BoolQueryBuilder;
@ -101,6 +102,7 @@ import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats;
import org.elasticsearch.xpack.core.ml.job.results.Influencer;
import org.elasticsearch.xpack.core.ml.job.results.ModelPlot;
import org.elasticsearch.xpack.core.ml.job.results.ReservedFieldNames;
import org.elasticsearch.xpack.core.ml.job.results.Result;
import org.elasticsearch.xpack.core.ml.stats.CountAccumulator;
import org.elasticsearch.xpack.core.ml.stats.ForecastStats;
@ -131,7 +133,6 @@ import java.util.function.Consumer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;
@ -279,7 +280,7 @@ public class JobResultsProvider {
}
final String indexName = tempIndexName;
final ActionListener<Boolean> createAliasListener = ActionListener.wrap(success -> {
ActionListener<Boolean> indexAndMappingsListener = ActionListener.wrap(success -> {
final IndicesAliasesRequest request = client.admin().indices().prepareAliases()
.addAlias(indexName, readAliasName, QueryBuilders.termQuery(Job.ID.getPreferredName(), job.getId()))
.addAlias(indexName, writeAliasName).request();
@ -293,54 +294,50 @@ public class JobResultsProvider {
if (!state.getMetaData().hasIndex(indexName)) {
LOGGER.trace("ES API CALL: create index {}", indexName);
CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
// This assumes the requested mapping will be merged with mappings from the template,
// and may need to be revisited if template merging is ever refactored
try (XContentBuilder termFieldsMapping = ElasticsearchMappings.termFieldsMapping(termFields)) {
createIndexRequest.mapping(SINGLE_MAPPING_NAME, termFieldsMapping);
}
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, createIndexRequest,
ActionListener.<CreateIndexResponse>wrap(
r -> createAliasListener.onResponse(r.isAcknowledged()),
// Add the term field mappings and alias. The complication is that the state at the
// beginning of the operation doesn't have any knowledge of the index, as it's only
// just been created. So we need yet another operation to get the mappings for it.
r -> getLatestIndexMappingsAndAddTerms(indexName, termFields, indexAndMappingsListener),
e -> {
// Possible that the index was created while the request was executing,
// so we need to handle that possibility
if (ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException) {
LOGGER.info("Index already exists");
// Add the term field mappings and alias. The complication is that the state at the
// beginning of the operation doesn't have any knowledge of the index, as it's only
// just been created. So we need yet another operation to get the mappings for it.
getLatestIndexMappings(indexName, ActionListener.wrap(
response -> {
// Expect one index and one type. If this is not the case then it means the
// index has been deleted almost immediately after being created, and this is
// so unlikely that it's reasonable to fail the whole operation.
ImmutableOpenMap<String, MappingMetaData> indexMappings =
response.getMappings().iterator().next().value;
MappingMetaData typeMappings = indexMappings.iterator().next().value;
addTermsAndAliases(typeMappings, indexName, termFields, createAliasListener);
},
finalListener::onFailure
));
LOGGER.info("Index [{}] already exists", indexName);
getLatestIndexMappingsAndAddTerms(indexName, termFields, indexAndMappingsListener);
} else {
finalListener.onFailure(e);
}
}
), client.admin().indices()::create);
} else {
MappingMetaData mapping = state.metaData().index(indexName).mapping();
addTermsAndAliases(mapping, indexName, termFields, createAliasListener);
MappingMetaData indexMappings = state.metaData().index(indexName).mapping();
addTermsMapping(indexMappings, indexName, termFields, indexAndMappingsListener);
}
}
private void getLatestIndexMappings(final String indexName, final ActionListener<GetMappingsResponse> listener) {
private void getLatestIndexMappingsAndAddTerms(String indexName, Collection<String> termFields, ActionListener<Boolean> listener) {
ActionListener<GetMappingsResponse> getMappingsListener = ActionListener.wrap(
getMappingsResponse -> {
// Expect one index and one type. If this is not the case then it means the
// index has been deleted almost immediately after being created, and this is
// so unlikely that it's reasonable to fail the whole operation.
ImmutableOpenMap<String, MappingMetaData> indexMappings = getMappingsResponse.getMappings().iterator().next().value;
MappingMetaData typeMappings = indexMappings.iterator().next().value;
addTermsMapping(typeMappings, indexName, termFields, listener);
},
listener::onFailure
);
GetMappingsRequest getMappingsRequest = client.admin().indices().prepareGetMappings(indexName).request();
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getMappingsRequest, listener,
executeAsyncWithOrigin(client.threadPool().getThreadContext(), ML_ORIGIN, getMappingsRequest, getMappingsListener,
client.admin().indices()::getMappings);
}
private void addTermsAndAliases(final MappingMetaData mapping, final String indexName, final Collection<String> termFields,
final ActionListener<Boolean> listener) {
private void addTermsMapping(MappingMetaData mapping, String indexName, Collection<String> termFields,
ActionListener<Boolean> listener) {
long fieldCountLimit = MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.get(settings);
if (violatedFieldCountLimit(termFields.size(), fieldCountLimit, mapping)) {
@ -380,8 +377,9 @@ public class JobResultsProvider {
private void updateIndexMappingWithTermFields(String indexName, String mappingType, Collection<String> termFields,
ActionListener<Boolean> listener) {
// Put the whole mapping, not just the term fields, otherwise we'll wipe the _meta section of the mapping
try (XContentBuilder termFieldsMapping = ElasticsearchMappings.resultsMapping(mappingType, termFields)) {
try (XContentBuilder termFieldsMapping = JsonXContent.contentBuilder()) {
createTermFieldsMapping(termFieldsMapping, mappingType, termFields);
final PutMappingRequest request = client.admin().indices().preparePutMapping(indexName)
.setType(mappingType)
.setSource(termFieldsMapping).request();
@ -401,6 +399,21 @@ public class JobResultsProvider {
}
}
// Visible for testing
static void createTermFieldsMapping(XContentBuilder builder, String mappingType, Collection<String> termFields) throws IOException {
builder.startObject();
builder.startObject(mappingType);
builder.startObject("properties");
for (String fieldName : termFields) {
if (ReservedFieldNames.isValidFieldName(fieldName)) {
builder.startObject(fieldName).field(ElasticsearchMappings.TYPE, ElasticsearchMappings.KEYWORD).endObject();
}
}
builder.endObject();
builder.endObject();
builder.endObject();
}
/**
* Get the job's data counts
*

2
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java
View File

@ -441,7 +441,7 @@ public class AutodetectProcessManager implements ClusterStateListener {
// Try adding the results doc mapping - this updates to the latest version if an old mapping is present
ElasticsearchMappings.addDocMappingIfMissing(AnomalyDetectorsIndex.jobResultsAliasedName(jobId),
ElasticsearchMappings::resultsMapping, client, clusterState, resultsMappingUpdateHandler);
AnomalyDetectorsIndex::resultsMapping, client, clusterState, resultsMappingUpdateHandler);
}
private boolean createProcessAndSetRunning(ProcessContext processContext,

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/AnomalyDetectionAuditor.java
View File

@ -7,7 +7,7 @@ package org.elasticsearch.xpack.ml.notifications;
import org.elasticsearch.client.Client;
import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.ml.notifications.AnomalyDetectionAuditMessage;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@ -15,6 +15,6 @@ import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
public class AnomalyDetectionAuditor extends AbstractAuditor<AnomalyDetectionAuditMessage> {
public AnomalyDetectionAuditor(Client client, String nodeName) {
super(client, nodeName, AuditorField.NOTIFICATIONS_INDEX, ML_ORIGIN, AnomalyDetectionAuditMessage::new);
super(client, nodeName, NotificationsIndex.NOTIFICATIONS_INDEX, ML_ORIGIN, AnomalyDetectionAuditMessage::new);
}
}

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/DataFrameAnalyticsAuditor.java
View File

@ -7,7 +7,7 @@ package org.elasticsearch.xpack.ml.notifications;
import org.elasticsearch.client.Client;
import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.ml.notifications.DataFrameAnalyticsAuditMessage;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@ -15,6 +15,6 @@ import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
public class DataFrameAnalyticsAuditor extends AbstractAuditor<DataFrameAnalyticsAuditMessage> {
public DataFrameAnalyticsAuditor(Client client, String nodeName) {
super(client, nodeName, AuditorField.NOTIFICATIONS_INDEX, ML_ORIGIN, DataFrameAnalyticsAuditMessage::new);
super(client, nodeName, NotificationsIndex.NOTIFICATIONS_INDEX, ML_ORIGIN, DataFrameAnalyticsAuditMessage::new);
}
}

4
x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/notifications/InferenceAuditor.java
View File

@ -7,7 +7,7 @@ package org.elasticsearch.xpack.ml.notifications;
import org.elasticsearch.client.Client;
import org.elasticsearch.xpack.core.common.notifications.AbstractAuditor;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.ml.notifications.InferenceAuditMessage;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
@ -15,6 +15,6 @@ import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
public class InferenceAuditor extends AbstractAuditor<InferenceAuditMessage> {
public InferenceAuditor(Client client, String nodeName) {
super(client, nodeName, AuditorField.NOTIFICATIONS_INDEX, ML_ORIGIN, InferenceAuditMessage::new);
super(client, nodeName, NotificationsIndex.NOTIFICATIONS_INDEX, ML_ORIGIN, InferenceAuditMessage::new);
}
}

4
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java
View File

@ -47,7 +47,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Operator;
import org.elasticsearch.xpack.core.ml.job.config.RuleCondition;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.ml.MachineLearning;
import org.elasticsearch.xpack.ml.job.JobNodeSelector;
import org.elasticsearch.xpack.ml.job.process.autodetect.AutodetectProcessManager;
@ -235,7 +235,7 @@ public class TransportOpenJobActionTests extends ESTestCase {
indices.add(AnomalyDetectorsIndex.configIndexName());
indices.add(AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX);
indices.add(MlMetaIndex.INDEX_NAME);
indices.add(AuditorField.NOTIFICATIONS_INDEX);
indices.add(NotificationsIndex.NOTIFICATIONS_INDEX);
indices.add(AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT);
for (String indexName : indices) {
IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName);

111
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobResultsProviderIT.java
View File

@ -5,8 +5,11 @@
*/
package org.elasticsearch.xpack.ml.integration;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsAction;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
@ -15,6 +18,7 @@ import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.OriginSettingClient;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.routing.OperationRouting;
import org.elasticsearch.cluster.routing.UnassignedInfo;
@ -32,10 +36,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.xpack.core.ClientHelper;
import org.elasticsearch.xpack.core.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.MlMetadata;
import org.elasticsearch.xpack.core.ml.action.PutJobAction;
import org.elasticsearch.xpack.core.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.calendars.Calendar;
import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
@ -78,8 +82,13 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.hasItems;
import static org.hamcrest.Matchers.hasSize;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.not;
@ -116,6 +125,73 @@ public class JobResultsProviderIT extends MlSingleNodeTestCase {
waitForMlTemplates();
}
public void testPutJob_CreatesResultsIndex() {
Job.Builder job1 = new Job.Builder("first_job");
job1.setAnalysisConfig(createAnalysisConfig("by_field_1", Collections.emptyList()));
job1.setDataDescription(new DataDescription.Builder());
// Put fist job. This should create the results index as it's the first job.
client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job1)).actionGet();
String sharedResultsIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
Map<String, Object> mappingProperties = getIndexMappingProperties(sharedResultsIndex);
// Assert mappings have a few fields from the template
assertThat(mappingProperties.keySet(), hasItems("anomaly_score", "bucket_count"));
// Assert mappings have the by field
assertThat(mappingProperties.keySet(), hasItem("by_field_1"));
// Check aliases have been created
assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
AnomalyDetectorsIndex.resultsWriteAlias(job1.getId())));
// Now let's create a second job to test things work when the index exists already
assertThat(mappingProperties.keySet(), not(hasItem("by_field_2")));
Job.Builder job2 = new Job.Builder("second_job");
job2.setAnalysisConfig(createAnalysisConfig("by_field_2", Collections.emptyList()));
job2.setDataDescription(new DataDescription.Builder());
client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job2)).actionGet();
mappingProperties = getIndexMappingProperties(sharedResultsIndex);
// Assert mappings have a few fields from the template
assertThat(mappingProperties.keySet(), hasItems("anomaly_score", "bucket_count"));
// Assert mappings have the by field
assertThat(mappingProperties.keySet(), hasItems("by_field_1", "by_field_2"));
// Check aliases have been created
assertThat(getAliases(sharedResultsIndex), containsInAnyOrder(
AnomalyDetectorsIndex.jobResultsAliasedName(job1.getId()),
AnomalyDetectorsIndex.resultsWriteAlias(job1.getId()),
AnomalyDetectorsIndex.jobResultsAliasedName(job2.getId()),
AnomalyDetectorsIndex.resultsWriteAlias(job2.getId())
));
}
public void testPutJob_WithCustomResultsIndex() {
Job.Builder job = new Job.Builder("foo");
job.setResultsIndexName("bar");
job.setAnalysisConfig(createAnalysisConfig("by_field", Collections.emptyList()));
job.setDataDescription(new DataDescription.Builder());
client().execute(PutJobAction.INSTANCE, new PutJobAction.Request(job)).actionGet();
String customIndex = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar";
Map<String, Object> mappingProperties = getIndexMappingProperties(customIndex);
// Assert mappings have a few fields from the template
assertThat(mappingProperties.keySet(), hasItems("anomaly_score", "bucket_count"));
// Assert mappings have the by field
assertThat(mappingProperties.keySet(), hasItem("by_field"));
// Check aliases have been created
assertThat(getAliases(customIndex), containsInAnyOrder(AnomalyDetectorsIndex.jobResultsAliasedName(job.getId()),
AnomalyDetectorsIndex.resultsWriteAlias(job.getId())));
}
@AwaitsFix(bugUrl ="https://github.com/elastic/elasticsearch/issues/40134")
public void testMultipleSimultaneousJobCreations() {
@ -268,6 +344,39 @@ public class JobResultsProviderIT extends MlSingleNodeTestCase {
}
}
private Map<String, Object> getIndexMappingProperties(String index) {
GetMappingsRequest request = new GetMappingsRequest().indices(index);
GetMappingsResponse response = client().execute(GetMappingsAction.INSTANCE, request).actionGet();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings = response.getMappings();
assertNotNull(indexMappings);
ImmutableOpenMap<String, MappingMetaData> typeMappings = indexMappings.get(index);
assertNotNull("expected " + index + " in " + indexMappings, typeMappings);
assertEquals("expected 1 type in " + typeMappings, 1, typeMappings.size());
Map<String, Object> mappings = typeMappings.iterator().next().value.getSourceAsMap();
assertNotNull(mappings);
// Assert _meta info is present
assertThat(mappings.keySet(), hasItem("_meta"));
@SuppressWarnings("unchecked")
Map<String, Object> meta = (Map<String, Object>) mappings.get("_meta");
assertThat(meta.keySet(), hasItem("version"));
assertThat(meta.get("version"), equalTo(Version.CURRENT.toString()));
@SuppressWarnings("unchecked")
Map<String, Object> properties = (Map<String, Object>) mappings.get("properties");
assertNotNull("expected 'properties' field in " + mappings, properties);
return properties;
}
private Set<String> getAliases(String index) {
GetAliasesResponse getAliasesResponse = client().admin().indices().getAliases(
new GetAliasesRequest().indices(index)).actionGet();
ImmutableOpenMap<String, List<AliasMetaData>> aliases = getAliasesResponse.getAliases();
assertThat(aliases.containsKey(index), is(true));
List<AliasMetaData> aliasMetaData = aliases.get(index);
return aliasMetaData.stream().map(AliasMetaData::alias).collect(Collectors.toSet());
}
private List<Calendar> getCalendars(String jobId) throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicReference<Exception> exceptionHolder = new AtomicReference<>();

209
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/JobResultsProviderTests.java
View File

@ -9,8 +9,6 @@ import org.apache.lucene.search.TotalHits;
import org.elasticsearch.ResourceNotFoundException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.search.MultiSearchAction;
import org.elasticsearch.action.search.MultiSearchRequest;
import org.elasticsearch.action.search.MultiSearchRequestBuilder;
@ -19,26 +17,19 @@ import org.elasticsearch.action.search.SearchAction;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.Index;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.test.ESTestCase;
@ -47,17 +38,14 @@ import org.elasticsearch.xpack.core.action.util.QueryPage;
import org.elasticsearch.xpack.core.ml.datafeed.DatafeedTimingStats;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.TimingStats;
import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord;
import org.elasticsearch.xpack.core.ml.job.results.Bucket;
import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition;
import org.elasticsearch.xpack.core.ml.job.results.Influencer;
import org.elasticsearch.xpack.core.ml.job.results.Result;
import org.elasticsearch.xpack.core.ml.utils.ExponentialAverageCalculationContext;
import org.elasticsearch.xpack.ml.job.persistence.InfluencersQueryBuilder.InfluencersQuery;
import org.mockito.ArgumentCaptor;
import java.io.IOException;
import java.time.Instant;
@ -68,183 +56,20 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import static org.elasticsearch.xpack.core.ml.job.config.JobTests.buildJobBuilder;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
public class JobResultsProviderTests extends ESTestCase {
private static final String CLUSTER_NAME = "myCluster";
@SuppressWarnings("unchecked")
public void testCreateJobResultsIndex() {
String resultsIndexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + AnomalyDetectorsIndexFields.RESULTS_INDEX_DEFAULT;
QueryBuilder jobFilter = QueryBuilders.termQuery("job_id", "foo");
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME);
ArgumentCaptor<CreateIndexRequest> captor = ArgumentCaptor.forClass(CreateIndexRequest.class);
clientBuilder.createIndexRequest(captor, resultsIndexName);
clientBuilder.prepareAlias(resultsIndexName, AnomalyDetectorsIndex.jobResultsAliasedName("foo"), jobFilter);
clientBuilder.prepareAlias(resultsIndexName, AnomalyDetectorsIndex.resultsWriteAlias("foo"));
Job.Builder job = buildJobBuilder("foo");
JobResultsProvider provider = createProvider(clientBuilder.build());
AtomicReference<Boolean> resultHolder = new AtomicReference<>();
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().indices(ImmutableOpenMap.of()))
.build();
ClusterService clusterService = mock(ClusterService.class);
doAnswer(invocationOnMock -> {
AckedClusterStateUpdateTask<Boolean> task = (AckedClusterStateUpdateTask<Boolean>) invocationOnMock.getArguments()[1];
task.execute(cs);
return null;
}).when(clusterService).submitStateUpdateTask(eq("put-job-foo"), any(AckedClusterStateUpdateTask.class));
provider.createJobResultIndex(job.build(), cs, new ActionListener<Boolean>() {
@Override
public void onResponse(Boolean aBoolean) {
CreateIndexRequest request = captor.getValue();
assertNotNull(request);
assertEquals(resultsIndexName, request.index());
clientBuilder.verifyIndexCreated(resultsIndexName);
resultHolder.set(aBoolean);
}
@Override
public void onFailure(Exception e) {
fail(e.toString());
}
});
assertNotNull(resultHolder.get());
assertTrue(resultHolder.get());
}
@SuppressWarnings("unchecked")
public void testCreateJobWithExistingIndex() {
QueryBuilder jobFilter = QueryBuilders.termQuery("job_id", "foo");
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME);
clientBuilder.prepareAlias(AnomalyDetectorsIndex.jobResultsAliasedName("foo"),
AnomalyDetectorsIndex.jobResultsAliasedName("foo123"), jobFilter);
clientBuilder.preparePutMapping(mock(AcknowledgedResponse.class), Result.TYPE.getPreferredName());
GetMappingsResponse getMappingsResponse = mock(GetMappingsResponse.class);
ImmutableOpenMap<String, MappingMetaData> typeMappings = ImmutableOpenMap.<String, MappingMetaData>of();
ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings =
ImmutableOpenMap.<String, ImmutableOpenMap<String, MappingMetaData>>builder()
.fPut(AnomalyDetectorsIndex.jobResultsAliasedName("foo"), typeMappings).build();
when(getMappingsResponse.mappings()).thenReturn(mappings);
clientBuilder.prepareGetMapping(getMappingsResponse);
Job.Builder job = buildJobBuilder("foo123");
job.setResultsIndexName("foo");
JobResultsProvider provider = createProvider(clientBuilder.build());
Index index = mock(Index.class);
when(index.getName()).thenReturn(AnomalyDetectorsIndex.jobResultsAliasedName("foo"));
IndexMetaData indexMetaData = mock(IndexMetaData.class);
when(indexMetaData.getIndex()).thenReturn(index);
ImmutableOpenMap<String, AliasMetaData> aliases = ImmutableOpenMap.of();
when(indexMetaData.getAliases()).thenReturn(aliases);
when(indexMetaData.getSettings()).thenReturn(Settings.EMPTY);
ImmutableOpenMap<String, IndexMetaData> indexMap = ImmutableOpenMap.<String, IndexMetaData>builder()
.fPut(AnomalyDetectorsIndex.jobResultsAliasedName("foo"), indexMetaData).build();
ClusterState cs2 = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().indices(indexMap)).build();
ClusterService clusterService = mock(ClusterService.class);
doAnswer(invocationOnMock -> {
AckedClusterStateUpdateTask<Boolean> task = (AckedClusterStateUpdateTask<Boolean>) invocationOnMock.getArguments()[1];
task.execute(cs2);
return null;
}).when(clusterService).submitStateUpdateTask(eq("put-job-foo123"), any(AckedClusterStateUpdateTask.class));
doAnswer(invocationOnMock -> {
AckedClusterStateUpdateTask<Boolean> task = (AckedClusterStateUpdateTask<Boolean>) invocationOnMock.getArguments()[1];
task.execute(cs2);
return null;
}).when(clusterService).submitStateUpdateTask(eq("index-aliases"), any(AckedClusterStateUpdateTask.class));
provider.createJobResultIndex(job.build(), cs2, new ActionListener<Boolean>() {
@Override
public void onResponse(Boolean aBoolean) {
assertTrue(aBoolean);
verify(clientBuilder.build().admin().indices(), times(1)).preparePutMapping(any());
}
@Override
public void onFailure(Exception e) {
fail(e.toString());
}
});
}
@SuppressWarnings("unchecked")
public void testCreateJobRelatedIndicies_createsAliasBecauseIndexNameIsSet() {
String indexName = AnomalyDetectorsIndexFields.RESULTS_INDEX_PREFIX + "custom-bar";
String readAliasName = AnomalyDetectorsIndex.jobResultsAliasedName("foo");
String writeAliasName = AnomalyDetectorsIndex.resultsWriteAlias("foo");
QueryBuilder jobFilter = QueryBuilders.termQuery("job_id", "foo");
MockClientBuilder clientBuilder = new MockClientBuilder(CLUSTER_NAME);
ArgumentCaptor<CreateIndexRequest> captor = ArgumentCaptor.forClass(CreateIndexRequest.class);
clientBuilder.createIndexRequest(captor, indexName);
clientBuilder.prepareAlias(indexName, readAliasName, jobFilter);
clientBuilder.prepareAlias(indexName, writeAliasName);
clientBuilder.preparePutMapping(mock(AcknowledgedResponse.class), Result.TYPE.getPreferredName());
Job.Builder job = buildJobBuilder("foo");
job.setResultsIndexName("bar");
Client client = clientBuilder.build();
JobResultsProvider provider = createProvider(client);
ImmutableOpenMap<String, IndexMetaData> indexMap = ImmutableOpenMap.<String, IndexMetaData>builder().build();
ClusterState cs = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().indices(indexMap)).build();
ClusterService clusterService = mock(ClusterService.class);
doAnswer(invocationOnMock -> {
AckedClusterStateUpdateTask<Boolean> task = (AckedClusterStateUpdateTask<Boolean>) invocationOnMock.getArguments()[1];
task.execute(cs);
return null;
}).when(clusterService).submitStateUpdateTask(eq("put-job-foo"), any(AckedClusterStateUpdateTask.class));
provider.createJobResultIndex(job.build(), cs, new ActionListener<Boolean>() {
@Override
public void onResponse(Boolean aBoolean) {
verify(client.admin().indices(), times(1)).prepareAliases();
verify(client.admin().indices().prepareAliases(), times(1)).addAlias(indexName, readAliasName, jobFilter);
verify(client.admin().indices().prepareAliases(), times(1)).addAlias(indexName, writeAliasName);
}
@Override
public void onFailure(Exception e) {
fail(e.toString());
}
});
}
public void testBuckets_OneBucketNoInterim() throws IOException {
String jobId = "TestJobIdentification";
@ -853,7 +678,7 @@ public class JobResultsProviderTests extends ESTestCase {
contextMap.put(ExponentialAverageCalculationContext.LATEST_TIMESTAMP.getPreferredName(), Instant.ofEpochMilli(1000_000_000));
contextMap.put(ExponentialAverageCalculationContext.PREVIOUS_EXPONENTIAL_AVERAGE_MS.getPreferredName(), 200.0);
timingStatsMap.put(DatafeedTimingStats.EXPONENTIAL_AVG_CALCULATION_CONTEXT.getPreferredName(), contextMap);
List<Map<String, Object>> source = Arrays.asList(timingStatsMap);
SearchResponse response = createSearchResponse(source);
Client client = getMockedClient(
@ -1041,6 +866,32 @@ public class JobResultsProviderTests extends ESTestCase {
verifyNoMoreInteractions(client);
}
@SuppressWarnings("unchecked")
public void testCreateTermFieldsMapping() throws IOException {
XContentBuilder termFieldsMapping = JsonXContent.contentBuilder();
JobResultsProvider.createTermFieldsMapping(termFieldsMapping, "_doc", Arrays.asList("apple", "strawberry",
AnomalyRecord.BUCKET_SPAN.getPreferredName()));
XContentParser parser = createParser(termFieldsMapping);
Map<String, Object> typeMappings = (Map<String, Object>) parser.map().get("_doc");
Map<String, Object> properties = (Map<String, Object>) typeMappings.get("properties");
Map<String, Object> instanceMapping = (Map<String, Object>) properties.get("apple");
assertNotNull(instanceMapping);
String dataType = (String)instanceMapping.get("type");
assertEquals("keyword", dataType);
instanceMapping = (Map<String, Object>) properties.get("strawberry");
assertNotNull(instanceMapping);
dataType = (String)instanceMapping.get("type");
assertEquals("keyword", dataType);
// check no mapping for the reserved field
instanceMapping = (Map<String, Object>) properties.get(AnomalyRecord.BUCKET_SPAN.getPreferredName());
assertNull(instanceMapping);
}
private JobResultsProvider createProvider(Client client) {
return new JobResultsProvider(client, Settings.EMPTY);
}

239
x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/persistence/MockClientBuilder.java
View File

@ -10,37 +10,22 @@ import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequestBuilder;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexAction;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequestBuilder;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetRequestBuilder;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ClusterAdminClient;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -54,22 +39,14 @@ import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class MockClientBuilder {
@ -79,14 +56,11 @@ public class MockClientBuilder {
private ClusterAdminClient clusterAdminClient;
private IndicesAdminClient indicesAdminClient;
private IndicesAliasesRequestBuilder aliasesRequestBuilder;
public MockClientBuilder(String clusterName) {
client = mock(Client.class);
adminClient = mock(AdminClient.class);
clusterAdminClient = mock(ClusterAdminClient.class);
indicesAdminClient = mock(IndicesAdminClient.class);
aliasesRequestBuilder = mock(IndicesAliasesRequestBuilder.class);
when(client.admin()).thenReturn(adminClient);
when(adminClient.cluster()).thenReturn(clusterAdminClient);
@ -99,7 +73,7 @@ public class MockClientBuilder {
}
@SuppressWarnings({ "unchecked" })
public MockClientBuilder addClusterStatusYellowResponse() throws InterruptedException, ExecutionException {
public MockClientBuilder addClusterStatusYellowResponse() {
PlainActionFuture<ClusterHealthResponse> actionFuture = mock(PlainActionFuture.class);
ClusterHealthRequestBuilder clusterHealthRequestBuilder = mock(ClusterHealthRequestBuilder.class);
@ -110,64 +84,6 @@ public class MockClientBuilder {
return this;
}
@SuppressWarnings({ "unchecked" })
public MockClientBuilder addClusterStatusYellowResponse(String index) throws InterruptedException, ExecutionException {
PlainActionFuture<ClusterHealthResponse> actionFuture = mock(PlainActionFuture.class);
ClusterHealthRequestBuilder clusterHealthRequestBuilder = mock(ClusterHealthRequestBuilder.class);
when(clusterAdminClient.prepareHealth(index)).thenReturn(clusterHealthRequestBuilder);
when(clusterHealthRequestBuilder.setWaitForYellowStatus()).thenReturn(clusterHealthRequestBuilder);
when(clusterHealthRequestBuilder.execute()).thenReturn(actionFuture);
when(actionFuture.actionGet()).thenReturn(mock(ClusterHealthResponse.class));
return this;
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public MockClientBuilder addIndicesExistsResponse(String index, boolean exists) throws InterruptedException, ExecutionException {
ActionFuture actionFuture = mock(ActionFuture.class);
ArgumentCaptor<IndicesExistsRequest> requestCaptor = ArgumentCaptor.forClass(IndicesExistsRequest.class);
when(indicesAdminClient.exists(requestCaptor.capture())).thenReturn(actionFuture);
doAnswer(invocation -> {
IndicesExistsRequest request = (IndicesExistsRequest) invocation.getArguments()[0];
return request.indices()[0].equals(index) ? actionFuture : null;
}).when(indicesAdminClient).exists(any(IndicesExistsRequest.class));
when(actionFuture.get()).thenReturn(new IndicesExistsResponse(exists));
when(actionFuture.actionGet()).thenReturn(new IndicesExistsResponse(exists));
return this;
}
@SuppressWarnings({ "unchecked" })
public MockClientBuilder addIndicesDeleteResponse(String index, boolean exists, boolean exception,
ActionListener<AcknowledgedResponse> actionListener) throws InterruptedException, ExecutionException, IOException {
StreamInput si = mock(StreamInput.class);
// this looks complicated but Mockito can't mock the final method
// DeleteIndexResponse.isAcknowledged() and the only way to create
// one with a true response is reading from a stream.
when(si.readByte()).thenReturn((byte) 0x01);
AcknowledgedResponse response = DeleteIndexAction.INSTANCE.getResponseReader().read(si);
doAnswer(invocation -> {
DeleteIndexRequest deleteIndexRequest = (DeleteIndexRequest) invocation.getArguments()[0];
assertArrayEquals(new String[] { index }, deleteIndexRequest.indices());
if (exception) {
actionListener.onFailure(new InterruptedException());
} else {
actionListener.onResponse(new AcknowledgedResponse(true));
}
return null;
}).when(indicesAdminClient).delete(any(DeleteIndexRequest.class), any(ActionListener.class));
return this;
}
public MockClientBuilder prepareGet(String index, String type, String id, GetResponse response) {
GetRequestBuilder getRequestBuilder = mock(GetRequestBuilder.class);
when(getRequestBuilder.get()).thenReturn(response);
when(getRequestBuilder.setFetchSource(false)).thenReturn(getRequestBuilder);
when(client.prepareGet(index, type, id)).thenReturn(getRequestBuilder);
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder get(GetResponse response) {
doAnswer(new Answer<Void>() {
@ -192,64 +108,6 @@ public class MockClientBuilder {
return this;
}
@SuppressWarnings({ "rawtypes", "unchecked" })
public MockClientBuilder createIndexRequest(ArgumentCaptor<CreateIndexRequest> requestCapture, final String index) {
doAnswer(invocation -> {
CreateIndexResponse response = new CreateIndexResponse(true, true, index) {};
((ActionListener) invocation.getArguments()[1]).onResponse(response);
return null;
}).when(indicesAdminClient).create(requestCapture.capture(), any(ActionListener.class));
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareSearchExecuteListener(String index, SearchResponse response) {
SearchRequestBuilder builder = mock(SearchRequestBuilder.class);
when(builder.setTypes(anyString())).thenReturn(builder);
when(builder.addSort(any(SortBuilder.class))).thenReturn(builder);
when(builder.setFetchSource(anyBoolean())).thenReturn(builder);
when(builder.setScroll(anyString())).thenReturn(builder);
when(builder.addDocValueField(any(String.class))).thenReturn(builder);
when(builder.addDocValueField(any(String.class), any(String.class))).thenReturn(builder);
when(builder.addSort(any(String.class), any(SortOrder.class))).thenReturn(builder);
when(builder.setQuery(any())).thenReturn(builder);
when(builder.setSize(anyInt())).thenReturn(builder);
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<SearchResponse> listener = (ActionListener<SearchResponse>) invocationOnMock.getArguments()[0];
listener.onResponse(response);
return null;
}
}).when(builder).execute(any());
when(client.prepareSearch(eq(index))).thenReturn(builder);
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareSearchScrollExecuteListener(SearchResponse response) {
SearchScrollRequestBuilder builder = mock(SearchScrollRequestBuilder.class);
when(builder.setScroll(anyString())).thenReturn(builder);
when(builder.setScrollId(anyString())).thenReturn(builder);
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<SearchResponse> listener = (ActionListener<SearchResponse>) invocationOnMock.getArguments()[0];
listener.onResponse(response);
return null;
}
}).when(builder).execute(any());
when(client.prepareSearchScroll(anyString())).thenReturn(builder);
return this;
}
public MockClientBuilder prepareSearch(String index, String type, int from, int size, SearchResponse response,
ArgumentCaptor<QueryBuilder> filter) {
SearchRequestBuilder builder = mock(SearchRequestBuilder.class);
@ -352,38 +210,6 @@ public class MockClientBuilder {
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareAlias(String indexName, String alias, QueryBuilder filter) {
when(aliasesRequestBuilder.addAlias(eq(indexName), eq(alias), eq(filter))).thenReturn(aliasesRequestBuilder);
when(indicesAdminClient.prepareAliases()).thenReturn(aliasesRequestBuilder);
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<AcknowledgedResponse> listener =
(ActionListener<AcknowledgedResponse>) invocationOnMock.getArguments()[0];
listener.onResponse(mock(AcknowledgedResponse.class));
return null;
}
}).when(aliasesRequestBuilder).execute(any());
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareAlias(String indexName, String alias) {
when(aliasesRequestBuilder.addAlias(eq(indexName), eq(alias))).thenReturn(aliasesRequestBuilder);
when(indicesAdminClient.prepareAliases()).thenReturn(aliasesRequestBuilder);
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<AcknowledgedResponse> listener =
(ActionListener<AcknowledgedResponse>) invocationOnMock.getArguments()[1];
listener.onResponse(mock(AcknowledgedResponse.class));
return null;
}
}).when(indicesAdminClient).aliases(any(IndicesAliasesRequest.class), any(ActionListener.class));
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareBulk(BulkResponse response) {
PlainActionFuture<BulkResponse> actionFuture = mock(PlainActionFuture.class);
@ -402,70 +228,7 @@ public class MockClientBuilder {
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder preparePutMapping(AcknowledgedResponse response, String type) {
PutMappingRequestBuilder requestBuilder = mock(PutMappingRequestBuilder.class);
when(requestBuilder.setType(eq(type))).thenReturn(requestBuilder);
when(requestBuilder.setSource(any(XContentBuilder.class))).thenReturn(requestBuilder);
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<AcknowledgedResponse> listener =
(ActionListener<AcknowledgedResponse>) invocationOnMock.getArguments()[0];
listener.onResponse(response);
return null;
}
}).when(requestBuilder).execute(any());
when(indicesAdminClient.preparePutMapping(any())).thenReturn(requestBuilder);
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder prepareGetMapping(GetMappingsResponse response) {
GetMappingsRequestBuilder builder = mock(GetMappingsRequestBuilder.class);
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<GetMappingsResponse> listener =
(ActionListener<GetMappingsResponse>) invocationOnMock.getArguments()[0];
listener.onResponse(response);
return null;
}
}).when(builder).execute(any());
when(indicesAdminClient.prepareGetMappings(any())).thenReturn(builder);
return this;
}
@SuppressWarnings("unchecked")
public MockClientBuilder putTemplate(ArgumentCaptor<PutIndexTemplateRequest> requestCaptor) {
doAnswer(new Answer<Void>() {
@Override
public Void answer(InvocationOnMock invocationOnMock) throws Throwable {
ActionListener<AcknowledgedResponse> listener =
(ActionListener<AcknowledgedResponse>) invocationOnMock.getArguments()[1];
listener.onResponse(mock(AcknowledgedResponse.class));
return null;
}
}).when(indicesAdminClient).putTemplate(requestCaptor.capture(), any(ActionListener.class));
return this;
}
public Client build() {
return client;
}
public void verifyIndexCreated(String index) {
ArgumentCaptor<CreateIndexRequest> requestCaptor = ArgumentCaptor.forClass(CreateIndexRequest.class);
verify(indicesAdminClient).create(requestCaptor.capture(), any());
assertEquals(index, requestCaptor.getValue().index());
}
public void resetIndices() {
reset(indicesAdminClient);
}
}

5
x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java
View File

@ -66,7 +66,6 @@ import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.elasticsearch.cluster.metadata.IndexMetaData.INDEX_FORMAT_SETTING;
@ -85,7 +84,7 @@ public class SecurityIndexManager implements ClusterStateListener {
public static final String SECURITY_MAIN_TEMPLATE_7 = "security-index-template-7";
public static final String SECURITY_TOKENS_TEMPLATE_7 = "security-tokens-index-template-7";
public static final String SECURITY_VERSION_STRING = "security-version";
public static final String TEMPLATE_VERSION_PATTERN = Pattern.quote("${security.template.version}");
public static final String TEMPLATE_VERSION_VARIABLE = "security.template.version";
private static final Logger logger = LogManager.getLogger(SecurityIndexManager.class);
@ -434,7 +433,7 @@ public class SecurityIndexManager implements ClusterStateListener {
private static byte[] readTemplateAsBytes(String templateName) {
return TemplateUtils.loadTemplate("/" + templateName + ".json", Version.CURRENT.toString(),
SecurityIndexManager.TEMPLATE_VERSION_PATTERN).getBytes(StandardCharsets.UTF_8);
SecurityIndexManager.TEMPLATE_VERSION_VARIABLE).getBytes(StandardCharsets.UTF_8);
}
private static Tuple<String, Settings> parseMappingAndSettingsFromTemplateBytes(byte[] template) throws IOException {

12
x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java
View File

@ -19,10 +19,10 @@ import java.util.function.Supplier;
import org.elasticsearch.ElasticsearchStatusException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.action.ActionType;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.FilterClient;
@ -61,16 +61,16 @@ import org.hamcrest.Matchers;
import org.junit.Before;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.SECURITY_MAIN_TEMPLATE_7;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_PATTERN;
import static org.elasticsearch.xpack.security.support.SecurityIndexManager.TEMPLATE_VERSION_VARIABLE;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class SecurityIndexManagerTests extends ESTestCase {
@ -444,7 +444,7 @@ public class SecurityIndexManagerTests extends ESTestCase {
private static String loadTemplate(String templateName) {
final String resource = "/" + templateName + ".json";
return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_PATTERN);
return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), TEMPLATE_VERSION_VARIABLE);
}
public void testMappingVersionMatching() throws IOException {
@ -535,7 +535,7 @@ public class SecurityIndexManagerTests extends ESTestCase {
private static IndexMetaData.Builder createIndexMetadata(String indexName, String templateString) throws IOException {
String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(),
SecurityIndexManager.TEMPLATE_VERSION_PATTERN);
SecurityIndexManager.TEMPLATE_VERSION_VARIABLE);
PutIndexTemplateRequest request = new PutIndexTemplateRequest();
request.source(template, XContentType.JSON);
IndexMetaData.Builder indexMetaData = IndexMetaData.builder(indexName);
@ -574,7 +574,7 @@ public class SecurityIndexManagerTests extends ESTestCase {
private static IndexTemplateMetaData.Builder getIndexTemplateMetaData(String templateName, String templateString) throws IOException {
String template = TemplateUtils.loadTemplate(templateString, Version.CURRENT.toString(),
SecurityIndexManager.TEMPLATE_VERSION_PATTERN);
SecurityIndexManager.TEMPLATE_VERSION_VARIABLE);
PutIndexTemplateRequest request = new PutIndexTemplateRequest();
request.source(template, XContentType.JSON);
IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(templateName)

4
x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java
View File

@ -26,7 +26,7 @@ import org.elasticsearch.xpack.core.ml.MlMetaIndex;
import org.elasticsearch.xpack.core.ml.integration.MlRestTestStateCleaner;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex;
import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields;
import org.elasticsearch.xpack.core.ml.notifications.AuditorField;
import org.elasticsearch.xpack.core.ml.notifications.NotificationsIndex;
import org.elasticsearch.xpack.core.rollup.job.RollupJob;
import org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField;
import org.junit.After;
@ -89,7 +89,7 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase {
List<String> templates = new ArrayList<>();
templates.addAll(
Arrays.asList(
AuditorField.NOTIFICATIONS_INDEX,
NotificationsIndex.NOTIFICATIONS_INDEX,
MlMetaIndex.INDEX_NAME,
AnomalyDetectorsIndexFields.STATE_INDEX_PREFIX,
AnomalyDetectorsIndex.jobResultsIndexPrefix(),

58
x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
View File

@ -11,22 +11,21 @@ import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.WarningFailureException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig;
import org.elasticsearch.xpack.core.ml.MlConfigIndex;
import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
import org.elasticsearch.xpack.core.ml.job.config.Detector;
import org.elasticsearch.xpack.core.ml.job.config.Job;
import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings;
import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
import org.junit.Before;
@ -39,8 +38,8 @@ import java.util.List;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClusterRestartTestCase {
@ -48,24 +47,6 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
private static final String OLD_CLUSTER_JOB_ID = "ml-config-mappings-old-cluster-job";
private static final String NEW_CLUSTER_JOB_ID = "ml-config-mappings-new-cluster-job";
private static final Map<String, Object> EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS = getDataFrameAnalysisMappings();
@SuppressWarnings("unchecked")
private static Map<String, Object> getDataFrameAnalysisMappings() {
try (XContentBuilder builder = JsonXContent.contentBuilder()) {
builder.startObject();
ElasticsearchMappings.addDataFrameAnalyticsFields(builder);
builder.endObject();
Map<String, Object> asMap = builder.generator().contentType().xContent().createParser(
NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, BytesReference.bytes(builder).streamInput()).map();
return (Map<String, Object>) asMap.get(DataFrameAnalyticsConfig.ANALYSIS.getPreferredName());
} catch (IOException e) {
fail("Failed to initialize expected data frame analysis mappings");
}
return null;
}
@Override
protected Settings restClientSettings() {
String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
@ -90,16 +71,16 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID);
if (getOldClusterVersion().onOrAfter(Version.V_7_3_0)) {
// .ml-config has mappings for analytics as the feature was introduced in 7.3.0
assertThat(mappingsForDataFrameAnalysis(), is(notNullValue()));
assertThat(getDataFrameAnalysisMappings().keySet(), hasItem("outlier_detection"));
} else {
// .ml-config does not yet have correct mappings, it will need an update after cluster is upgraded
assertThat(mappingsForDataFrameAnalysis(), is(nullValue()));
assertThat(getDataFrameAnalysisMappings(), is(nullValue()));
}
} else {
// trigger .ml-config index mappings update
createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID);
// assert that the mappings are updated
assertThat(mappingsForDataFrameAnalysis(), is(equalTo(EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS)));
assertThat(getDataFrameAnalysisMappings(), equalTo(loadDataFrameAnalysisMappings()));
}
}
@ -110,8 +91,7 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
}
private void createAnomalyDetectorJob(String jobId) throws IOException {
Detector.Builder detector = new Detector.Builder("metric", "responsetime")
.setByFieldName("airline");
Detector.Builder detector = new Detector.Builder("metric", "responsetime");
AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()))
.setBucketSpan(TimeValue.timeValueMinutes(10));
Job.Builder job = new Job.Builder(jobId)
@ -125,7 +105,7 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
}
@SuppressWarnings("unchecked")
private Map<String, Object> mappingsForDataFrameAnalysis() throws Exception {
private Map<String, Object> getConfigIndexMappings() throws Exception {
Request getIndexMappingsRequest = new Request("GET", ".ml-config/_mappings");
Response getIndexMappingsResponse;
try {
@ -140,7 +120,25 @@ public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClust
if (mappings.containsKey("doc")) {
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "doc");
}
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "properties", "analysis");
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "properties");
return mappings;
}
@SuppressWarnings("unchecked")
private Map<String, Object> getDataFrameAnalysisMappings() throws Exception {
Map<String, Object> mappings = getConfigIndexMappings();
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "analysis", "properties");
return mappings;
}
@SuppressWarnings("unchecked")
private Map<String, Object> loadDataFrameAnalysisMappings() throws IOException {
String mapping = MlConfigIndex.mapping();
try (XContentParser parser = JsonXContent.jsonXContent.createParser(
NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, new BytesArray(mapping).streamInput())) {
Map<String, Object> mappings = parser.map();
mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "_doc", "properties", "analysis", "properties");
return mappings;
}
}
}

1
x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java
View File

@ -99,6 +99,7 @@ public class MlMappingsUpgradeIT extends AbstractUpgradeTestCase {
assertNotNull(indexLevel);
Map<String, Object> mappingsLevel = (Map<String, Object>) indexLevel.get("mappings");
assertNotNull(mappingsLevel);
Map<String, Object> metaLevel = (Map<String, Object>) mappingsLevel.get("_meta");
assertEquals(Collections.singletonMap("version", Version.CURRENT.toString()), metaLevel);
Map<String, Object> propertiesLevel = (Map<String, Object>) mappingsLevel.get("properties");