honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove types from internal monitoring templates and bump to api 7 (#39888) (#39926) 

This commit removes the "doc" type from monitoring internal indexes.
The template still carries the "_doc" type since that is needed for
the internal representation.

This change impacts the following templates:
monitoring-alerts.json
monitoring-beats.json
monitoring-es.json
monitoring-kibana.json
monitoring-logstash.json

As part of the required changes, the system_api_version has been
bumped from "6" to "7" and support for version "2" has been dropped.

A new empty pipeline is now introduced for the version "7", and
the formerly empty "6" pipeline will now remove the type and re-direct
the request to the "7" index.

Additionally, to due to a difference in the internal representation
(which requires the inclusion of "_doc" type) and external representation
(which requires the exclusion of any type) a helper method is introduced
to help convert internal to external representation, and used by the
monitoring HTTP template exporter.

Relates #38637
This commit is contained in:
Jake Landis 2019-03-11 13:17:27 -05:00 committed by GitHub
parent 4f941c6963
commit b0b0f66669
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 150 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View File

@ -345,6 +345,19 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
builder.endObject();
}
/**
* Removes the nested type in the xContent representation of {@link IndexTemplateMetaData}.
*
* This method is useful to help bridge the gap between an the internal representation which still uses (the legacy format) a
* nested type in the mapping, and the external representation which does not use a nested type in the mapping.
*/
public static void removeType(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder) throws IOException {
builder.startObject();
toInnerXContent(indexTemplateMetaData, builder,
new ToXContent.MapParams(Collections.singletonMap("reduce_mappings", "true")), false);
builder.endObject();
}
/**
* Serializes the template to xContent, making sure not to nest mappings under the
* type name.
@ -361,6 +374,7 @@ public class IndexTemplateMetaData extends AbstractDiffable<IndexTemplateMetaDat
builder.endObject();
}
static void toInnerXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
XContentBuilder builder,
ToXContent.Params params) throws IOException {

2
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java
View File

@ -148,7 +148,7 @@ public class AutoFollowStatsMonitoringDocTests extends BaseMonitoringDocTestCase
Map<String, Object> template =
XContentHelper.convertToMap(XContentType.JSON.xContent(), MonitoringTemplateUtils.loadTemplate("es"), false);
Map<?, ?> autoFollowStatsMapping =
(Map<?, ?>) XContentMapValues.extractValue("mappings.doc.properties.ccr_auto_follow_stats.properties", template);
(Map<?, ?>) XContentMapValues.extractValue("mappings._doc.properties.ccr_auto_follow_stats.properties", template);
assertThat(serializedStatus.size(), equalTo(autoFollowStatsMapping.size()));
for (Map.Entry<String, Object> entry : serializedStatus.entrySet()) {

3
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java
View File

@ -237,7 +237,8 @@ public class FollowStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Fol
Map<String, Object> template =
XContentHelper.convertToMap(XContentType.JSON.xContent(), MonitoringTemplateUtils.loadTemplate("es"), false);
Map<?, ?> followStatsMapping = (Map<?, ?>) XContentMapValues.extractValue("mappings.doc.properties.ccr_stats.properties", template);
Map<?, ?> followStatsMapping = (Map<?, ?>) XContentMapValues
.extractValue("mappings._doc.properties.ccr_stats.properties", template);
assertThat(serializedStatus.size(), equalTo(followStatsMapping.size()));
for (Map.Entry<String, Object> entry : serializedStatus.entrySet()) {
String fieldName = entry.getKey();

3
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java
View File

@ -66,14 +66,13 @@ public class MonitoringBulkRequest extends ActionRequest {
* Parses a monitoring bulk request and builds the list of documents to be indexed.
*/
public MonitoringBulkRequest add(final MonitoredSystem system,
final String defaultType,
final BytesReference content,
final XContentType xContentType,
final long timestamp,
final long intervalMillis) throws IOException {
// MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest
new BulkRequestParser(false).parse(content, null, defaultType, null, null, null, true, xContentType,
new BulkRequestParser(false).parse(content, null, null, null, null, true, xContentType,
indexRequest -> {
// we no longer accept non-timestamped indexes from Kibana, LS, or Beats because we do not use the data
// and it was duplicated anyway; by simply dropping it, we allow BWC for older clients that still send it

3
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequestBuilder.java
View File

@ -26,12 +26,11 @@ public class MonitoringBulkRequestBuilder
}
public MonitoringBulkRequestBuilder add(final MonitoredSystem system,
final String type,
final BytesReference content,
final XContentType xContentType,
final long timestamp,
final long intervalMillis) throws IOException {
request.add(system, type, content, xContentType, timestamp, intervalMillis);
request.add(system, content, xContentType, timestamp, intervalMillis);
return this;
}

84
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java
View File

@ -33,18 +33,19 @@ public final class MonitoringTemplateUtils {
/**
* Current version of templates used in their name to differentiate from breaking changes (separate from product version).
* Version 7 has the same structure as version 6, but uses the `_doc` type.
*/
public static final String TEMPLATE_VERSION = "6";
public static final String TEMPLATE_VERSION = "7";
/**
* The previous version of templates, which we still support via the REST /_monitoring/bulk endpoint because
* nothing changed for those documents.
*/
public static final String OLD_TEMPLATE_VERSION = "2";
public static final String OLD_TEMPLATE_VERSION = "6";
/**
* IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate}.
*/
public static final String[] TEMPLATE_IDS = { "alerts", "es", "kibana", "logstash", "beats" };
public static final String[] TEMPLATE_IDS = { "alerts-7", "es", "kibana", "logstash", "beats" };
/**
* IDs of templates that can be used with {@linkplain #createEmptyTemplate(String) createEmptyTemplate} that are not managed by a
@ -54,7 +55,7 @@ public final class MonitoringTemplateUtils {
* instances will attempt to create a named template based on the templates that they expect (e.g., ".monitoring-es-2") and not the
* ones that we are creating.
*/
public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash", "alerts" };
public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash" }; //excluding alerts since 6.x watches use it
/**
* IDs of pipelines that can be used with
@ -99,7 +100,7 @@ public final class MonitoringTemplateUtils {
* @see #OLD_TEMPLATE_VERSION
*/
public static String createEmptyTemplate(final String id) {
// e.g., { "index_patterns": [ ".monitoring-data-2*" ], "version": 6000002 }
// e.g., { "index_patterns": [ ".monitoring-data-6*" ], "version": 6000002 }
return "{\"index_patterns\":[\".monitoring-" + id + "-" + OLD_TEMPLATE_VERSION + "*\"],\"version\":" + LAST_UPDATED_VERSION + "}";
}
@ -120,7 +121,7 @@ public final class MonitoringTemplateUtils {
* The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
* {@link BytesReference#bytes(XContentBuilder)}}.
*
* @param id The API version (e.g., "2") to use
* @param id The API version (e.g., "6") to use
* @param type The type of data you want to format for the request
* @return Never {@code null}. Always an ended-object.
* @throws IllegalArgumentException if {@code apiVersion} is unrecognized
@ -131,7 +132,7 @@ public final class MonitoringTemplateUtils {
case TEMPLATE_VERSION:
return emptyPipeline(type);
case OLD_TEMPLATE_VERSION:
return pipelineForApiVersion2(type);
return pipelineForApiVersion6(type);
}
throw new IllegalArgumentException("unrecognized pipeline API version [" + id + "]");
@ -139,13 +140,6 @@ public final class MonitoringTemplateUtils {
/**
* Create a pipeline to upgrade documents from {@link MonitoringTemplateUtils#OLD_TEMPLATE_VERSION}
* <pre><code>
* {
* "description" : "This pipeline upgrades documents ...",
* "version": 6000001,
* "processors": [ ]
* }
* </code></pre>
* The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
* {@link BytesReference#bytes(XContentBuilder)}}.
*
@ -153,81 +147,39 @@ public final class MonitoringTemplateUtils {
* @return Never {@code null}. Always an ended-object.
* @see #LAST_UPDATED_VERSION
*/
static XContentBuilder pipelineForApiVersion2(final XContentType type) {
static XContentBuilder pipelineForApiVersion6(final XContentType type) {
try {
// For now: We prepend the API version to the string so that it's easy to parse in the future; if we ever add metadata
// to pipelines, then it would better serve this use case
return XContentBuilder.builder(type.xContent()).startObject()
.field("description", "This pipeline upgrades documents from the older version of the Monitoring API to " +
"the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
"changes in those older documents before they are indexed from the older version (" +
OLD_TEMPLATE_VERSION + ").")
"the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
"changes in those older documents before they are indexed from the older version (" +
OLD_TEMPLATE_VERSION + ").")
.field("version", LAST_UPDATED_VERSION)
.startArray("processors")
.startObject()
// Drop the .monitoring-data-2 index and effectively drop unnecessary data (duplicate or simply unused)
// remove the type
.startObject("script")
.field("source",
"boolean legacyIndex = ctx._index == '.monitoring-data-2';" +
"if (legacyIndex || ctx._index.startsWith('.monitoring-es-2')) {" +
"if (ctx._type == 'cluster_info') {" +
"ctx._type = 'cluster_stats';" +
"ctx._id = null;" +
"} else if (legacyIndex || ctx._type == 'cluster_stats' || ctx._type == 'node') {" +
"String index = ctx._index;" +
"Object clusterUuid = ctx.cluster_uuid;" +
"Object timestamp = ctx.timestamp;" +
"ctx.clear();" +
"ctx._id = 'xpack_monitoring_2_drop_bucket';" +
"ctx._index = index;" +
"ctx._type = 'legacy_data';" +
"ctx.timestamp = timestamp;" +
"ctx.cluster_uuid = clusterUuid;" +
"}" +
"if (legacyIndex) {" +
"ctx._index = '<.monitoring-es-" + TEMPLATE_VERSION + "-{now}>';" +
"}" +
"}")
.endObject()
.endObject()
.startObject()
.startObject("rename")
.field("field", "_type")
.field("target_field", "type")
.endObject()
.endObject()
.startObject()
.startObject("set")
.field("field", "_type")
.field("value", "doc")
.field("source","ctx._type = null" )
.endObject()
.endObject()
.startObject()
// ensure the data lands in the correct index
.startObject("gsub")
.field("field", "_index")
.field("pattern", "(.monitoring-\\w+-)2(-.+)")
.field("pattern", "(.monitoring-\\w+-)6(-.+)")
.field("replacement", "$1" + TEMPLATE_VERSION + "$2")
.endObject()
.endObject()
.endArray()
.endObject();
.endObject();
} catch (final IOException e) {
throw new RuntimeException("Failed to create pipeline to upgrade from older version [" + OLD_TEMPLATE_VERSION +
"] to the newer version [" + TEMPLATE_VERSION + "].", e);
"] to the newer version [" + TEMPLATE_VERSION + "].", e);
}
}
/**
* Create an empty pipeline.
* <pre><code>
* {
* "description" : "This is a placeholder pipeline ...",
* "version": 6000001,
* "processors": [ ]
* }
* </code></pre>
* The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
* {@link BytesReference#bytes(XContentBuilder)}}.
*

4
x-pack/plugin/core/src/main/resources/monitoring-alerts.json → x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json
View File

@ -6,12 +6,12 @@
"number_of_shards": 1,
"number_of_replicas": 0,
"auto_expand_replicas": "0-1",
"format": 6,
"format": 7,
"codec": "best_compression"
}
},
"mappings": {
"doc": {
"_doc": {
"dynamic": false,
"properties": {
"timestamp": {

4
x-pack/plugin/core/src/main/resources/monitoring-beats.json
View File

@ -5,13 +5,13 @@
"settings": {
"index.auto_expand_replicas": "0-1",
"index.codec": "best_compression",
"index.format": 6,
"index.format": 7,
"index.number_of_replicas": 0,
"index.number_of_shards": 1
},
"version": 7000099,
"mappings": {
"doc": {
"_doc": {
"dynamic": false,
"properties": {
"beats_state": {

4
x-pack/plugin/core/src/main/resources/monitoring-es.json
View File

@ -5,11 +5,11 @@
"index.number_of_shards": 1,
"index.number_of_replicas": 0,
"index.auto_expand_replicas": "0-1",
"index.format": 6,
"index.format": 7,
"index.codec": "best_compression"
},
"mappings": {
"doc": {
"_doc": {
"date_detection": false,
"dynamic": false,
"properties": {

4
x-pack/plugin/core/src/main/resources/monitoring-kibana.json
View File

@ -5,11 +5,11 @@
"index.number_of_shards": 1,
"index.number_of_replicas": 0,
"index.auto_expand_replicas": "0-1",
"index.format": 6,
"index.format": 7,
"index.codec": "best_compression"
},
"mappings": {
"doc": {
"_doc": {
"dynamic": false,
"properties": {
"cluster_uuid": {

4
x-pack/plugin/core/src/main/resources/monitoring-logstash.json
View File

@ -5,11 +5,11 @@
"index.number_of_shards": 1,
"index.number_of_replicas": 0,
"index.auto_expand_replicas": "0-1",
"index.format": 6,
"index.format": 7,
"index.codec": "best_compression"
},
"mappings": {
"doc": {
"_doc": {
"dynamic": false,
"properties": {
"cluster_uuid": {

3
x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
View File

@ -592,7 +592,8 @@ public class HttpExporter extends Exporter {
resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, templateLoader));
}
// add old templates, like ".monitoring-data-2" and ".monitoring-es-2" so that other versions can continue to work
// Add dummy templates (e.g. ".monitoring-es-6") to enable the ability to check which version of the actual
// index template (e.g. ".monitoring-es") should be applied.
boolean createLegacyTemplates =
TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
if (createLegacyTemplates) {

27
x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
View File

@ -12,19 +12,26 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.function.Supplier;
import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
/**
* {@code TemplateHttpResource}s allow the checking and uploading of templates to a remote cluster.
* <p>
@ -87,9 +94,8 @@ public class TemplateHttpResource extends PublishableHttpResource {
*/
@Override
protected void doPublish(final RestClient client, final ActionListener<Boolean> listener) {
Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
putResource(client, listener, logger,
"/_template", templateName, parameters, this::templateToHttpEntity, "monitoring template",
"/_template", templateName, Collections.emptyMap(), this::templateToHttpEntity, "monitoring template",
resourceOwnerName, "monitoring cluster");
}
@ -98,8 +104,17 @@ public class TemplateHttpResource extends PublishableHttpResource {
*
* @return Never {@code null}.
*/
HttpEntity templateToHttpEntity() {
return new StringEntity(template.get(), ContentType.APPLICATION_JSON);
HttpEntity templateToHttpEntity() {
// the internal representation of a template has type nested under mappings.
// this uses xContent to help remove the type before sending to the remote cluster
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template.get())) {
XContentBuilder builder = JsonXContent.contentBuilder();
IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, templateName), builder);
return new StringEntity(BytesReference.bytes(builder).utf8ToString(), ContentType.APPLICATION_JSON);
} catch (IOException ex) {
throw new IllegalStateException("Cannot serialize template [" + templateName + "] for monitoring export", ex);
}
}
}

14
x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/rest/action/RestMonitoringBulkAction.java
View File

@ -6,6 +6,7 @@
package org.elasticsearch.xpack.monitoring.rest.action;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.logging.DeprecationLogger;
@ -40,7 +41,8 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
public static final String MONITORING_ID = "system_id";
public static final String MONITORING_VERSION = "system_api_version";
public static final String INTERVAL = "interval";
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestMonitoringBulkAction.class));
private static final Logger logger = LogManager.getLogger(RestMonitoringBulkAction.class);
private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
private final Map<MonitoredSystem, List<String>> supportedApiVersions;
public RestMonitoringBulkAction(Settings settings, RestController controller) {
@ -50,10 +52,6 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
POST, "/_xpack/monitoring/_bulk", deprecationLogger);
controller.registerWithDeprecatedHandler(PUT, "/_monitoring/bulk", this,
PUT, "/_xpack/monitoring/_bulk", deprecationLogger);
controller.registerWithDeprecatedHandler(POST, "/_monitoring/{type}/bulk", this,
POST, "/_xpack/monitoring/{type}/_bulk", deprecationLogger);
controller.registerWithDeprecatedHandler(PUT, "/_monitoring/{type}/bulk", this,
PUT, "/_xpack/monitoring/{type}/_bulk", deprecationLogger);
final List<String> allVersions = Arrays.asList(
MonitoringTemplateUtils.TEMPLATE_VERSION,
@ -63,8 +61,7 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
final Map<MonitoredSystem, List<String>> versionsMap = new HashMap<>();
versionsMap.put(MonitoredSystem.KIBANA, allVersions);
versionsMap.put(MonitoredSystem.LOGSTASH, allVersions);
// Beats did not report data in the 5.x timeline, so it should never send the original version
versionsMap.put(MonitoredSystem.BEATS, Collections.singletonList(MonitoringTemplateUtils.TEMPLATE_VERSION));
versionsMap.put(MonitoredSystem.BEATS, allVersions);
supportedApiVersions = Collections.unmodifiableMap(versionsMap);
}
@ -75,7 +72,6 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
@Override
public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient client) throws IOException {
final String defaultType = request.param("type");
final String id = request.param(MONITORING_ID);
if (Strings.isEmpty(id)) {
@ -106,7 +102,7 @@ public class RestMonitoringBulkAction extends XPackRestHandler {
final long intervalMillis = parseTimeValue(intervalAsString, INTERVAL).getMillis();
final MonitoringBulkRequestBuilder requestBuilder = client.monitoring().prepareMonitoringBulk();
requestBuilder.add(system, defaultType, request.content(), request.getXContentType(), timestamp, intervalMillis);
requestBuilder.add(system, request.content(), request.getXContentType(), timestamp, intervalMillis);
return channel -> requestBuilder.execute(new RestBuilderListener<MonitoringBulkResponse>(channel) {
@Override
public RestResponse buildResponse(MonitoringBulkResponse response, XContentBuilder builder) throws Exception {

8
x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json
View File

@ -2,7 +2,7 @@
"metadata": {
"name": "X-Pack Monitoring: Cluster Status (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-6",
"alert_index": ".monitoring-alerts-7",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "elasticsearch/indices",
"severity": 2100,
@ -71,7 +71,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6"
".monitoring-alerts-7"
],
"body": {
"size": 1,
@ -99,7 +99,7 @@
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-kibana-6-*"
".monitoring-kibana-7-*"
],
"body": {
"size": 1,
@ -140,7 +140,7 @@
"actions": {
"add_to_alerts_index": {
"index": {
"index": ".monitoring-alerts-6",
"index": ".monitoring-alerts-7",
"doc_id": "${monitoring.watch.unique_id}"
}
},

8
x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json
View File

@ -2,7 +2,7 @@
"metadata": {
"name": "X-Pack Monitoring: Nodes Changed (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-6",
"alert_index": ".monitoring-alerts-7",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "elasticsearch/nodes",
"severity": 1999,
@ -76,7 +76,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6"
".monitoring-alerts-7"
],
"body": {
"size": 1,
@ -116,7 +116,7 @@
"search": {
"request": {
"indices": [
".monitoring-kibana-*"
".monitoring-kibana-7-*"
],
"body": {
"size": 1,
@ -157,7 +157,7 @@
"actions": {
"add_to_alerts_index": {
"index": {
"index": ".monitoring-alerts-6"
"index": ".monitoring-alerts-7"
}
},
"send_email_to_admin": {

8
x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json
View File

@ -2,7 +2,7 @@
"metadata": {
"name": "X-Pack Monitoring: Elasticsearch Version Mismatch (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-6",
"alert_index": ".monitoring-alerts-7",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "elasticsearch/nodes",
"severity": 1000,
@ -67,7 +67,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6"
".monitoring-alerts-7"
],
"body": {
"size": 1,
@ -95,7 +95,7 @@
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-kibana-6-*"
".monitoring-kibana-7-*"
],
"body": {
"size": 1,
@ -136,7 +136,7 @@
"actions": {
"add_to_alerts_index": {
"index": {
"index": ".monitoring-alerts-6",
"index": ".monitoring-alerts-7",
"doc_id": "${monitoring.watch.unique_id}"
}
},

8
x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json
View File

@ -2,7 +2,7 @@
"metadata": {
"name": "X-Pack Monitoring: Kibana Version Mismatch (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-6",
"alert_index": ".monitoring-alerts-7",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "kibana/instances",
"severity": 1000,
@ -87,7 +87,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6"
".monitoring-alerts-7"
],
"body": {
"size": 1,
@ -115,7 +115,7 @@
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-kibana-6-*"
".monitoring-kibana-7-*"
],
"body": {
"size": 1,
@ -156,7 +156,7 @@
"actions": {
"add_to_alerts_index": {
"index": {
"index": ".monitoring-alerts-6",
"index": ".monitoring-alerts-7",
"doc_id": "${monitoring.watch.unique_id}"
}
},

8
x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json
View File

@ -2,7 +2,7 @@
"metadata": {
"name": "X-Pack Monitoring: Logstash Version Mismatch (${monitoring.watch.cluster_uuid})",
"xpack": {
"alert_index": ".monitoring-alerts-6",
"alert_index": ".monitoring-alerts-7",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"link": "logstash/instances",
"severity": 1000,
@ -87,7 +87,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6"
".monitoring-alerts-7"
],
"body": {
"size": 1,
@ -115,7 +115,7 @@
"request": {
"search_type": "query_then_fetch",
"indices": [
".monitoring-kibana-6-*"
".monitoring-kibana-7-*"
],
"body": {
"size": 1,
@ -156,7 +156,7 @@
"actions": {
"add_to_alerts_index": {
"index": {
"index": ".monitoring-alerts-6",
"index": ".monitoring-alerts-7",
"doc_id": "${monitoring.watch.unique_id}"
}
},

8
x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json
View File

@ -5,7 +5,7 @@
"link": "license",
"expires_days": [ 60, 30, 14, 7 ],
"severity": 0,
"alert_index": ".monitoring-alerts-6",
"alert_index": ".monitoring-alerts-7",
"cluster_uuid": "${monitoring.watch.cluster_uuid}",
"type": "monitoring",
"version_created": 7000099,
@ -72,7 +72,7 @@
"search": {
"request": {
"indices": [
".monitoring-alerts-6"
".monitoring-alerts-7"
],
"body": {
"size": 1,
@ -99,7 +99,7 @@
"search": {
"request": {
"indices": [
".monitoring-kibana-6-*"
".monitoring-kibana-7-*"
],
"body": {
"size": 1,
@ -140,7 +140,7 @@
"actions": {
"add_to_alerts_index": {
"index": {
"index": ".monitoring-alerts-6",
"index": ".monitoring-alerts-7",
"doc_id": "${monitoring.watch.unique_id}"
}
},

6
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java
View File

@ -124,7 +124,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
final long interval = randomNonNegativeLong();
final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
bulkRequest.add(system, defaultType, content.bytes(), xContentType, timestamp, interval);
bulkRequest.add(system, content.bytes(), xContentType, timestamp, interval);
final Collection<MonitoringBulkDoc> bulkDocs = bulkRequest.getDocs();
assertNotNull(bulkDocs);
@ -184,7 +184,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
bulkRequest.add(randomFrom(MonitoredSystem.values()), null, content.bytes(), xContentType, 0L, 0L)
bulkRequest.add(randomFrom(MonitoredSystem.values()), content.bytes(), xContentType, 0L, 0L)
);
assertThat(e.getMessage(), containsString("source is missing for monitoring document [][doc][" + nbDocs + "]"));
@ -221,7 +221,7 @@ public class MonitoringBulkRequestTests extends ESTestCase {
final MonitoringBulkRequest bulkRequest = new MonitoringBulkRequest();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
bulkRequest.add(randomFrom(MonitoredSystem.values()), null, content.bytes(), xContentType, 0L, 0L)
bulkRequest.add(randomFrom(MonitoredSystem.values()), content.bytes(), xContentType, 0L, 0L)
);
assertThat(e.getMessage(), containsString("unrecognized index name [" + indexName + "]"));

3
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/MonitoringTemplateUtilsTests.java
View File

@ -58,7 +58,7 @@ public class MonitoringTemplateUtilsTests extends ESTestCase {
assertTemplate(source, equalTo("{\n" +
" \"index_patterns\": \".monitoring-data-" + TEMPLATE_VERSION + "\",\n" +
" \"mappings\": {\n" +
" \"doc\": {\n" +
" \"_doc\": {\n" +
" \"_meta\": {\n" +
" \"template.version\": \"" + TEMPLATE_VERSION + "\"\n" +
" }\n" +
@ -115,5 +115,4 @@ public class MonitoringTemplateUtilsTests extends ESTestCase {
assertThat(indexName(formatter, MonitoredSystem.BEATS, timestamp),
equalTo(".monitoring-beats-" + TEMPLATE_VERSION + "-2017-03-08-13.47.58"));
}
}

31
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
View File

@ -13,6 +13,7 @@ import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
@ -21,8 +22,14 @@ import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.TestEnvironment;
import org.elasticsearch.license.XPackLicenseState;
@ -60,7 +67,6 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.LAST_UPDATED_VERSION;
import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.TEMPLATE_VERSION;
import static org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils.indexName;
@ -288,9 +294,8 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
recordedRequest = secondWebServer.takeRequest();
assertThat(recordedRequest.getMethod(), equalTo("PUT"));
assertThat(recordedRequest.getUri().getPath(), equalTo(resourcePrefix + template.v1()));
final Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
assertMonitorVersionQueryString(recordedRequest.getUri().getQuery(), parameters);
assertThat(recordedRequest.getBody(), equalTo(template.v2()));
assertMonitorVersionQueryString(recordedRequest.getUri().getQuery(), Collections.emptyMap());
assertThat(recordedRequest.getBody(), equalTo(getExternalTemplateRepresentation(template.v2())));
}
}
assertMonitorPipelines(secondWebServer, !pipelineExistsAlready, null, null);
@ -474,11 +479,13 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
assertThat(putRequest.getMethod(), equalTo("PUT"));
assertThat(putRequest.getUri().getPath(), equalTo(pathPrefix + resourcePrefix + resource.v1()));
Map<String, String> parameters = resourcePrefix.startsWith("/_template")
? Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true")
: Collections.emptyMap();
Map<String, String> parameters = Collections.emptyMap();
assertMonitorVersionQueryString(putRequest.getUri().getQuery(), parameters);
assertThat(putRequest.getBody(), equalTo(resource.v2()));
if (resourcePrefix.startsWith("/_template")) {
assertThat(putRequest.getBody(), equalTo(getExternalTemplateRepresentation(resource.v2())));
} else {
assertThat(putRequest.getBody(), equalTo(resource.v2()));
}
assertHeaders(putRequest, customHeaders);
}
}
@ -908,4 +915,12 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
return expectedTemplateNames;
}
private String getExternalTemplateRepresentation(String internalRepresentation) throws IOException {
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, internalRepresentation)) {
XContentBuilder builder = JsonXContent.contentBuilder();
IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, ""), builder);
return BytesReference.bytes(builder).utf8ToString();
}
}
}

20
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResourceTests.java
View File

@ -14,10 +14,8 @@ import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.Map;
import java.util.function.Supplier;
import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
import static org.hamcrest.Matchers.is;
/**
@ -26,14 +24,20 @@ import static org.hamcrest.Matchers.is;
public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTestCase {
private final String templateName = ".my_template";
private final String templateValue = "{\"template\":\".xyz-*\",\"mappings\":{}}";
private final Supplier<String> template = () -> templateValue;
//the internal representation has the type, the external representation should not
private final String templateValueInternal = "{\"order\":0,\"index_patterns\":[\".xyz-*\"],\"settings\":{},\"mappings\":{\"_doc\"" +
":{\"properties\":{\"one\":{\"properties\":{\"two\":{\"properties\":{\"name\":{\"type\":\"keyword\"}}}}}}}},\"aliases\":{}}";
private final String templateValueExternal = "{\"order\":0,\"index_patterns\":[\".xyz-*\"],\"settings\":{},\"mappings\"" +
":{\"properties\":{\"one\":{\"properties\":{\"two\":{\"properties\":{\"name\":{\"type\":\"keyword\"}}}}}}},\"aliases\":{}}";
private final Supplier<String> template = () -> templateValueInternal;
private final int minimumVersion = Math.min(MonitoringTemplateUtils.LAST_UPDATED_VERSION, Version.CURRENT.id);
private final TemplateHttpResource resource = new TemplateHttpResource(owner, masterTimeout, templateName, template);
public void testTemplateToHttpEntity() throws IOException {
final byte[] templateValueBytes = templateValue.getBytes(ContentType.APPLICATION_JSON.getCharset());
//the internal representation is converted to the external representation for the resource
final byte[] templateValueBytes = templateValueExternal.getBytes(ContentType.APPLICATION_JSON.getCharset());
final HttpEntity entity = resource.templateToHttpEntity();
assertThat(entity.getContentType().getValue(), is(ContentType.APPLICATION_JSON.toString()));
@ -80,13 +84,11 @@ public class TemplateHttpResourceTests extends AbstractPublishableHttpResourceTe
}
public void testDoPublishTrue() {
Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
assertPublishSucceeds(resource, "/_template", templateName, parameters, StringEntity.class);
assertPublishSucceeds(resource, "/_template", templateName, Collections.emptyMap(), StringEntity.class);
}
public void testDoPublishFalseWithException() {
Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
assertPublishWithException(resource, "/_template", templateName, parameters, StringEntity.class);
assertPublishWithException(resource, "/_template", templateName, Collections.emptyMap(), StringEntity.class);
}
public void testParameters() {

2
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java
View File

@ -218,7 +218,7 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase {
*/
private void checkMonitoringTemplates() {
final Set<String> templates = new HashSet<>();
templates.add(".monitoring-alerts");
templates.add(".monitoring-alerts-7");
templates.add(".monitoring-es");
templates.add(".monitoring-kibana");
templates.add(".monitoring-logstash");

6
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
View File

@ -81,8 +81,8 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
.field("index.number_of_replicas", 0)
.endObject()
.startObject("mappings")
// Still need use type, RestPutIndexTemplateAction#prepareRequestSource has logic that adds type if missing
.startObject("doc")
// The internal representation still requires a default type of _doc
.startObject("_doc")
.startObject("_meta")
.field("test", true)
.endObject()
@ -194,7 +194,7 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase
final String name = MonitoringTemplateUtils.templateName(system.getSystem());
for (IndexTemplateMetaData template : client().admin().indices().prepareGetTemplates(name).get().getIndexTemplates()) {
final String docMapping = template.getMappings().get("doc").toString();
final String docMapping = template.getMappings().get("_doc").toString();
assertThat(docMapping, notNullValue());
assertThat(docMapping, containsString("test"));

16
x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
View File

@ -104,13 +104,13 @@ public class MonitoringIT extends ESSingleNodeTestCase {
}
private String createBulkEntity() {
return "{\"index\":{}}\n" +
"{\"foo\":{\"bar\":0}}\n" +
"{\"index\":{}}\n" +
"{\"foo\":{\"bar\":1}}\n" +
"{\"index\":{}}\n" +
"{\"foo\":{\"bar\":2}}\n" +
"\n";
return "{\"index\":{\"_type\":\"monitoring_data_type\"}}\n" +
"{\"foo\":{\"bar\":0}}\n" +
"{\"index\":{\"_type\":\"monitoring_data_type\"}}\n" +
"{\"foo\":{\"bar\":1}}\n" +
"{\"index\":{\"_type\":\"monitoring_data_type\"}}\n" +
"{\"foo\":{\"bar\":2}}\n" +
"\n";
}
/**
@ -127,7 +127,7 @@ public class MonitoringIT extends ESSingleNodeTestCase {
final MonitoringBulkResponse bulkResponse =
new MonitoringBulkRequestBuilder(client())
.add(system, "monitoring_data_type", new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON,
.add(system, new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON,
System.currentTimeMillis(), interval.millis())
.get();

2
x-pack/plugin/monitoring/src/test/resources/monitoring-test.json
View File

@ -1,7 +1,7 @@
{
"index_patterns": ".monitoring-data-${monitoring.template.version}",
"mappings": {
"doc": {
"_doc": {
"_meta": {
"template.version": "${monitoring.template.version}"
}