From be1bb0ec7d72a63232c99015836c9034af4e016e Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 4 Feb 2019 10:58:03 -0800 Subject: [PATCH] Remove types from Monitoring plugin "backend" code (#37745) This PR removes the use of document types from the monitoring exporters and template + watches setup code. It does not remove the notion of types from the monitoring bulk API endpoint "front end" code as that code will eventually just go away in 8.0 and be replaced with Beats as collectors/shippers directly to the monitoring cluster. --- .../monitoring/exporter/http/HttpExportBulk.java | 6 ++++-- .../xpack/monitoring/exporter/local/LocalBulk.java | 6 +++--- .../watches/elasticsearch_cluster_status.json | 1 - .../monitoring/watches/elasticsearch_nodes.json | 3 +-- .../watches/elasticsearch_version_mismatch.json | 1 - .../monitoring/watches/kibana_version_mismatch.json | 1 - .../watches/logstash_version_mismatch.json | 1 - .../watches/xpack_license_expiration.json | 1 - .../action/MonitoringBulkRequestTests.java | 1 - .../exporter/local/LocalExporterIntegTests.java | 1 - .../local/LocalExporterResourceIntegTests.java | 1 + .../xpack/monitoring/integration/MonitoringIT.java | 13 +++++-------- 12 files changed, 14 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java index 7d17705decf..cd307322cb5 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExportBulk.java @@ -143,7 +143,6 @@ class HttpExportBulk extends ExportBulk { builder.startObject("index"); { builder.field("_index", index); - builder.field("_type", "doc"); if (id != null) { builder.field("_id", id); } @@ -163,7 +162,10 @@ class HttpExportBulk extends ExportBulk { // Adds final bulk separator out.write(xContent.streamSeparator()); - logger.trace("added index request [index={}, type={}, id={}]", index, doc.getType(), id); + logger.trace( + "http exporter [{}] - added index request [index={}, id={}, monitoring data type={}]", + name, index, id, doc.getType() + ); return BytesReference.toBytes(out.bytes()); } catch (Exception e) { diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java index b8337bea17d..3eb92fd68a3 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalBulk.java @@ -66,7 +66,7 @@ public class LocalBulk extends ExportBulk { try { final String index = MonitoringTemplateUtils.indexName(formatter, doc.getSystem(), doc.getTimestamp()); - final IndexRequest request = new IndexRequest(index, "doc"); + final IndexRequest request = new IndexRequest(index); if (Strings.hasText(doc.getId())) { request.id(doc.getId()); } @@ -82,8 +82,8 @@ public class LocalBulk extends ExportBulk { requestBuilder.add(request); if (logger.isTraceEnabled()) { - logger.trace("local exporter [{}] - added index request [index={}, type={}, id={}, pipeline={}]", - name, request.index(), request.type(), request.id(), request.getPipeline()); + logger.trace("local exporter [{}] - added index request [index={}, id={}, pipeline={}, monitoring data type={}]", + name, request.index(), request.id(), request.getPipeline(), doc.getType()); } } catch (Exception e) { if (exception == null) { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json index 1a177a26e7c..fca6171b623 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_cluster_status.json @@ -152,7 +152,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json index 4b45cb38f07..0566d03f21f 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_nodes.json @@ -157,8 +157,7 @@ "actions": { "add_to_alerts_index": { "index": { - "index": ".monitoring-alerts-6", - "doc_type": "doc" + "index": ".monitoring-alerts-6" } }, "send_email_to_admin": { diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json index 7cb494ce8e7..61d77d2b602 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/elasticsearch_version_mismatch.json @@ -148,7 +148,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json index 7ec176c5789..95de1a1a638 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/kibana_version_mismatch.json @@ -168,7 +168,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json index 69fc05f7cce..7448000fa85 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/logstash_version_mismatch.json @@ -168,7 +168,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json index e7c05fbcde1..d61bb3cd952 100644 --- a/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json +++ b/x-pack/plugin/monitoring/src/main/resources/monitoring/watches/xpack_license_expiration.json @@ -141,7 +141,6 @@ "add_to_alerts_index": { "index": { "index": ".monitoring-alerts-6", - "doc_type": "doc", "doc_id": "${monitoring.watch.unique_id}" } }, diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java index a8ab2960194..fc3bf633a39 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/MonitoringBulkRequestTests.java @@ -232,7 +232,6 @@ public class MonitoringBulkRequestTests extends ESTestCase { assertThat(e.getMessage(), containsString("unrecognized index name [" + indexName + "]")); //This test's JSON contains outdated references to types assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE); - } public void testSerialization() throws IOException { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java index cf82f3e55cd..171eeedf88c 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTests.java @@ -275,7 +275,6 @@ public class LocalExporterIntegTests extends LocalExporterIntegTestCase { assertTrue("document is missing cluster_uuid field", Strings.hasText((String) source.get("cluster_uuid"))); assertTrue("document is missing timestamp field", Strings.hasText(timestamp)); assertTrue("document is missing type field", Strings.hasText(type)); - assertEquals("document _type is 'doc'", "doc", hit.getType()); @SuppressWarnings("unchecked") Map docSource = (Map) source.get("doc"); diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java index 5448521594d..16ea27488d8 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java @@ -81,6 +81,7 @@ public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase .field("index.number_of_replicas", 0) .endObject() .startObject("mappings") + // Still need use type, RestPutIndexTemplateAction#prepareRequestSource has logic that adds type if missing .startObject("doc") .startObject("_meta") .field("test", true) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 1685de2667a..f5ed570acb5 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.license.License; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.document.RestBulkAction; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.collapse.CollapseBuilder; @@ -105,11 +104,11 @@ public class MonitoringIT extends ESSingleNodeTestCase { } private String createBulkEntity() { - return "{\"index\":{\"_type\":\"test\"}}\n" + + return "{\"index\":{}}\n" + "{\"foo\":{\"bar\":0}}\n" + - "{\"index\":{\"_type\":\"test\"}}\n" + + "{\"index\":{}}\n" + "{\"foo\":{\"bar\":1}}\n" + - "{\"index\":{\"_type\":\"test\"}}\n" + + "{\"index\":{}}\n" + "{\"foo\":{\"bar\":2}}\n" + "\n"; } @@ -128,7 +127,7 @@ public class MonitoringIT extends ESSingleNodeTestCase { final MonitoringBulkResponse bulkResponse = new MonitoringBulkRequestBuilder(client()) - .add(system, null, new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON, + .add(system, "monitoring_data_type", new BytesArray(createBulkEntity().getBytes("UTF-8")), XContentType.JSON, System.currentTimeMillis(), interval.millis()) .get(); @@ -179,10 +178,9 @@ public class MonitoringIT extends ESSingleNodeTestCase { equalTo(1L)); for (final SearchHit hit : hits.getHits()) { - assertMonitoringDoc(toMap(hit), system, "test", interval); + assertMonitoringDoc(toMap(hit), system, "monitoring_data_type", interval); } }); - assertWarnings(RestBulkAction.TYPES_DEPRECATION_MESSAGE); } /** @@ -265,7 +263,6 @@ public class MonitoringIT extends ESSingleNodeTestCase { final String index = (String) document.get("_index"); assertThat(index, containsString(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-")); - assertThat(document.get("_type"), equalTo("doc")); assertThat((String) document.get("_id"), not(isEmptyOrNullString())); final Map source = (Map) document.get("_source");