[Monitoring] Remove unused "node" _type from .monitoring-data-2 (elastic/x-pack-elasticsearch#1333)

This removes the "node" type from `.monitoring-data-2`. This data is sent to _both_ the time-based and non-time-based indexes for Elasticsearch, but the UI only used the time-based variant already.

This is another step in the process of removing the `.monitoring-data-2` index. There is now only one `_type` left in that index: `cluster_info`, which is used by the UI and phone home stats because it contains the license details _and_ the `stack_stats` (e.g., `xpack_usage`).

Original commit: elastic/x-pack-elasticsearch@2cadb5939d

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollector.java

@@ -68,10 +68,6 @@ public class ClusterStateCollector extends Collector {
                 // Adds a document for every node in the monitoring timestamped index
                 results.add(new ClusterStateNodeMonitoringDoc(monitoringId(), monitoringVersion(),
                         clusterUUID, timestamp, sourceNode, stateUUID, node.getId()));
-
-                // Adds a document for every node in the monitoring data index
-                results.add(new DiscoveryNodeMonitoringDoc(monitoringId(), monitoringVersion(),
-                        clusterUUID, timestamp, node));
             }
         }
 

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/DiscoveryNodeMonitoringDoc.java

@@ -1,30 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.collector.cluster;
-
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
-
-/**
- * Monitoring document collected by {@link ClusterStateCollector} that contains information
- * about every node of the cluster.
- */
-public class DiscoveryNodeMonitoringDoc extends MonitoringDoc {
-
-    public static final String TYPE = "node";
-
-    private final DiscoveryNode node;
-
-    public DiscoveryNodeMonitoringDoc(String monitoringId, String monitoringVersion,
-                                      String clusterUUID, long timestamp, DiscoveryNode node) {
-        super(monitoringId, monitoringVersion, TYPE, node.getId(), clusterUUID, timestamp, node);
-        this.node = node;
-    }
-
-    public DiscoveryNode getNode() {
-        return node;
-    }
-}

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java

@@ -13,7 +13,6 @@ import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitorin
 import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateNodeMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.collector.cluster.DiscoveryNodeMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc;
 import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc;
@@ -27,7 +26,6 @@ import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterInfoResolver;
 import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateNodeResolver;
 import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateResolver;
 import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStatsResolver;
-import org.elasticsearch.xpack.monitoring.resolver.cluster.DiscoveryNodeResolver;
 import org.elasticsearch.xpack.monitoring.resolver.indices.IndexRecoveryResolver;
 import org.elasticsearch.xpack.monitoring.resolver.indices.IndexStatsResolver;
 import org.elasticsearch.xpack.monitoring.resolver.indices.IndicesStatsResolver;
@@ -62,7 +60,6 @@ public class ResolversRegistry implements Iterable<MonitoringIndexNameResolver>
         registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, settings)));
         registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, settings)));
         registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, settings)));
-        registrations.add(resolveByClass(DiscoveryNodeMonitoringDoc.class, new DiscoveryNodeResolver()));
         registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, settings)));
         registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, settings)));
         registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, settings)));

plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/DiscoveryNodeResolver.java

@@ -1,49 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.cluster;
-
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.xcontent.ToXContent;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.xpack.monitoring.collector.cluster.DiscoveryNodeMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
-
-import java.io.IOException;
-import java.util.Map;
-
-public class DiscoveryNodeResolver extends MonitoringIndexNameResolver.Data<DiscoveryNodeMonitoringDoc> {
-
-    public static final String TYPE = "node";
-
-    @Override
-    protected void buildXContent(DiscoveryNodeMonitoringDoc document,
-                                 XContentBuilder builder, ToXContent.Params params) throws IOException {
-        builder.startObject(Fields.NODE);
-
-        DiscoveryNode node = document.getNode();
-        if (node != null) {
-            builder.field(Fields.NAME, node.getName());
-            builder.field(Fields.TRANSPORT_ADDRESS, node.getAddress().toString());
-
-            builder.startObject(Fields.ATTRIBUTES);
-            for (Map.Entry<String, String> entry : node.getAttributes().entrySet()) {
-                builder.field(entry.getKey(), entry.getValue());
-            }
-            builder.endObject();
-            builder.field(Fields.ID, node.getId());
-        }
-
-        builder.endObject();
-    }
-
-    static final class Fields {
-        static final String NODE = TYPE;
-        static final String NAME = "name";
-        static final String TRANSPORT_ADDRESS = "transport_address";
-        static final String ATTRIBUTES = "attributes";
-        static final String ID = "id";
-    }
-}

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollectorTests.java

@@ -124,14 +124,13 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
     }
 
     private void assertMonitoringDocs(Collection<MonitoringDoc> results, final int nbShards) {
-        assertThat("expecting 1 document for cluster state and 2 documents per node", results, hasSize(1 + internalCluster().size() * 2));
+        assertThat("expecting 1 document for cluster state and 1 document per node", results, hasSize(1 + internalCluster().size()));
 
         final ClusterState clusterState = securedClient().admin().cluster().prepareState().get().getState();
         final String clusterUUID = clusterState.getMetaData().clusterUUID();
         final String stateUUID = clusterState.stateUUID();
 
         List<ClusterStateNodeMonitoringDoc> clusterStateNodes = new ArrayList<>();
-        List<DiscoveryNodeMonitoringDoc> discoveryNodes = new ArrayList<>();
 
         for (MonitoringDoc doc : results) {
             assertThat(doc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem()));
@@ -139,8 +138,7 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
             assertThat(doc.getClusterUUID(), equalTo(clusterUUID));
             assertThat(doc.getTimestamp(), greaterThan(0L));
             assertThat(doc.getSourceNode(), notNullValue());
-            assertThat(doc, anyOf(instanceOf(ClusterStateMonitoringDoc.class),
-                    instanceOf(ClusterStateNodeMonitoringDoc.class), instanceOf(DiscoveryNodeMonitoringDoc.class)));
+            assertThat(doc, anyOf(instanceOf(ClusterStateMonitoringDoc.class), instanceOf(ClusterStateNodeMonitoringDoc.class)));
 
             if (doc instanceof ClusterStateMonitoringDoc) {
                 ClusterStateMonitoringDoc clusterStateMonitoringDoc = (ClusterStateMonitoringDoc) doc;
@@ -153,18 +151,12 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
                 assertThat(clusterStateNodeMonitoringDoc.getNodeId(), not(isEmptyOrNullString()));
                 clusterStateNodes.add(clusterStateNodeMonitoringDoc);
 
-            } else if (doc instanceof DiscoveryNodeMonitoringDoc) {
-                DiscoveryNodeMonitoringDoc discoveryNodeMonitoringDoc = (DiscoveryNodeMonitoringDoc) doc;
-                assertNotNull(discoveryNodeMonitoringDoc.getNode());
-                discoveryNodes.add(discoveryNodeMonitoringDoc);
-
             } else {
                 fail("unknown monitoring document type " + doc);
             }
         }
 
         assertThat(clusterStateNodes, hasSize(internalCluster().size()));
-        assertThat(discoveryNodes, hasSize(internalCluster().size()));
 
         for (final String nodeName : internalCluster().getNodeNames()) {
             final String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
@@ -176,15 +168,6 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase {
                     break;
                 }
             }
-            assertTrue("Could not find node id [" + nodeName + "]", found);
-
-            found = false;
-            for (DiscoveryNodeMonitoringDoc doc : discoveryNodes) {
-                if (nodeName.equals(doc.getNode().getName())) {
-                    found = true;
-                    break;
-                }
-            }
             assertTrue("Could not find node name [" + nodeName + "]", found);
         }
     }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java

@@ -215,9 +215,6 @@ public class LocalExporterTests extends MonitoringIntegTestCase {
             assertThat(client().prepareSearch(".monitoring-data-2").setTypes("cluster_info")
                     .get().getHits().getTotalHits(), greaterThan(0L));
 
-            assertEquals(numNodes, client().prepareSearch(".monitoring-data-2").setTypes("node")
-                    .get().getHits().getTotalHits());
-
             SearchResponse response = client().prepareSearch(".monitoring-es-2-*")
                     .setTypes("node_stats")
                     .setSize(0)

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateTests.java

@@ -30,7 +30,6 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
-import static org.hamcrest.core.Is.is;
 
 //test is just too slow, please fix it to not be sleep-based
 @LuceneTestCase.BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007")
@@ -150,62 +149,4 @@ public class ClusterStateTests extends MonitoringIntegTestCase {
         logger.debug("--> cluster state nodes successfully collected");
     }
 
-    public void testDiscoveryNodes() throws Exception {
-        final long nbNodes = internalCluster().size();
-
-        MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(MonitoringTemplateUtils.TEMPLATE_VERSION);
-        assertNotNull(dataResolver);
-
-        String dataIndex = dataResolver.indexPattern();
-        awaitIndexExists(dataIndex);
-
-        logger.debug("--> waiting for documents to be collected");
-        awaitMonitoringDocsCountOnPrimary(greaterThanOrEqualTo(nbNodes), DiscoveryNodeResolver.TYPE);
-
-        logger.debug("--> searching for monitoring documents of type [{}]", DiscoveryNodeResolver.TYPE);
-        SearchResponse response = client().prepareSearch(dataIndex).setTypes(DiscoveryNodeResolver.TYPE).setPreference("_primary").get();
-        assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes));
-
-        logger.debug("--> checking that every document contains the expected fields");
-        String[] filters = {
-                MonitoringIndexNameResolver.Fields.CLUSTER_UUID,
-                MonitoringIndexNameResolver.Fields.TIMESTAMP,
-                MonitoringIndexNameResolver.Fields.SOURCE_NODE,
-                DiscoveryNodeResolver.Fields.NODE,
-                DiscoveryNodeResolver.Fields.NODE + "."
-                        + DiscoveryNodeResolver.Fields.ID,
-                DiscoveryNodeResolver.Fields.NODE + "."
-                        + DiscoveryNodeResolver.Fields.NAME,
-                DiscoveryNodeResolver.Fields.NODE + "."
-                        + DiscoveryNodeResolver.Fields.ATTRIBUTES,
-                DiscoveryNodeResolver.Fields.NODE + "."
-                        + DiscoveryNodeResolver.Fields.TRANSPORT_ADDRESS,
-        };
-
-        for (SearchHit searchHit : response.getHits().getHits()) {
-            Map<String, Object> fields = searchHit.getSourceAsMap();
-
-            for (String filter : filters) {
-                assertContains(filter, fields);
-            }
-        }
-
-        for (final String nodeName : internalCluster().getNodeNames()) {
-            final String nodeId = internalCluster().clusterService(nodeName).localNode().getId();
-
-            logger.debug("--> getting monitoring document for node id [{}]", nodeId);
-            assertThat(client().prepareGet(dataIndex, DiscoveryNodeResolver.TYPE, nodeId).setPreference("_primary").get().isExists(),
-                    is(true));
-
-            // checks that document is not indexed
-            assertHitCount(client().prepareSearch(dataIndex).setSize(0)
-                    .setTypes(DiscoveryNodeResolver.TYPE)
-                    .setPreference("_primary")
-                    .setQuery(QueryBuilders.boolQuery()
-                            .should(matchQuery("node.id", nodeId))
-                            .should(matchQuery("node.name", nodeName))).get(), 0);
-        }
-
-        logger.debug("--> cluster state nodes successfully collected");
-    }
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/DiscoveryNodeResolverTests.java

@@ -1,58 +0,0 @@
-/*
- * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License;
- * you may not use this file except in compliance with the Elastic License.
- */
-package org.elasticsearch.xpack.monitoring.resolver.node;
-
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.node.DiscoveryNode;
-import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.common.xcontent.XContentType;
-import org.elasticsearch.test.VersionUtils;
-import org.elasticsearch.xpack.monitoring.collector.cluster.DiscoveryNodeMonitoringDoc;
-import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
-import org.elasticsearch.xpack.monitoring.resolver.cluster.DiscoveryNodeResolver;
-
-import java.util.UUID;
-
-import static java.util.Collections.emptyMap;
-import static java.util.Collections.emptySet;
-import static org.elasticsearch.test.VersionUtils.randomVersionBetween;
-import static org.hamcrest.Matchers.equalTo;
-
-public class DiscoveryNodeResolverTests extends MonitoringIndexNameResolverTestCase<DiscoveryNodeMonitoringDoc, DiscoveryNodeResolver> {
-
-    @Override
-    protected DiscoveryNodeMonitoringDoc newMonitoringDoc() {
-        DiscoveryNodeMonitoringDoc doc = new DiscoveryNodeMonitoringDoc(randomMonitoringId(),
-                randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L,
-                new DiscoveryNode(randomAlphaOfLength(3), UUID.randomUUID().toString(),
-                    buildNewFakeTransportAddress(), emptyMap(), emptySet(),
-                    randomVersionBetween(random(), VersionUtils.getFirstVersion(), Version.CURRENT)));
-        return doc;
-    }
-
-    @Override
-    protected boolean checkFilters() {
-        return false;
-    }
-
-    public void testDiscoveryNodeResolver() throws Exception {
-        DiscoveryNodeMonitoringDoc doc = newMonitoringDoc();
-
-        DiscoveryNodeResolver resolver = newResolver();
-        assertThat(resolver.index(doc), equalTo(".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION));
-
-        assertSource(resolver.source(doc, XContentType.JSON),
-                Sets.newHashSet(
-                        "cluster_uuid",
-                        "timestamp",
-                        "source_node",
-                        "node.id",
-                        "node.name",
-                        "node.transport_address",
-                        "node.attributes"), XContentType.JSON);
-    }
-}