From f18a7f0f68bb5638a234eee06433ce047bb9f91f Mon Sep 17 00:00:00 2001 From: Chris Earle Date: Fri, 5 May 2017 17:48:30 -0400 Subject: [PATCH] [Monitoring] Remove unused "node" _type from .monitoring-data-2 (elastic/x-pack-elasticsearch#1333) This removes the "node" type from `.monitoring-data-2`. This data is sent to _both_ the time-based and non-time-based indexes for Elasticsearch, but the UI only used the time-based variant already. This is another step in the process of removing the `.monitoring-data-2` index. There is now only one `_type` left in that index: `cluster_info`, which is used by the UI and phone home stats because it contains the license details _and_ the `stack_stats` (e.g., `xpack_usage`). Original commit: elastic/x-pack-elasticsearch@2cadb5939d8b9aa71c42b40870c12bb14f0332a2 --- .../cluster/ClusterStateCollector.java | 4 -- .../cluster/DiscoveryNodeMonitoringDoc.java | 30 ---------- .../resolver/ResolversRegistry.java | 3 - .../cluster/DiscoveryNodeResolver.java | 49 --------------- .../cluster/ClusterStateCollectorTests.java | 21 +------ .../exporter/local/LocalExporterTests.java | 3 - .../resolver/cluster/ClusterStateTests.java | 59 ------------------- .../node/DiscoveryNodeResolverTests.java | 58 ------------------ 8 files changed, 2 insertions(+), 225 deletions(-) delete mode 100644 plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/DiscoveryNodeMonitoringDoc.java delete mode 100644 plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/DiscoveryNodeResolver.java delete mode 100644 plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/DiscoveryNodeResolverTests.java diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollector.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollector.java index 2e48dc1e314..0603d2feaa2 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollector.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollector.java @@ -68,10 +68,6 @@ public class ClusterStateCollector extends Collector { // Adds a document for every node in the monitoring timestamped index results.add(new ClusterStateNodeMonitoringDoc(monitoringId(), monitoringVersion(), clusterUUID, timestamp, sourceNode, stateUUID, node.getId())); - - // Adds a document for every node in the monitoring data index - results.add(new DiscoveryNodeMonitoringDoc(monitoringId(), monitoringVersion(), - clusterUUID, timestamp, node)); } } diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/DiscoveryNodeMonitoringDoc.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/DiscoveryNodeMonitoringDoc.java deleted file mode 100644 index dba491b1cd4..00000000000 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/DiscoveryNodeMonitoringDoc.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.monitoring.collector.cluster; - -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc; - -/** - * Monitoring document collected by {@link ClusterStateCollector} that contains information - * about every node of the cluster. - */ -public class DiscoveryNodeMonitoringDoc extends MonitoringDoc { - - public static final String TYPE = "node"; - - private final DiscoveryNode node; - - public DiscoveryNodeMonitoringDoc(String monitoringId, String monitoringVersion, - String clusterUUID, long timestamp, DiscoveryNode node) { - super(monitoringId, monitoringVersion, TYPE, node.getId(), clusterUUID, timestamp, node); - this.node = node; - } - - public DiscoveryNode getNode() { - return node; - } -} \ No newline at end of file diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java index c43ba1ee7ff..bb706cbeeec 100644 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java +++ b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/ResolversRegistry.java @@ -13,7 +13,6 @@ import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitorin import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateMonitoringDoc; import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStateNodeMonitoringDoc; import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterStatsMonitoringDoc; -import org.elasticsearch.xpack.monitoring.collector.cluster.DiscoveryNodeMonitoringDoc; import org.elasticsearch.xpack.monitoring.collector.indices.IndexRecoveryMonitoringDoc; import org.elasticsearch.xpack.monitoring.collector.indices.IndexStatsMonitoringDoc; import org.elasticsearch.xpack.monitoring.collector.indices.IndicesStatsMonitoringDoc; @@ -27,7 +26,6 @@ import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterInfoResolver; import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateNodeResolver; import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStateResolver; import org.elasticsearch.xpack.monitoring.resolver.cluster.ClusterStatsResolver; -import org.elasticsearch.xpack.monitoring.resolver.cluster.DiscoveryNodeResolver; import org.elasticsearch.xpack.monitoring.resolver.indices.IndexRecoveryResolver; import org.elasticsearch.xpack.monitoring.resolver.indices.IndexStatsResolver; import org.elasticsearch.xpack.monitoring.resolver.indices.IndicesStatsResolver; @@ -62,7 +60,6 @@ public class ResolversRegistry implements Iterable registrations.add(resolveByClass(ClusterStateNodeMonitoringDoc.class, new ClusterStateNodeResolver(id, settings))); registrations.add(resolveByClass(ClusterStateMonitoringDoc.class, new ClusterStateResolver(id, settings))); registrations.add(resolveByClass(ClusterStatsMonitoringDoc.class, new ClusterStatsResolver(id, settings))); - registrations.add(resolveByClass(DiscoveryNodeMonitoringDoc.class, new DiscoveryNodeResolver())); registrations.add(resolveByClass(IndexRecoveryMonitoringDoc.class, new IndexRecoveryResolver(id, settings))); registrations.add(resolveByClass(IndexStatsMonitoringDoc.class, new IndexStatsResolver(id, settings))); registrations.add(resolveByClass(IndicesStatsMonitoringDoc.class, new IndicesStatsResolver(id, settings))); diff --git a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/DiscoveryNodeResolver.java b/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/DiscoveryNodeResolver.java deleted file mode 100644 index ba8d7ca21ef..00000000000 --- a/plugin/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/DiscoveryNodeResolver.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.monitoring.resolver.cluster; - -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.monitoring.collector.cluster.DiscoveryNodeMonitoringDoc; -import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver; - -import java.io.IOException; -import java.util.Map; - -public class DiscoveryNodeResolver extends MonitoringIndexNameResolver.Data { - - public static final String TYPE = "node"; - - @Override - protected void buildXContent(DiscoveryNodeMonitoringDoc document, - XContentBuilder builder, ToXContent.Params params) throws IOException { - builder.startObject(Fields.NODE); - - DiscoveryNode node = document.getNode(); - if (node != null) { - builder.field(Fields.NAME, node.getName()); - builder.field(Fields.TRANSPORT_ADDRESS, node.getAddress().toString()); - - builder.startObject(Fields.ATTRIBUTES); - for (Map.Entry entry : node.getAttributes().entrySet()) { - builder.field(entry.getKey(), entry.getValue()); - } - builder.endObject(); - builder.field(Fields.ID, node.getId()); - } - - builder.endObject(); - } - - static final class Fields { - static final String NODE = TYPE; - static final String NAME = "name"; - static final String TRANSPORT_ADDRESS = "transport_address"; - static final String ATTRIBUTES = "attributes"; - static final String ID = "id"; - } -} diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollectorTests.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollectorTests.java index a051643ac4a..f8f4ce942ec 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollectorTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStateCollectorTests.java @@ -124,14 +124,13 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { } private void assertMonitoringDocs(Collection results, final int nbShards) { - assertThat("expecting 1 document for cluster state and 2 documents per node", results, hasSize(1 + internalCluster().size() * 2)); + assertThat("expecting 1 document for cluster state and 1 document per node", results, hasSize(1 + internalCluster().size())); final ClusterState clusterState = securedClient().admin().cluster().prepareState().get().getState(); final String clusterUUID = clusterState.getMetaData().clusterUUID(); final String stateUUID = clusterState.stateUUID(); List clusterStateNodes = new ArrayList<>(); - List discoveryNodes = new ArrayList<>(); for (MonitoringDoc doc : results) { assertThat(doc.getMonitoringId(), equalTo(MonitoredSystem.ES.getSystem())); @@ -139,8 +138,7 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { assertThat(doc.getClusterUUID(), equalTo(clusterUUID)); assertThat(doc.getTimestamp(), greaterThan(0L)); assertThat(doc.getSourceNode(), notNullValue()); - assertThat(doc, anyOf(instanceOf(ClusterStateMonitoringDoc.class), - instanceOf(ClusterStateNodeMonitoringDoc.class), instanceOf(DiscoveryNodeMonitoringDoc.class))); + assertThat(doc, anyOf(instanceOf(ClusterStateMonitoringDoc.class), instanceOf(ClusterStateNodeMonitoringDoc.class))); if (doc instanceof ClusterStateMonitoringDoc) { ClusterStateMonitoringDoc clusterStateMonitoringDoc = (ClusterStateMonitoringDoc) doc; @@ -153,18 +151,12 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { assertThat(clusterStateNodeMonitoringDoc.getNodeId(), not(isEmptyOrNullString())); clusterStateNodes.add(clusterStateNodeMonitoringDoc); - } else if (doc instanceof DiscoveryNodeMonitoringDoc) { - DiscoveryNodeMonitoringDoc discoveryNodeMonitoringDoc = (DiscoveryNodeMonitoringDoc) doc; - assertNotNull(discoveryNodeMonitoringDoc.getNode()); - discoveryNodes.add(discoveryNodeMonitoringDoc); - } else { fail("unknown monitoring document type " + doc); } } assertThat(clusterStateNodes, hasSize(internalCluster().size())); - assertThat(discoveryNodes, hasSize(internalCluster().size())); for (final String nodeName : internalCluster().getNodeNames()) { final String nodeId = internalCluster().clusterService(nodeName).localNode().getId(); @@ -176,15 +168,6 @@ public class ClusterStateCollectorTests extends AbstractCollectorTestCase { break; } } - assertTrue("Could not find node id [" + nodeName + "]", found); - - found = false; - for (DiscoveryNodeMonitoringDoc doc : discoveryNodes) { - if (nodeName.equals(doc.getNode().getName())) { - found = true; - break; - } - } assertTrue("Could not find node name [" + nodeName + "]", found); } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java index 59e0eec91e7..f8332e3b137 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterTests.java @@ -215,9 +215,6 @@ public class LocalExporterTests extends MonitoringIntegTestCase { assertThat(client().prepareSearch(".monitoring-data-2").setTypes("cluster_info") .get().getHits().getTotalHits(), greaterThan(0L)); - assertEquals(numNodes, client().prepareSearch(".monitoring-data-2").setTypes("node") - .get().getHits().getTotalHits()); - SearchResponse response = client().prepareSearch(".monitoring-es-2-*") .setTypes("node_stats") .setSize(0) diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateTests.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateTests.java index 86e6ed38c68..ebef2288721 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterStateTests.java @@ -30,7 +30,6 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.core.Is.is; //test is just too slow, please fix it to not be sleep-based @LuceneTestCase.BadApple(bugUrl = "https://github.com/elastic/x-plugins/issues/1007") @@ -150,62 +149,4 @@ public class ClusterStateTests extends MonitoringIntegTestCase { logger.debug("--> cluster state nodes successfully collected"); } - public void testDiscoveryNodes() throws Exception { - final long nbNodes = internalCluster().size(); - - MonitoringIndexNameResolver.Data dataResolver = new MockDataIndexNameResolver(MonitoringTemplateUtils.TEMPLATE_VERSION); - assertNotNull(dataResolver); - - String dataIndex = dataResolver.indexPattern(); - awaitIndexExists(dataIndex); - - logger.debug("--> waiting for documents to be collected"); - awaitMonitoringDocsCountOnPrimary(greaterThanOrEqualTo(nbNodes), DiscoveryNodeResolver.TYPE); - - logger.debug("--> searching for monitoring documents of type [{}]", DiscoveryNodeResolver.TYPE); - SearchResponse response = client().prepareSearch(dataIndex).setTypes(DiscoveryNodeResolver.TYPE).setPreference("_primary").get(); - assertThat(response.getHits().getTotalHits(), greaterThanOrEqualTo(nbNodes)); - - logger.debug("--> checking that every document contains the expected fields"); - String[] filters = { - MonitoringIndexNameResolver.Fields.CLUSTER_UUID, - MonitoringIndexNameResolver.Fields.TIMESTAMP, - MonitoringIndexNameResolver.Fields.SOURCE_NODE, - DiscoveryNodeResolver.Fields.NODE, - DiscoveryNodeResolver.Fields.NODE + "." - + DiscoveryNodeResolver.Fields.ID, - DiscoveryNodeResolver.Fields.NODE + "." - + DiscoveryNodeResolver.Fields.NAME, - DiscoveryNodeResolver.Fields.NODE + "." - + DiscoveryNodeResolver.Fields.ATTRIBUTES, - DiscoveryNodeResolver.Fields.NODE + "." - + DiscoveryNodeResolver.Fields.TRANSPORT_ADDRESS, - }; - - for (SearchHit searchHit : response.getHits().getHits()) { - Map fields = searchHit.getSourceAsMap(); - - for (String filter : filters) { - assertContains(filter, fields); - } - } - - for (final String nodeName : internalCluster().getNodeNames()) { - final String nodeId = internalCluster().clusterService(nodeName).localNode().getId(); - - logger.debug("--> getting monitoring document for node id [{}]", nodeId); - assertThat(client().prepareGet(dataIndex, DiscoveryNodeResolver.TYPE, nodeId).setPreference("_primary").get().isExists(), - is(true)); - - // checks that document is not indexed - assertHitCount(client().prepareSearch(dataIndex).setSize(0) - .setTypes(DiscoveryNodeResolver.TYPE) - .setPreference("_primary") - .setQuery(QueryBuilders.boolQuery() - .should(matchQuery("node.id", nodeId)) - .should(matchQuery("node.name", nodeName))).get(), 0); - } - - logger.debug("--> cluster state nodes successfully collected"); - } } diff --git a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/DiscoveryNodeResolverTests.java b/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/DiscoveryNodeResolverTests.java deleted file mode 100644 index f497d757217..00000000000 --- a/plugin/src/test/java/org/elasticsearch/xpack/monitoring/resolver/node/DiscoveryNodeResolverTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.monitoring.resolver.node; - -import org.elasticsearch.Version; -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.VersionUtils; -import org.elasticsearch.xpack.monitoring.collector.cluster.DiscoveryNodeMonitoringDoc; -import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils; -import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase; -import org.elasticsearch.xpack.monitoring.resolver.cluster.DiscoveryNodeResolver; - -import java.util.UUID; - -import static java.util.Collections.emptyMap; -import static java.util.Collections.emptySet; -import static org.elasticsearch.test.VersionUtils.randomVersionBetween; -import static org.hamcrest.Matchers.equalTo; - -public class DiscoveryNodeResolverTests extends MonitoringIndexNameResolverTestCase { - - @Override - protected DiscoveryNodeMonitoringDoc newMonitoringDoc() { - DiscoveryNodeMonitoringDoc doc = new DiscoveryNodeMonitoringDoc(randomMonitoringId(), - randomAlphaOfLength(2), randomAlphaOfLength(5), 1437580442979L, - new DiscoveryNode(randomAlphaOfLength(3), UUID.randomUUID().toString(), - buildNewFakeTransportAddress(), emptyMap(), emptySet(), - randomVersionBetween(random(), VersionUtils.getFirstVersion(), Version.CURRENT))); - return doc; - } - - @Override - protected boolean checkFilters() { - return false; - } - - public void testDiscoveryNodeResolver() throws Exception { - DiscoveryNodeMonitoringDoc doc = newMonitoringDoc(); - - DiscoveryNodeResolver resolver = newResolver(); - assertThat(resolver.index(doc), equalTo(".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION)); - - assertSource(resolver.source(doc, XContentType.JSON), - Sets.newHashSet( - "cluster_uuid", - "timestamp", - "source_node", - "node.id", - "node.name", - "node.transport_address", - "node.attributes"), XContentType.JSON); - } -}