[Monitoring] Add "cluster_state.nodes_hash" to document (elastic/x-pack-elasticsearch#2798)

Adding this field enables a very simple mechanism for detecting node changes in the cluster state via Watcher (and other mechanisms). The next step is to add the cluster alert that uses it.

Original commit: elastic/x-pack-elasticsearch@1eacc25cff

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java

@@ -8,6 +8,8 @@ package org.elasticsearch.xpack.monitoring.collector.cluster;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.node.DiscoveryNode;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.hash.MessageDigests;
@@ -128,6 +130,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
 
         if (clusterState != null) {
             builder.startObject("cluster_state");
+            builder.field("nodes_hash", nodesHash(clusterState.nodes()));
             builder.field("status", status.name().toLowerCase(Locale.ROOT));
             clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
             builder.endObject();
@@ -148,6 +151,23 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
         }
     }
 
+    /**
+     * Create a simple hash value that can be used to determine if the nodes listing has changed since the last report.
+     *
+     * @param nodes All nodes in the cluster state.
+     * @return A hash code value whose value can be used to determine if the node listing has changed (including node restarts).
+     */
+    public static int nodesHash(final DiscoveryNodes nodes) {
+        final StringBuilder temp = new StringBuilder();
+
+        // adds the Ephemeral ID (as opposed to the Persistent UUID) to catch node restarts, which is critical for 1 node clusters
+        for (final DiscoveryNode node : nodes) {
+            temp.append(node.getEphemeralId());
+        }
+
+        return temp.toString().hashCode();
+    }
+
     public static String hash(License license, String clusterName) {
         return hash(license.status().label(), license.uid(), license.type(), String.valueOf(license.expiryDate()), clusterName);
     }

plugin/src/main/resources/monitoring-es.json

@@ -411,6 +411,9 @@
             "version": {
               "type": "long"
             },
+            "nodes_hash": {
+              "type": "integer"
+            },
             "master_node": {
               "type": "keyword"
             },

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java

@@ -5,6 +5,7 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.cluster;
 
+import java.util.Map;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
@@ -55,6 +56,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.List;
 
 import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
 import static java.util.Collections.singleton;
 import static java.util.Collections.singletonList;
 import static java.util.Collections.singletonMap;
@@ -94,6 +96,15 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                          .expiryDate(timestamp + randomIntBetween(1, 10) * 1_000L)
                          .maxNodes(randomIntBetween(1, 5))
                          .build();
+
+        final DiscoveryNode masterNode = masterNode();
+        final DiscoveryNodes.Builder builder =
+                DiscoveryNodes.builder()
+                              .masterNodeId(masterNode.getId())
+                              .localNodeId(masterNode.getId())
+                              .add(masterNode);
+
+        when(clusterState.nodes()).thenReturn(builder.build());
     }
 
     @Override
@@ -136,6 +147,38 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                         clusterName, version, null, license, usages, clusterStats, clusterState));
     }
 
+    public void testNodesHash() {
+        final int nodeCount = randomIntBetween(0, 5);
+        final Map<String, String> emptyMap = emptyMap();
+        final DiscoveryNode masterNode = masterNode();
+        final DiscoveryNodes.Builder builder =
+                DiscoveryNodes.builder()
+                              .masterNodeId(masterNode.getId())
+                              .localNodeId(masterNode.getId());
+
+        for (int i = 0; i < nodeCount; ++i) {
+            builder.add(
+                new DiscoveryNode(randomAlphaOfLength(5),
+                                  randomAlphaOfLength(2 + i),
+                                  randomAlphaOfLength(5),
+                                  randomAlphaOfLength(5),
+                                  randomAlphaOfLength(5),
+                                  new TransportAddress(TransportAddress.META_ADDRESS, 9301 + i),
+                                  randomBoolean() ? singletonMap("attr", randomAlphaOfLength(3)) : emptyMap,
+                                  singleton(randomFrom(DiscoveryNode.Role.values())),
+                                  Version.CURRENT));
+        }
+
+        final DiscoveryNodes nodes = builder.build();
+        String ephemeralIds = "";
+
+        for (final DiscoveryNode node : nodes) {
+            ephemeralIds += node.getEphemeralId();
+        }
+
+        assertThat(ClusterStatsMonitoringDoc.nodesHash(nodes), equalTo(ephemeralIds.hashCode()));
+    }
+
     public void testHash() {
         assertEquals("cc6628dbcfd052fba57870dc4eed4bc9f5cd5d43b4df46e97867aeb2dd7bd2e8",
                 ClusterStatsMonitoringDoc.hash("licenseStatus", "licenseUid", "licenseType", "licenseExpiryDate", "clusterUUID"));
@@ -475,6 +518,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                     + "}"
                   + "},"
                   + "\"cluster_state\":{"
+                    + "\"nodes_hash\":1314980060,"
                     + "\"status\":\"green\","
                     + "\"version\":12,"
                     + "\"state_uuid\":\"_state_uuid\","
@@ -500,4 +544,17 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                   + "}"
                 + "}" , xContent.utf8ToString());
     }
+
+    private DiscoveryNode masterNode() {
+        return new DiscoveryNode("_node_name",
+                                 "_node_id",
+                                 "_ephemeral_id",
+                                 "_host_name",
+                                 "_host_address",
+                                 new TransportAddress(TransportAddress.META_ADDRESS, 9300),
+                                 singletonMap("attr", "value"),
+                                 singleton(DiscoveryNode.Role.MASTER),
+                                 Version.CURRENT);
+    }
+
 }

plugin/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -343,7 +343,8 @@ public class MonitoringIT extends ESRestTestCase {
 
         final Map<String, Object> clusterState = (Map<String, Object>) source.get("cluster_state");
         assertThat(clusterState, notNullValue());
-        assertThat(clusterState.size(), equalTo(5));
+        assertThat(clusterState.size(), equalTo(6));
+        assertThat(clusterState.remove("nodes_hash"), notNullValue());
         assertThat(clusterState.remove("status"), notNullValue());
         assertThat(clusterState.remove("version"), notNullValue());
         assertThat(clusterState.remove("state_uuid"), notNullValue());