[Monitoring] Add "apm" to "stack_stats" for Phone Home (elastic/x-pack-elasticsearch#2848)

This checks if `apm-*` indices exist in the cluster to try to determine if APM is in use on the Elasticsearch cluster.

Original commit: elastic/x-pack-elasticsearch@7f9a9a4eee

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java

@@ -10,13 +10,17 @@ import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.license.LicenseUtils;
@@ -29,6 +33,7 @@ import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 import static org.elasticsearch.xpack.XPackSettings.SECURITY_ENABLED;
 import static org.elasticsearch.xpack.XPackSettings.TRANSPORT_SSL_ENABLED;
@@ -50,6 +55,7 @@ public class ClusterStatsCollector extends Collector {
      */
     public static final Setting<TimeValue> CLUSTER_STATS_TIMEOUT = collectionTimeoutSetting("cluster.stats.timeout");
 
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
     private final LicenseService licenseService;
     private final Client client;
 
@@ -58,9 +64,20 @@ public class ClusterStatsCollector extends Collector {
                                  final XPackLicenseState licenseState,
                                  final Client client,
                                  final LicenseService licenseService) {
+        this(settings, clusterService, licenseState, client, licenseService, new IndexNameExpressionResolver(Settings.EMPTY));
+    }
+
+    ClusterStatsCollector(final Settings settings,
+                          final ClusterService clusterService,
+                          final XPackLicenseState licenseState,
+                          final Client client,
+                          final LicenseService licenseService,
+                          final IndexNameExpressionResolver indexNameExpressionResolver) {
         super(settings, ClusterStatsMonitoringDoc.TYPE, clusterService, CLUSTER_STATS_TIMEOUT, licenseState);
+
         this.client = client;
         this.licenseService = licenseService;
+        this.indexNameExpressionResolver = Objects.requireNonNull(indexNameExpressionResolver);
     }
 
     @Override
@@ -82,7 +99,8 @@ public class ClusterStatsCollector extends Collector {
         final String version = Version.CURRENT.toString();
         final ClusterState clusterState = clusterService.state();
         final License license = licenseService.getLicense();
-        final List<XPackFeatureSet.Usage> usage = collect(usageSupplier);
+        final List<XPackFeatureSet.Usage> xpackUsage = collect(usageSupplier);
+        final boolean apmIndicesExist = doAPMIndicesExist(clusterState);
         // if they have any other type of license, then they are either okay or already know
         final boolean clusterNeedsTLSEnabled = license.operationMode() == License.OperationMode.TRIAL &&
                                                SECURITY_ENABLED.get(settings) &&
@@ -91,7 +109,18 @@ public class ClusterStatsCollector extends Collector {
         // Adds a cluster stats document
         return Collections.singleton(
                 new ClusterStatsMonitoringDoc(clusterUUID(), timestamp(), interval, node, clusterName, version,  clusterStats.getStatus(),
-                        license, usage, clusterStats, clusterState, clusterNeedsTLSEnabled));
+                                              license, apmIndicesExist, xpackUsage, clusterStats, clusterState, clusterNeedsTLSEnabled));
+    }
+
+    boolean doAPMIndicesExist(final ClusterState clusterState) {
+        try {
+            final Index[] indices =
+                indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*");
+
+            return indices.length > 0;
+        } catch (IndexNotFoundException | IllegalArgumentException e) {
+            return false;
+        }
     }
 
     @Nullable

plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java

@@ -51,6 +51,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
     private final String clusterName;
     private final String version;
     private final License license;
+    private final boolean apmIndicesExist;
     private final List<XPackFeatureSet.Usage> usages;
     private final ClusterStatsResponse clusterStats;
     private final ClusterState clusterState;
@@ -65,6 +66,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
                               final String version,
                               final ClusterHealthStatus status,
                               @Nullable final License license,
+                              final boolean apmIndicesExist,
                               @Nullable final List<XPackFeatureSet.Usage> usages,
                               @Nullable final ClusterStatsResponse clusterStats,
                               @Nullable final ClusterState clusterState,
@@ -75,6 +77,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
         this.version = Objects.requireNonNull(version);
         this.status = Objects.requireNonNull(status);
         this.license = license;
+        this.apmIndicesExist = apmIndicesExist;
         this.usages = usages;
         this.clusterStats = clusterStats;
         this.clusterState = clusterState;
@@ -93,6 +96,10 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
         return license;
     }
 
+    boolean getAPMIndicesExist() {
+        return apmIndicesExist;
+    }
+
     List<XPackFeatureSet.Usage> getUsages() {
         return usages;
     }
@@ -120,6 +127,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
 
         if (license != null) {
             builder.startObject("license");
+            {
                 Map<String, String> extraParams = new MapBuilder<String, String>()
                         .put(License.REST_VIEW_MODE, "true")
                         .map();
@@ -129,36 +137,47 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
                 if (clusterNeedsTLSEnabled) {
                     builder.field("cluster_needs_tls", true);
                 }
+            }
             builder.endObject();
         }
 
         if (clusterStats != null) {
             builder.startObject("cluster_stats");
+            {
                 clusterStats.toXContent(builder, params);
+            }
             builder.endObject();
         }
 
         if (clusterState != null) {
             builder.startObject("cluster_state");
+            {
                 builder.field("nodes_hash", nodesHash(clusterState.nodes()));
                 builder.field("status", status.name().toLowerCase(Locale.ROOT));
                 clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
+            }
             builder.endObject();
         }
 
-        if (usages != null) {
-            // in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
-            // it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
         builder.startObject("stack_stats");
         {
+            // in the future, it may be useful to pass in an object that represents APM (and others), but for now this
+            // is good enough
+            builder.startObject("apm");
+            {
+                builder.field("found", apmIndicesExist);
+            }
+            builder.endObject();
+
+            if (usages != null) {
                 builder.startObject("xpack");
                 for (final XPackFeatureSet.Usage usage : usages) {
                     builder.field(usage.name(), usage);
                 }
                 builder.endObject();
             }
-            builder.endObject();
         }
+        builder.endObject();
     }
 
     /**

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java

@@ -11,12 +11,16 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsIndices;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder;
 import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.AdminClient;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.ClusterAdminClient;
 import org.elasticsearch.cluster.health.ClusterHealthStatus;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.license.License;
 import org.elasticsearch.license.LicenseService;
 import org.elasticsearch.xpack.action.XPackUsageAction;
@@ -82,6 +86,40 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
         verify(nodes).isLocalNodeElectedMaster();
     }
 
+    public void testDoAPMIndicesExistReturnsBasedOnIndices() {
+        final boolean apmIndicesExist = randomBoolean();
+        final Index[] indices = new Index[apmIndicesExist ? randomIntBetween(1, 3) : 0];
+        final IndexNameExpressionResolver resolver = mock(IndexNameExpressionResolver.class);
+        when(resolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenReturn(indices);
+
+        final ClusterStatsCollector collector =
+                new ClusterStatsCollector(Settings.EMPTY, clusterService, licenseState, client, licenseService, resolver);
+
+        assertThat(collector.doAPMIndicesExist(clusterState), is(apmIndicesExist));
+    }
+
+    public void testDoAPMIndicesExistReturnsFalseForExpectedExceptions() {
+        final Exception exception = randomFrom(new IndexNotFoundException("TEST - expected"), new IllegalArgumentException());
+        final IndexNameExpressionResolver resolver = mock(IndexNameExpressionResolver.class);
+        when(resolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenThrow(exception);
+
+        final ClusterStatsCollector collector =
+                new ClusterStatsCollector(Settings.EMPTY, clusterService, licenseState, client, licenseService, resolver);
+
+        assertThat(collector.doAPMIndicesExist(clusterState), is(false));
+    }
+
+    public void testDoAPMIndicesExistRethrowsUnexpectedExceptions() {
+        final RuntimeException exception = new RuntimeException();
+        final IndexNameExpressionResolver resolver = mock(IndexNameExpressionResolver.class);
+        when(resolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenThrow(exception);
+
+        final ClusterStatsCollector collector =
+                new ClusterStatsCollector(Settings.EMPTY, clusterService, licenseState, client, licenseService, resolver);
+
+        expectThrows(RuntimeException.class, () -> collector.doAPMIndicesExist(clusterState));
+    }
+
     public void testDoCollect() throws Exception {
         final Settings.Builder settings = Settings.builder();
         final License.OperationMode mode =
@@ -161,6 +199,11 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
         final Client client = mock(Client.class);
         when(client.admin()).thenReturn(adminClient);
 
+        final IndexNameExpressionResolver indexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
+        final boolean apmIndicesExist = randomBoolean();
+        final Index[] indices = new Index[apmIndicesExist ? randomIntBetween(1, 5) : 0];
+        when(indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenReturn(indices);
+
         final XPackUsageResponse xPackUsageResponse = new XPackUsageResponse(singletonList(new LogstashFeatureSet.Usage(true, true)));
 
         @SuppressWarnings("unchecked")
@@ -169,7 +212,9 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
         when(xPackUsageFuture.actionGet()).thenReturn(xPackUsageResponse);
 
         final ClusterStatsCollector collector =
-                new ClusterStatsCollector(settings.build(), clusterService, licenseState, client, licenseService);
+                new ClusterStatsCollector(settings.build(), clusterService, licenseState, client, licenseService,
+                                          indexNameExpressionResolver);
+
         assertEquals(timeout, collector.getCollectionTimeout());
 
         final long interval = randomNonNegativeLong();
@@ -201,6 +246,7 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
         assertThat(document.getClusterStats().getStatus(), equalTo(clusterStatus));
         assertThat(document.getClusterStats().getIndicesStats().getIndexCount(), equalTo(nbIndices));
 
+        assertThat(document.getAPMIndicesExist(), is(apmIndicesExist));
         assertThat(document.getUsages(), hasSize(1));
         assertThat(document.getUsages().iterator().next().name(), equalTo(Logstash.NAME));
 

plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.monitoring.collector.cluster;
 
-import java.util.Map;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
@@ -54,6 +53,7 @@ import org.junit.Before;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.Map;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
@@ -77,6 +77,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
     private ClusterState clusterState;
     private License license;
     private final boolean needToEnableTLS = randomBoolean();
+    private final boolean apmIndicesExist = randomBoolean();
 
     @Override
     @Before
@@ -112,7 +113,8 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
     protected ClusterStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, long interval, MonitoringDoc.Node node,
                                                             MonitoredSystem system, String type, String id) {
         return new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
-                                             clusterName, version, clusterStatus, license, usages, clusterStats, clusterState,
+                                             clusterName, version, clusterStatus, license,
+                                             apmIndicesExist, usages, clusterStats, clusterState,
                                              needToEnableTLS);
     }
 
@@ -126,6 +128,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
         assertThat(document.getVersion(), equalTo(version));
         assertThat(document.getStatus(), equalTo(clusterStatus));
         assertThat(document.getLicense(), equalTo(license));
+        assertThat(document.getAPMIndicesExist(), is(apmIndicesExist));
         assertThat(document.getUsages(), is(usages));
         assertThat(document.getClusterStats(), is(clusterStats));
         assertThat(document.getClusterState(), is(clusterState));
@@ -134,21 +137,21 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
     public void testConstructorClusterNameMustNotBeNull() {
         expectThrows(NullPointerException.class,
                 () -> new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
-                        null, version, clusterStatus, license, usages, clusterStats, clusterState,
+                        null, version, clusterStatus, license, apmIndicesExist, usages, clusterStats, clusterState,
                         needToEnableTLS));
     }
 
     public void testConstructorVersionMustNotBeNull() {
         expectThrows(NullPointerException.class,
                 () -> new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
-                        clusterName, null, clusterStatus, license, usages, clusterStats, clusterState,
+                        clusterName, null, clusterStatus, license, apmIndicesExist, usages, clusterStats, clusterState,
                         needToEnableTLS));
     }
 
     public void testConstructorClusterHealthStatusMustNotBeNull() {
         expectThrows(NullPointerException.class,
                 () -> new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
-                        clusterName, version, null, license, usages, clusterStats, clusterState,
+                        clusterName, version, null, license, apmIndicesExist, usages, clusterStats, clusterState,
                         needToEnableTLS));
     }
 
@@ -175,13 +178,13 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
         }
 
         final DiscoveryNodes nodes = builder.build();
-        String ephemeralIds = "";
+        StringBuilder ephemeralIds = new StringBuilder();
 
         for (final DiscoveryNode node : nodes) {
-            ephemeralIds += node.getEphemeralId();
+            ephemeralIds.append(node.getEphemeralId());
         }
 
-        assertThat(ClusterStatsMonitoringDoc.nodesHash(nodes), equalTo(ephemeralIds.hashCode()));
+        assertThat(ClusterStatsMonitoringDoc.nodesHash(nodes), equalTo(ephemeralIds.toString().hashCode()));
     }
 
     public void testHash() {
@@ -250,7 +253,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                                         .maxNodes(2)
                                         .build();
 
-        List<XPackFeatureSet.Usage> usages = singletonList(new LogstashFeatureSet.Usage(false, true));
+        final List<XPackFeatureSet.Usage> usages = singletonList(new LogstashFeatureSet.Usage(false, true));
 
         final NodeInfo mockNodeInfo = mock(NodeInfo.class);
         when(mockNodeInfo.getVersion()).thenReturn(Version.V_6_0_0_alpha2);
@@ -342,6 +345,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                                                                             "_version",
                                                                             ClusterHealthStatus.GREEN,
                                                                             license,
+                                                                            apmIndicesExist,
                                                                             usages,
                                                                             clusterStats,
                                                                             clusterState,
@@ -542,6 +546,9 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
                     + "}"
                   + "},"
                   + "\"stack_stats\":{"
+                    + "\"apm\":{"
+                      + "\"found\":" + apmIndicesExist
+                    + "},"
                     + "\"xpack\":{"
                       + "\"logstash\":{"
                         + "\"available\":false,"

plugin/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java

@@ -190,8 +190,10 @@ public class MonitoringIT extends ESRestTestCase {
      */
     @SuppressWarnings("unchecked")
     public void testMonitoringService() throws Exception {
+        final boolean createAPMIndex = randomBoolean();
+        final String indexName = createAPMIndex ? "apm-2017.11.06" : "books";
         final HttpEntity document = new StringEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON);
-        assertThat(client().performRequest("POST", "/books/book/0", singletonMap("refresh", "true"), document)
+        assertThat(client().performRequest("POST", "/" + indexName + "/doc/0", singletonMap("refresh", "true"), document)
                            .getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_CREATED));
 
         whenExportersAreReady(() -> {
@@ -209,7 +211,7 @@ public class MonitoringIT extends ESRestTestCase {
                 final String type = (String) extractValue("_source.type", searchHit);
 
                 if (ClusterStatsMonitoringDoc.TYPE.equals(type)) {
-                    assertClusterStatsMonitoringDoc(searchHit, collectionInterval);
+                    assertClusterStatsMonitoringDoc(searchHit, collectionInterval, createAPMIndex);
                 } else if (IndexRecoveryMonitoringDoc.TYPE.equals(type)) {
                     assertIndexRecoveryMonitoringDoc(searchHit, collectionInterval);
                 } else if (IndicesStatsMonitoringDoc.TYPE.equals(type)) {
@@ -294,7 +296,9 @@ public class MonitoringIT extends ESRestTestCase {
      * Assert that a {@link ClusterStatsMonitoringDoc} contains the expected information
      */
     @SuppressWarnings("unchecked")
-    private static void assertClusterStatsMonitoringDoc(final Map<String, Object> document, final TimeValue interval) throws Exception {
+    private static void assertClusterStatsMonitoringDoc(final Map<String, Object> document,
+                                                        final TimeValue interval,
+                                                        final boolean apmIndicesExist) throws Exception {
         assertMonitoringDoc(document, MonitoredSystem.ES, ClusterStatsMonitoringDoc.TYPE, interval);
 
         final Map<String, Object> source = (Map<String, Object>) document.get("_source");
@@ -335,7 +339,13 @@ public class MonitoringIT extends ESRestTestCase {
 
         final Map<String, Object> stackStats = (Map<String, Object>) source.get("stack_stats");
         assertThat(stackStats, notNullValue());
-        assertThat(stackStats.size(), equalTo(1));
+        assertThat(stackStats.size(), equalTo(2));
+
+        final Map<String, Object> apm = (Map<String, Object>) stackStats.get("apm");
+        assertThat(apm, notNullValue());
+        assertThat(apm.size(), equalTo(1));
+        assertThat(apm.remove("found"), is(apmIndicesExist));
+        assertThat(apm.isEmpty(), is(true));
 
         final Map<String, Object> xpackStats = (Map<String, Object>) stackStats.get("xpack");
         assertThat(xpackStats, notNullValue());