honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Monitoring] Add "apm" to "stack_stats" for Phone Home (elastic/x-pack-elasticsearch#2848) 

This checks if `apm-*` indices exist in the cluster to try to determine if APM is in use on the Elasticsearch cluster.

Original commit: elastic/x-pack-elasticsearch@7f9a9a4eee
This commit is contained in:
Chris Earle 2017-11-13 14:24:07 -05:00 committed by GitHub
parent 00ccac9203
commit def8c48dcb
5 changed files with 144 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
View File

@ -10,13 +10,17 @@ import org.apache.logging.log4j.util.Supplier;
import org.elasticsearch.ElasticsearchSecurityException;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.license.License;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.LicenseUtils;
@ -29,6 +33,7 @@ import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import static org.elasticsearch.xpack.XPackSettings.SECURITY_ENABLED;
import static org.elasticsearch.xpack.XPackSettings.TRANSPORT_SSL_ENABLED;
@ -50,6 +55,7 @@ public class ClusterStatsCollector extends Collector {
*/
public static final Setting<TimeValue> CLUSTER_STATS_TIMEOUT = collectionTimeoutSetting("cluster.stats.timeout");
private final IndexNameExpressionResolver indexNameExpressionResolver;
private final LicenseService licenseService;
private final Client client;
@ -58,9 +64,20 @@ public class ClusterStatsCollector extends Collector {
final XPackLicenseState licenseState,
final Client client,
final LicenseService licenseService) {
this(settings, clusterService, licenseState, client, licenseService, new IndexNameExpressionResolver(Settings.EMPTY));
}
ClusterStatsCollector(final Settings settings,
final ClusterService clusterService,
final XPackLicenseState licenseState,
final Client client,
final LicenseService licenseService,
final IndexNameExpressionResolver indexNameExpressionResolver) {
super(settings, ClusterStatsMonitoringDoc.TYPE, clusterService, CLUSTER_STATS_TIMEOUT, licenseState);
this.client = client;
this.licenseService = licenseService;
this.indexNameExpressionResolver = Objects.requireNonNull(indexNameExpressionResolver);
}
@Override
@ -82,7 +99,8 @@ public class ClusterStatsCollector extends Collector {
final String version = Version.CURRENT.toString();
final ClusterState clusterState = clusterService.state();
final License license = licenseService.getLicense();
final List<XPackFeatureSet.Usage> usage = collect(usageSupplier);
final List<XPackFeatureSet.Usage> xpackUsage = collect(usageSupplier);
final boolean apmIndicesExist = doAPMIndicesExist(clusterState);
// if they have any other type of license, then they are either okay or already know
final boolean clusterNeedsTLSEnabled = license.operationMode() == License.OperationMode.TRIAL &&
SECURITY_ENABLED.get(settings) &&
@ -91,7 +109,18 @@ public class ClusterStatsCollector extends Collector {
// Adds a cluster stats document
return Collections.singleton(
new ClusterStatsMonitoringDoc(clusterUUID(), timestamp(), interval, node, clusterName, version, clusterStats.getStatus(),
license, usage, clusterStats, clusterState, clusterNeedsTLSEnabled));
license, apmIndicesExist, xpackUsage, clusterStats, clusterState, clusterNeedsTLSEnabled));
}
boolean doAPMIndicesExist(final ClusterState clusterState) {
try {
final Index[] indices =
indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*");
return indices.length > 0;
} catch (IndexNotFoundException | IllegalArgumentException e) {
return false;
}
}
@Nullable

53
plugin/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDoc.java
View File

@ -51,6 +51,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
private final String clusterName;
private final String version;
private final License license;
private final boolean apmIndicesExist;
private final List<XPackFeatureSet.Usage> usages;
private final ClusterStatsResponse clusterStats;
private final ClusterState clusterState;
@ -65,6 +66,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
final String version,
final ClusterHealthStatus status,
@Nullable final License license,
final boolean apmIndicesExist,
@Nullable final List<XPackFeatureSet.Usage> usages,
@Nullable final ClusterStatsResponse clusterStats,
@Nullable final ClusterState clusterState,
@ -75,6 +77,7 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
this.version = Objects.requireNonNull(version);
this.status = Objects.requireNonNull(status);
this.license = license;
this.apmIndicesExist = apmIndicesExist;
this.usages = usages;
this.clusterStats = clusterStats;
this.clusterState = clusterState;
@ -93,6 +96,10 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
return license;
}
boolean getAPMIndicesExist() {
return apmIndicesExist;
}
List<XPackFeatureSet.Usage> getUsages() {
return usages;
}
@ -120,45 +127,57 @@ public class ClusterStatsMonitoringDoc extends MonitoringDoc {
if (license != null) {
builder.startObject("license");
Map<String, String> extraParams = new MapBuilder<String, String>()
.put(License.REST_VIEW_MODE, "true")
.map();
params = new ToXContent.DelegatingMapParams(extraParams, params);
license.toInnerXContent(builder, params);
builder.field("hkey", hash(license, getCluster()));
if (clusterNeedsTLSEnabled) {
builder.field("cluster_needs_tls", true);
{
Map<String, String> extraParams = new MapBuilder<String, String>()
.put(License.REST_VIEW_MODE, "true")
.map();
params = new ToXContent.DelegatingMapParams(extraParams, params);
license.toInnerXContent(builder, params);
builder.field("hkey", hash(license, getCluster()));
if (clusterNeedsTLSEnabled) {
builder.field("cluster_needs_tls", true);
}
}
builder.endObject();
}
if (clusterStats != null) {
builder.startObject("cluster_stats");
clusterStats.toXContent(builder, params);
{
clusterStats.toXContent(builder, params);
}
builder.endObject();
}
if (clusterState != null) {
builder.startObject("cluster_state");
builder.field("nodes_hash", nodesHash(clusterState.nodes()));
builder.field("status", status.name().toLowerCase(Locale.ROOT));
clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
{
builder.field("nodes_hash", nodesHash(clusterState.nodes()));
builder.field("status", status.name().toLowerCase(Locale.ROOT));
clusterState.toXContent(builder, CLUSTER_STATS_PARAMS);
}
builder.endObject();
}
if (usages != null) {
// in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
// it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
builder.startObject("stack_stats");
builder.startObject("stack_stats");
{
// in the future, it may be useful to pass in an object that represents APM (and others), but for now this
// is good enough
builder.startObject("apm");
{
builder.field("found", apmIndicesExist);
}
builder.endObject();
if (usages != null) {
builder.startObject("xpack");
for (final XPackFeatureSet.Usage usage : usages) {
builder.field(usage.name(), usage);
}
builder.endObject();
}
builder.endObject();
}
builder.endObject();
}
/**

48
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollectorTests.java
View File

@ -11,12 +11,16 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsIndices;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsNodes;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequestBuilder;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.ClusterAdminClient;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.license.License;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.xpack.action.XPackUsageAction;
@ -82,6 +86,40 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
verify(nodes).isLocalNodeElectedMaster();
}
public void testDoAPMIndicesExistReturnsBasedOnIndices() {
final boolean apmIndicesExist = randomBoolean();
final Index[] indices = new Index[apmIndicesExist ? randomIntBetween(1, 3) : 0];
final IndexNameExpressionResolver resolver = mock(IndexNameExpressionResolver.class);
when(resolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenReturn(indices);
final ClusterStatsCollector collector =
new ClusterStatsCollector(Settings.EMPTY, clusterService, licenseState, client, licenseService, resolver);
assertThat(collector.doAPMIndicesExist(clusterState), is(apmIndicesExist));
}
public void testDoAPMIndicesExistReturnsFalseForExpectedExceptions() {
final Exception exception = randomFrom(new IndexNotFoundException("TEST - expected"), new IllegalArgumentException());
final IndexNameExpressionResolver resolver = mock(IndexNameExpressionResolver.class);
when(resolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenThrow(exception);
final ClusterStatsCollector collector =
new ClusterStatsCollector(Settings.EMPTY, clusterService, licenseState, client, licenseService, resolver);
assertThat(collector.doAPMIndicesExist(clusterState), is(false));
}
public void testDoAPMIndicesExistRethrowsUnexpectedExceptions() {
final RuntimeException exception = new RuntimeException();
final IndexNameExpressionResolver resolver = mock(IndexNameExpressionResolver.class);
when(resolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenThrow(exception);
final ClusterStatsCollector collector =
new ClusterStatsCollector(Settings.EMPTY, clusterService, licenseState, client, licenseService, resolver);
expectThrows(RuntimeException.class, () -> collector.doAPMIndicesExist(clusterState));
}
public void testDoCollect() throws Exception {
final Settings.Builder settings = Settings.builder();
final License.OperationMode mode =
@ -161,6 +199,11 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
final Client client = mock(Client.class);
when(client.admin()).thenReturn(adminClient);
final IndexNameExpressionResolver indexNameExpressionResolver = mock(IndexNameExpressionResolver.class);
final boolean apmIndicesExist = randomBoolean();
final Index[] indices = new Index[apmIndicesExist ? randomIntBetween(1, 5) : 0];
when(indexNameExpressionResolver.concreteIndices(clusterState, IndicesOptions.lenientExpandOpen(), "apm-*")).thenReturn(indices);
final XPackUsageResponse xPackUsageResponse = new XPackUsageResponse(singletonList(new LogstashFeatureSet.Usage(true, true)));
@SuppressWarnings("unchecked")
@ -169,7 +212,9 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
when(xPackUsageFuture.actionGet()).thenReturn(xPackUsageResponse);
final ClusterStatsCollector collector =
new ClusterStatsCollector(settings.build(), clusterService, licenseState, client, licenseService);
new ClusterStatsCollector(settings.build(), clusterService, licenseState, client, licenseService,
indexNameExpressionResolver);
assertEquals(timeout, collector.getCollectionTimeout());
final long interval = randomNonNegativeLong();
@ -201,6 +246,7 @@ public class ClusterStatsCollectorTests extends BaseCollectorTestCase {
assertThat(document.getClusterStats().getStatus(), equalTo(clusterStatus));
assertThat(document.getClusterStats().getIndicesStats().getIndexCount(), equalTo(nbIndices));
assertThat(document.getAPMIndicesExist(), is(apmIndicesExist));
assertThat(document.getUsages(), hasSize(1));
assertThat(document.getUsages().iterator().next().name(), equalTo(Logstash.NAME));

25
plugin/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java
View File

@ -5,7 +5,6 @@
*/
package org.elasticsearch.xpack.monitoring.collector.cluster;
import java.util.Map;
import org.elasticsearch.Version;
import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
@ -54,6 +53,7 @@ import org.junit.Before;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
@ -77,6 +77,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
private ClusterState clusterState;
private License license;
private final boolean needToEnableTLS = randomBoolean();
private final boolean apmIndicesExist = randomBoolean();
@Override
@Before
@ -112,7 +113,8 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
protected ClusterStatsMonitoringDoc createMonitoringDoc(String cluster, long timestamp, long interval, MonitoringDoc.Node node,
MonitoredSystem system, String type, String id) {
return new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
clusterName, version, clusterStatus, license, usages, clusterStats, clusterState,
clusterName, version, clusterStatus, license,
apmIndicesExist, usages, clusterStats, clusterState,
needToEnableTLS);
}
@ -126,6 +128,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
assertThat(document.getVersion(), equalTo(version));
assertThat(document.getStatus(), equalTo(clusterStatus));
assertThat(document.getLicense(), equalTo(license));
assertThat(document.getAPMIndicesExist(), is(apmIndicesExist));
assertThat(document.getUsages(), is(usages));
assertThat(document.getClusterStats(), is(clusterStats));
assertThat(document.getClusterState(), is(clusterState));
@ -134,21 +137,21 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
public void testConstructorClusterNameMustNotBeNull() {
expectThrows(NullPointerException.class,
() -> new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
null, version, clusterStatus, license, usages, clusterStats, clusterState,
null, version, clusterStatus, license, apmIndicesExist, usages, clusterStats, clusterState,
needToEnableTLS));
}
public void testConstructorVersionMustNotBeNull() {
expectThrows(NullPointerException.class,
() -> new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
clusterName, null, clusterStatus, license, usages, clusterStats, clusterState,
clusterName, null, clusterStatus, license, apmIndicesExist, usages, clusterStats, clusterState,
needToEnableTLS));
}
public void testConstructorClusterHealthStatusMustNotBeNull() {
expectThrows(NullPointerException.class,
() -> new ClusterStatsMonitoringDoc(cluster, timestamp, interval, node,
clusterName, version, null, license, usages, clusterStats, clusterState,
clusterName, version, null, license, apmIndicesExist, usages, clusterStats, clusterState,
needToEnableTLS));
}
@ -175,13 +178,13 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
}
final DiscoveryNodes nodes = builder.build();
String ephemeralIds = "";
StringBuilder ephemeralIds = new StringBuilder();
for (final DiscoveryNode node : nodes) {
ephemeralIds += node.getEphemeralId();
ephemeralIds.append(node.getEphemeralId());
}
assertThat(ClusterStatsMonitoringDoc.nodesHash(nodes), equalTo(ephemeralIds.hashCode()));
assertThat(ClusterStatsMonitoringDoc.nodesHash(nodes), equalTo(ephemeralIds.toString().hashCode()));
}
public void testHash() {
@ -250,7 +253,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
.maxNodes(2)
.build();
List<XPackFeatureSet.Usage> usages = singletonList(new LogstashFeatureSet.Usage(false, true));
final List<XPackFeatureSet.Usage> usages = singletonList(new LogstashFeatureSet.Usage(false, true));
final NodeInfo mockNodeInfo = mock(NodeInfo.class);
when(mockNodeInfo.getVersion()).thenReturn(Version.V_6_0_0_alpha2);
@ -342,6 +345,7 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
"_version",
ClusterHealthStatus.GREEN,
license,
apmIndicesExist,
usages,
clusterStats,
clusterState,
@ -542,6 +546,9 @@ public class ClusterStatsMonitoringDocTests extends BaseMonitoringDocTestCase<Cl
+ "}"
+ "},"
+ "\"stack_stats\":{"
+ "\"apm\":{"
+ "\"found\":" + apmIndicesExist
+ "},"
+ "\"xpack\":{"
+ "\"logstash\":{"
+ "\"available\":false,"

18
plugin/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java
View File

@ -190,8 +190,10 @@ public class MonitoringIT extends ESRestTestCase {
*/
@SuppressWarnings("unchecked")
public void testMonitoringService() throws Exception {
final boolean createAPMIndex = randomBoolean();
final String indexName = createAPMIndex ? "apm-2017.11.06" : "books";
final HttpEntity document = new StringEntity("{\"field\":\"value\"}", ContentType.APPLICATION_JSON);
assertThat(client().performRequest("POST", "/books/book/0", singletonMap("refresh", "true"), document)
assertThat(client().performRequest("POST", "/" + indexName + "/doc/0", singletonMap("refresh", "true"), document)
.getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_CREATED));
whenExportersAreReady(() -> {
@ -209,7 +211,7 @@ public class MonitoringIT extends ESRestTestCase {
final String type = (String) extractValue("_source.type", searchHit);
if (ClusterStatsMonitoringDoc.TYPE.equals(type)) {
assertClusterStatsMonitoringDoc(searchHit, collectionInterval);
assertClusterStatsMonitoringDoc(searchHit, collectionInterval, createAPMIndex);
} else if (IndexRecoveryMonitoringDoc.TYPE.equals(type)) {
assertIndexRecoveryMonitoringDoc(searchHit, collectionInterval);
} else if (IndicesStatsMonitoringDoc.TYPE.equals(type)) {
@ -294,7 +296,9 @@ public class MonitoringIT extends ESRestTestCase {
* Assert that a {@link ClusterStatsMonitoringDoc} contains the expected information
*/
@SuppressWarnings("unchecked")
private static void assertClusterStatsMonitoringDoc(final Map<String, Object> document, final TimeValue interval) throws Exception {
private static void assertClusterStatsMonitoringDoc(final Map<String, Object> document,
final TimeValue interval,
final boolean apmIndicesExist) throws Exception {
assertMonitoringDoc(document, MonitoredSystem.ES, ClusterStatsMonitoringDoc.TYPE, interval);
final Map<String, Object> source = (Map<String, Object>) document.get("_source");
@ -335,7 +339,13 @@ public class MonitoringIT extends ESRestTestCase {
final Map<String, Object> stackStats = (Map<String, Object>) source.get("stack_stats");
assertThat(stackStats, notNullValue());
assertThat(stackStats.size(), equalTo(1));
assertThat(stackStats.size(), equalTo(2));
final Map<String, Object> apm = (Map<String, Object>) stackStats.get("apm");
assertThat(apm, notNullValue());
assertThat(apm.size(), equalTo(1));
assertThat(apm.remove("found"), is(apmIndicesExist));
assertThat(apm.isEmpty(), is(true));
final Map<String, Object> xpackStats = (Map<String, Object>) stackStats.get("xpack");
assertThat(xpackStats, notNullValue());