honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Monitoring] Publish X-Pack Usage with Cluster Info 

This publishes X-Pack usage data to the cluster info from the elected master node. This allows phone home to retrieve this data from the index, rather than fetching it live from the connected cluster (thereby not getting it from any n - 1 clusers that are not connceted).

Original commit: elastic/x-pack-elasticsearch@79bfaaaf0b
This commit is contained in:
Chris Earle 2016-08-31 15:36:08 -04:00
parent 2e010d52e9
commit b60e8aebd2
5 changed files with 103 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterInfoMonitoringDoc.java
View File

@ -7,13 +7,17 @@ package org.elasticsearch.xpack.monitoring.collector.cluster;
import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
import java.util.List;
public class ClusterInfoMonitoringDoc extends MonitoringDoc {
private String clusterName;
private String version;
private License license;
private List<XPackFeatureSet.Usage> usage;
private ClusterStatsResponse clusterStats;
public ClusterInfoMonitoringDoc(String monitoringId, String monitoringVersion) {
@ -44,6 +48,14 @@ public class ClusterInfoMonitoringDoc extends MonitoringDoc {
this.license = license;
}
public List<XPackFeatureSet.Usage> getUsage() {
return usage;
}
public void setUsage(List<XPackFeatureSet.Usage> usage) {
this.usage = usage;
}
public ClusterStatsResponse getClusterStats() {
return clusterStats;
}

47
elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsCollector.java
View File

@ -13,10 +13,13 @@ import org.elasticsearch.action.admin.cluster.stats.ClusterStatsResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.LicenseService;
import org.elasticsearch.license.LicenseUtils;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.action.XPackUsageRequestBuilder;
import org.elasticsearch.xpack.monitoring.MonitoringSettings;
import org.elasticsearch.xpack.monitoring.collector.AbstractCollector;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringDoc;
@ -60,25 +63,17 @@ public class ClusterStatsCollector extends AbstractCollector {
@Override
protected Collection<MonitoringDoc> doCollect() throws Exception {
List<MonitoringDoc> results = new ArrayList<>(1);
final Supplier<ClusterStatsResponse> clusterStatsSupplier =
() -> client.admin().cluster().prepareClusterStats().get(monitoringSettings.clusterStatsTimeout());
final Supplier<List<XPackFeatureSet.Usage>> usageSupplier = () -> new XPackUsageRequestBuilder(client).get().getUsages();
// Retrieves cluster stats
ClusterStatsResponse clusterStats = null;
try {
clusterStats = client.admin().cluster().prepareClusterStats().get(monitoringSettings.clusterStatsTimeout());
} catch (ElasticsearchSecurityException e) {
if (LicenseUtils.isLicenseExpiredException(e)) {
logger.trace(
(Supplier<?>) () -> new ParameterizedMessage(
"collector [{}] - unable to collect data because of expired license", name()), e);
} else {
throw e;
}
}
final ClusterStatsResponse clusterStats = clusterStatsSupplier.get();
long timestamp = System.currentTimeMillis();
String clusterUUID = clusterUUID();
DiscoveryNode sourceNode = localNode();
final long timestamp = System.currentTimeMillis();
final String clusterUUID = clusterUUID();
final DiscoveryNode sourceNode = localNode();
final List<MonitoringDoc> results = new ArrayList<>(1);
// Adds a cluster info document
ClusterInfoMonitoringDoc clusterInfoDoc = new ClusterInfoMonitoringDoc(monitoringId(), monitoringVersion());
@ -89,6 +84,7 @@ public class ClusterStatsCollector extends AbstractCollector {
clusterInfoDoc.setVersion(Version.CURRENT.toString());
clusterInfoDoc.setLicense(licenseService.getLicense());
clusterInfoDoc.setClusterStats(clusterStats);
clusterInfoDoc.setUsage(collect(usageSupplier));
results.add(clusterInfoDoc);
// Adds a cluster stats document
@ -103,4 +99,21 @@ public class ClusterStatsCollector extends AbstractCollector {
return Collections.unmodifiableCollection(results);
}
@Nullable
private <T> T collect(final Supplier<T> supplier) {
try {
return supplier.get();
} catch (ElasticsearchSecurityException e) {
if (LicenseUtils.isLicenseExpiredException(e)) {
logger.trace((Supplier<?>) () -> new ParameterizedMessage(
"collector [{}] - unable to collect data because of expired license", name()), e);
} else {
throw e;
}
}
return null;
}
}

40
elasticsearch/x-pack/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterInfoResolver.java
View File

@ -11,11 +11,13 @@ import org.elasticsearch.common.hash.MessageDigests;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.XPackFeatureSet;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitoringDoc;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolver;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
public class ClusterInfoResolver extends MonitoringIndexNameResolver.Data<ClusterInfoMonitoringDoc> {
@ -34,27 +36,38 @@ public class ClusterInfoResolver extends MonitoringIndexNameResolver.Data<Cluste
@Override
protected void buildXContent(ClusterInfoMonitoringDoc document, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.field(Fields.CLUSTER_NAME, document.getClusterName());
builder.field(Fields.VERSION, document.getVersion());
builder.field("cluster_name", document.getClusterName());
builder.field("version", document.getVersion());
License license = document.getLicense();
final License license = document.getLicense();
if (license != null) {
builder.startObject(Fields.LICENSE);
builder.startObject("license");
Map<String, String> extraParams = new MapBuilder<String, String>()
.put(License.REST_VIEW_MODE, "true")
.map();
params = new ToXContent.DelegatingMapParams(extraParams, params);
license.toInnerXContent(builder, params);
builder.field(Fields.HKEY, hash(license, document.getClusterUUID()));
builder.field("hkey", hash(license, document.getClusterUUID()));
builder.endObject();
}
builder.startObject(Fields.CLUSTER_STATS);
ClusterStatsResponse clusterStats = document.getClusterStats();
final ClusterStatsResponse clusterStats = document.getClusterStats();
if (clusterStats != null) {
builder.startObject("cluster_stats");
clusterStats.toXContent(builder, params);
builder.endObject();
}
final List<XPackFeatureSet.Usage> usages = document.getUsage();
if (usages != null) {
// in the future we may choose to add other usages under the stack_stats section, but it is only xpack for now
// it may also be combined on the UI side of phone-home to add things like "kibana" and "logstash" under "stack_stats"
builder.startObject("stack_stats").startObject("xpack");
for (final XPackFeatureSet.Usage usage : usages) {
builder.field(usage.name(), usage);
}
builder.endObject().endObject();
}
builder.endObject();
}
public static String hash(License license, String clusterName) {
@ -66,15 +79,4 @@ public class ClusterInfoResolver extends MonitoringIndexNameResolver.Data<Cluste
return MessageDigests.toHexString(MessageDigests.sha256().digest(toHash.getBytes(StandardCharsets.UTF_8)));
}
static final class Fields {
static final String CLUSTER_NAME = "cluster_name";
static final String LICENSE = "license";
static final String VERSION = "version";
static final String CLUSTER_STATS = "cluster_stats";
static final String HKEY = "hkey";
static final String UID = "uid";
static final String TYPE = "type";
}
}

19
elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterInfoResolverTests.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.license.License;
import org.elasticsearch.xpack.monitoring.MonitoringFeatureSet;
import org.elasticsearch.xpack.monitoring.collector.cluster.ClusterInfoMonitoringDoc;
import org.elasticsearch.xpack.monitoring.exporter.MonitoringTemplateUtils;
import org.elasticsearch.xpack.monitoring.resolver.MonitoringIndexNameResolverTestCase;
@ -49,6 +50,7 @@ public class ClusterInfoResolverTests extends MonitoringIndexNameResolverTestCas
doc.setClusterName(randomAsciiOfLength(5));
doc.setClusterStats(new ClusterStatsResponse(Math.abs(randomLong()), ClusterName.CLUSTER_NAME_SETTING
.getDefault(Settings.EMPTY), randomAsciiOfLength(5), Collections.emptyList(), Collections.emptyList()));
doc.setUsage(Collections.singletonList(new MonitoringFeatureSet.Usage(randomBoolean(), randomBoolean(), emptyMap())));
return doc;
} catch (Exception e) {
throw new IllegalStateException("Failed to generated random ClusterInfoMonitoringDoc", e);
@ -72,13 +74,14 @@ public class ClusterInfoResolverTests extends MonitoringIndexNameResolverTestCas
assertThat(resolver.id(doc), equalTo(clusterUUID));
assertSource(resolver.source(doc, XContentType.JSON),
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"cluster_name",
"version",
"license",
"cluster_stats"));
Sets.newHashSet(
"cluster_uuid",
"timestamp",
"source_node",
"cluster_name",
"version",
"license",
"cluster_stats",
"stack_stats.xpack"));
}
}

43
elasticsearch/x-pack/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/resolver/cluster/ClusterInfoTests.java
View File

@ -27,6 +27,7 @@ import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.isEmptyOrNullString;
import static org.hamcrest.Matchers.not;
@ -61,14 +62,14 @@ public class ClusterInfoTests extends MonitoringIntegTestCase {
final String clusterUUID = client().admin().cluster().prepareState().setMetaData(true).get().getState().metaData().clusterUUID();
assertTrue(Strings.hasText(clusterUUID));
logger.debug("--> waiting for the monitoring data index to be created (it should have been created by the ClusterInfoCollector)");
// waiting for the monitoring data index to be created (it should have been created by the ClusterInfoCollector
String dataIndex = ".monitoring-data-" + MonitoringTemplateUtils.TEMPLATE_VERSION;
awaitIndexExists(dataIndex);
logger.debug("--> waiting for cluster info collector to collect data");
// waiting for cluster info collector to collect data
awaitMonitoringDocsCount(equalTo(1L), ClusterInfoResolver.TYPE);
logger.debug("--> retrieving cluster info document");
// retrieving cluster info document
GetResponse response = client().prepareGet(dataIndex, ClusterInfoResolver.TYPE, clusterUUID).get();
assertTrue("cluster_info document does not exist in data index", response.isExists());
@ -80,20 +81,19 @@ public class ClusterInfoTests extends MonitoringIntegTestCase {
assertThat(source.get(MonitoringIndexNameResolver.Fields.CLUSTER_UUID), notNullValue());
assertThat(source.get(MonitoringIndexNameResolver.Fields.TIMESTAMP), notNullValue());
assertThat(source.get(MonitoringIndexNameResolver.Fields.SOURCE_NODE), notNullValue());
assertThat(source.get(ClusterInfoResolver.Fields.CLUSTER_NAME), equalTo(cluster().getClusterName()));
assertThat(source.get(ClusterInfoResolver.Fields.VERSION), equalTo(Version.CURRENT.toString()));
assertThat(source.get("cluster_name"), equalTo(cluster().getClusterName()));
assertThat(source.get("version"), equalTo(Version.CURRENT.toString()));
logger.debug("--> checking that the document contains license information");
Object licenseObj = source.get(ClusterInfoResolver.Fields.LICENSE);
Object licenseObj = source.get("license");
assertThat(licenseObj, instanceOf(Map.class));
Map license = (Map) licenseObj;
assertThat(license, instanceOf(Map.class));
String uid = (String) license.get(ClusterInfoResolver.Fields.UID);
String uid = (String) license.get("uid");
assertThat(uid, not(isEmptyOrNullString()));
String type = (String) license.get(ClusterInfoResolver.Fields.TYPE);
String type = (String) license.get("type");
assertThat(type, not(isEmptyOrNullString()));
String status = (String) license.get(License.Fields.STATUS);
@ -103,7 +103,7 @@ public class ClusterInfoTests extends MonitoringIntegTestCase {
assertThat(expiryDate, greaterThan(0L));
// We basically recompute the hash here
String hkey = (String) license.get(ClusterInfoResolver.Fields.HKEY);
String hkey = (String) license.get("hkey");
String recalculated = ClusterInfoResolver.hash(status, uid, type, String.valueOf(expiryDate), clusterUUID);
assertThat(hkey, equalTo(recalculated));
@ -112,14 +112,30 @@ public class ClusterInfoTests extends MonitoringIntegTestCase {
assertThat((Long) license.get(License.Fields.ISSUE_DATE_IN_MILLIS), greaterThan(0L));
assertThat((Integer) license.get(License.Fields.MAX_NODES), greaterThan(0));
Object clusterStats = source.get(ClusterInfoResolver.Fields.CLUSTER_STATS);
Object clusterStats = source.get("cluster_stats");
assertNotNull(clusterStats);
assertThat(clusterStats, instanceOf(Map.class));
assertThat(((Map) clusterStats).size(), greaterThan(0));
Object stackStats = source.get("stack_stats");
assertNotNull(stackStats);
assertThat(stackStats, instanceOf(Map.class));
assertThat(((Map) stackStats).size(), equalTo(1));
Object xpack = ((Map)stackStats).get("xpack");
assertNotNull(xpack);
assertThat(xpack, instanceOf(Map.class));
// it must have at least monitoring, but others may be hidden
assertThat(((Map) xpack).size(), greaterThanOrEqualTo(1));
Object monitoring = ((Map)xpack).get("monitoring");
assertNotNull(monitoring);
// we don't make any assumptions about what's in it, only that it's there
assertThat(monitoring, instanceOf(Map.class));
waitForMonitoringTemplates();
logger.debug("--> check that the cluster_info is not indexed");
// check that the cluster_info is not indexed
securedFlush();
securedRefresh();
@ -131,8 +147,7 @@ public class ClusterInfoTests extends MonitoringIntegTestCase {
.should(QueryBuilders.matchQuery(License.Fields.STATUS, License.Status.ACTIVE.label()))
.should(QueryBuilders.matchQuery(License.Fields.STATUS, License.Status.INVALID.label()))
.should(QueryBuilders.matchQuery(License.Fields.STATUS, License.Status.EXPIRED.label()))
.should(QueryBuilders.matchQuery(ClusterInfoResolver.Fields.CLUSTER_NAME,
cluster().getClusterName()))
.should(QueryBuilders.matchQuery("cluster_name", cluster().getClusterName()))
.minimumNumberShouldMatch(1)
).get(), 0L);
}