diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java index c73adeab8ce..e1eadbbd379 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java @@ -11,6 +11,7 @@ import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector; import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector; import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector; +import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector; import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector; import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; @@ -24,6 +25,7 @@ public class CollectorModule extends AbstractModule { public CollectorModule() { // Registers default collectors registerCollector(LicensesCollector.class); + registerCollector(IndicesStatsCollector.class); registerCollector(IndexStatsCollector.class); registerCollector(ClusterStatsCollector.class); registerCollector(ClusterStateCollector.class); diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java new file mode 100644 index 00000000000..b3fdf4328ae --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.collector.indices; + +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.marvel.agent.collector.AbstractCollector; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.marvel.agent.settings.MarvelSettings; +import org.elasticsearch.marvel.license.LicenseService; + +import java.util.Collection; +import java.util.Collections; + +/** + * Collector for indices statistics. + *

+ * This collector runs on the master node only and collect one {@link IndicesStatsMarvelDoc} document. + */ +public class IndicesStatsCollector extends AbstractCollector { + + public static final String NAME = "indices-stats-collector"; + public static final String TYPE = "marvel_indices_stats"; + + private final Client client; + + @Inject + public IndicesStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService, + Client client) { + super(settings, NAME, clusterService, marvelSettings, licenseService); + this.client = client; + } + + @Override + protected boolean canCollect() { + return super.canCollect() && isLocalNodeMaster(); + } + + @Override + protected Collection doCollect() throws Exception { + IndicesStatsResponse indicesStats = client.admin().indices().prepareStats() + .setRefresh(true) + .get(marvelSettings.indicesStatsTimeout()); + + MarvelDoc result = new IndicesStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), indicesStats); + return Collections.singletonList(result); + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsMarvelDoc.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsMarvelDoc.java new file mode 100644 index 00000000000..4ed0d782a4b --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsMarvelDoc.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.collector.indices; + +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; + +public class IndicesStatsMarvelDoc extends MarvelDoc { + + private final IndicesStatsResponse indicesStats; + + public IndicesStatsMarvelDoc(String clusterUUID, String type, long timestamp, IndicesStatsResponse indicesStats) { + super(clusterUUID, type, timestamp); + this.indicesStats = indicesStats; + } + + public IndicesStatsResponse getIndicesStats() { + return indicesStats; + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java index cf8ff05d7da..3eab2f96ac5 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java @@ -11,12 +11,14 @@ import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector; import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector; import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector; import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector; +import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector; import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector; import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector; import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStateRenderer; import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStatsRenderer; import org.elasticsearch.marvel.agent.renderer.indices.IndexRecoveryRenderer; import org.elasticsearch.marvel.agent.renderer.indices.IndexStatsRenderer; +import org.elasticsearch.marvel.agent.renderer.indices.IndicesStatsRenderer; import org.elasticsearch.marvel.agent.renderer.licenses.LicensesRenderer; import org.elasticsearch.marvel.agent.renderer.node.NodeStatsRenderer; @@ -39,6 +41,9 @@ public class RendererModule extends AbstractModule { bind(LicensesRenderer.class).asEagerSingleton(); mbinder.addBinding(LicensesCollector.TYPE).to(LicensesRenderer.class); + bind(IndicesStatsRenderer.class).asEagerSingleton(); + mbinder.addBinding(IndicesStatsCollector.TYPE).to(IndicesStatsRenderer.class); + bind(IndexStatsRenderer.class).asEagerSingleton(); mbinder.addBinding(IndexStatsCollector.TYPE).to(IndexStatsRenderer.class); diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRenderer.java new file mode 100644 index 00000000000..73e55128eb6 --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRenderer.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer.indices; + +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMarvelDoc; +import org.elasticsearch.marvel.agent.renderer.AbstractRenderer; + +import java.io.IOException; + +public class IndicesStatsRenderer extends AbstractRenderer { + + public static final String[] FILTERS = { + "indices_stats._all.primaries.docs.count", + "indices_stats._all.primaries.indexing.index_time_in_millis", + "indices_stats._all.primaries.indexing.index_total", + "indices_stats._all.primaries.indexing.is_throttled", + "indices_stats._all.primaries.indexing.throttle_time_in_millis", + "indices_stats._all.primaries.search.query_time_in_millis", + "indices_stats._all.primaries.search.query_total", + "indices_stats._all.primaries.store.size_in_bytes", + "indices_stats._all.total.docs.count", + "indices_stats._all.total.indexing.index_time_in_millis", + "indices_stats._all.total.indexing.index_total", + "indices_stats._all.total.indexing.is_throttled", + "indices_stats._all.total.indexing.throttle_time_in_millis", + "indices_stats._all.total.search.query_time_in_millis", + "indices_stats._all.total.search.query_total", + "indices_stats._all.total.store.size_in_bytes", + }; + + public IndicesStatsRenderer() { + super(FILTERS, true); + } + + @Override + protected void doRender(IndicesStatsMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException { + builder.startObject(Fields.INDICES_STATS); + + IndicesStatsResponse indicesStats = marvelDoc.getIndicesStats(); + if (indicesStats != null) { + indicesStats.toXContent(builder, params); + } + + builder.endObject(); + } + + static final class Fields { + static final XContentBuilderString INDICES_STATS = new XContentBuilderString("indices_stats"); + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java index 171772ee488..977ce62048f 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java @@ -27,6 +27,7 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer public static final String INTERVAL = PREFIX + "interval"; public static final String STARTUP_DELAY = PREFIX + "startup.delay"; public static final String INDEX_STATS_TIMEOUT = PREFIX + "index.stats.timeout"; + public static final String INDICES_STATS_TIMEOUT = PREFIX + "indices.stats.timeout"; public static final String INDICES = PREFIX + "indices"; public static final String CLUSTER_STATE_TIMEOUT = PREFIX + "cluster.state.timeout"; public static final String CLUSTER_STATS_TIMEOUT = PREFIX + "cluster.stats.timeout"; @@ -44,7 +45,9 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer map.put(STARTUP_DELAY, timeSetting(STARTUP_DELAY, null, "Waiting time before the agent start to collect data (default to sampling interval)", false)); map.put(INDEX_STATS_TIMEOUT, timeoutSetting(INDEX_STATS_TIMEOUT, TimeValue.timeValueMinutes(10), - "Timeout value when collecting indices statistics (default to 10m)", true)); + "Timeout value when collecting index statistics (default to 10m)", true)); + map.put(INDICES_STATS_TIMEOUT, timeoutSetting(INDICES_STATS_TIMEOUT, TimeValue.timeValueMinutes(10), + "Timeout value when collecting total indices statistics (default to 10m)", true)); map.put(INDICES, arraySetting(INDICES, Strings.EMPTY_ARRAY, "List of indices names whose stats will be exported (default to all indices)", true)); map.put(CLUSTER_STATE_TIMEOUT, timeoutSetting(CLUSTER_STATE_TIMEOUT, TimeValue.timeValueMinutes(10), @@ -54,7 +57,7 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer map.put(INDEX_RECOVERY_TIMEOUT, timeoutSetting(INDEX_RECOVERY_TIMEOUT, TimeValue.timeValueMinutes(10), "Timeout value when collecting the recovery information (default to 10m)", true)); map.put(INDEX_RECOVERY_ACTIVE_ONLY, booleanSetting(INDEX_RECOVERY_ACTIVE_ONLY, Boolean.FALSE, - "INDEX_RECOVERY_ACTIVE_ONLY to indicate if only active recoveries should be collected (default to false: all recoveries are collected)", true)); + "Flag to indicate if only active recoveries should be collected (default to false: all recoveries are collected)", true)); map.put(COLLECTORS, arraySetting(COLLECTORS, Strings.EMPTY_ARRAY, "List of collectors allowed to collect data (default to all)", false)); map.put(LICENSE_GRACE_PERIOD, timeSetting(LICENSE_GRACE_PERIOD, MAX_LICENSE_GRACE_PERIOD, @@ -145,6 +148,10 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer return getSettingValue(INDEX_STATS_TIMEOUT); } + public TimeValue indicesStatsTimeout() { + return getSettingValue(INDICES_STATS_TIMEOUT); + } + public String[] indices() { return getSettingValue(INDICES); } diff --git a/marvel/src/main/resources/marvel_index_template.json b/marvel/src/main/resources/marvel_index_template.json index ab694fb848a..5dfa3de6250 100644 --- a/marvel/src/main/resources/marvel_index_template.json +++ b/marvel/src/main/resources/marvel_index_template.json @@ -25,6 +25,40 @@ } } }, + "marvel_indices_stats": { + "properties": { + "indices_stats": { + "properties": { + "_all": { + "properties": { + "primaries": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + }, "marvel_index_stats": { "properties": { "index_stats": { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsIT.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsIT.java new file mode 100644 index 00000000000..7e037c63296 --- /dev/null +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsIT.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer.indices; + +import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector; +import org.elasticsearch.marvel.agent.renderer.AbstractRendererTestCase; +import org.elasticsearch.search.SearchHit; +import org.junit.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.Matchers.greaterThan; + +public class IndicesStatsIT extends AbstractRendererTestCase { + + @Override + protected Collection collectors() { + return Collections.singletonList(IndicesStatsCollector.NAME); + } + + @Test + public void testIndicesStats() throws Exception { + logger.debug("--> creating some indices for future indices stats"); + final int nbIndices = randomIntBetween(1, 5); + for (int i = 0; i < nbIndices; i++) { + createIndex("stat" + i); + } + + final long[] nbDocsPerIndex = new long[nbIndices]; + for (int i = 0; i < nbIndices; i++) { + nbDocsPerIndex[i] = randomIntBetween(1, 50); + for (int j = 0; j < nbDocsPerIndex[i]; j++) { + client().prepareIndex("stat" + i, "type1").setSource("num", i).get(); + } + } + + waitForMarvelDocs(IndicesStatsCollector.TYPE); + + logger.debug("--> wait for indicesx stats collector to collect global stat"); + assertBusy(new Runnable() { + @Override + public void run() { + for (int i = 0; i < nbIndices; i++) { + CountResponse count = client().prepareCount() + .setTypes(IndicesStatsCollector.TYPE) + .get(); + assertThat(count.getCount(), greaterThan(0L)); + } + } + }); + + logger.debug("--> searching for marvel documents of type [{}]", IndicesStatsCollector.TYPE); + SearchResponse response = client().prepareSearch().setTypes(IndicesStatsCollector.TYPE).get(); + assertThat(response.getHits().getTotalHits(), greaterThan(0L)); + + logger.debug("--> checking that every document contains the expected fields"); + String[] filters = IndicesStatsRenderer.FILTERS; + for (SearchHit searchHit : response.getHits().getHits()) { + Map fields = searchHit.sourceAsMap(); + + for (String filter : filters) { + assertContains(filter, fields); + } + } + + logger.debug("--> indices stats successfully collected"); + } +} diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRendererTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRendererTests.java new file mode 100644 index 00000000000..6a6bb5bc079 --- /dev/null +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRendererTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer.indices; + +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMarvelDoc; +import org.elasticsearch.marvel.agent.renderer.Renderer; +import org.elasticsearch.marvel.agent.renderer.RendererTestUtils; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.StreamsUtils; +import org.junit.Test; + +public class IndicesStatsRendererTests extends ESSingleNodeTestCase { + + private static final String SAMPLE_FILE = "/samples/marvel_indices_stats.json"; + + @Test + public void testIndexStatsRenderer() throws Exception { + createIndex("index-0"); + + logger.debug("--> retrieving indices stats response"); + IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().get(); + + logger.debug("--> creating the indices stats marvel document"); + IndicesStatsMarvelDoc marvelDoc = new IndicesStatsMarvelDoc("test", "marvel_indices_stats", 1437580442979L, indicesStats); + + logger.debug("--> rendering the document"); + Renderer renderer = new IndicesStatsRenderer(); + String result = RendererTestUtils.renderAsJSON(marvelDoc, renderer); + + logger.debug("--> loading sample document from file {}", SAMPLE_FILE); + String expected = StreamsUtils.copyToStringFromClasspath(SAMPLE_FILE); + + logger.debug("--> comparing both documents, they must have the same structure"); + RendererTestUtils.assertJSONStructure(result, expected); + } +} diff --git a/marvel/src/test/resources/samples/marvel_indices_stats.json b/marvel/src/test/resources/samples/marvel_indices_stats.json new file mode 100644 index 00000000000..bed581d1e00 --- /dev/null +++ b/marvel/src/test/resources/samples/marvel_indices_stats.json @@ -0,0 +1,44 @@ +{ + "cluster_uuid": "dsFPzYRyQCe6cq48a0wxkQ", + "timestamp": "2015-07-22T15:54:02.979Z", + "indices_stats": { + "_all": { + "primaries": { + "docs": { + "count": 0 + }, + "store": { + "size_in_bytes": 127 + }, + "indexing": { + "index_total": 1, + "index_time_in_millis": 286, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "search": { + "query_total": 0, + "query_time_in_millis": 0 + } + }, + "total": { + "docs": { + "count": 0 + }, + "store": { + "size_in_bytes": 127 + }, + "indexing": { + "index_total": 1, + "index_time_in_millis": 286, + "is_throttled": false, + "throttle_time_in_millis": 0 + }, + "search": { + "query_total": 0, + "query_time_in_millis": 0 + } + } + } + } +}