honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Add Indices Stats collector 

Closes elastic/elasticsearch#555

Original commit: elastic/x-pack-elasticsearch@63a0e02258
This commit is contained in:
Tanguy Leroux 2015-09-01 14:00:26 +02:00
parent 9e5053a5c0
commit 7a59749392
10 changed files with 345 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/CollectorModule.java
View File

@ -11,6 +11,7 @@ import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector;
import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector;
import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
@ -24,6 +25,7 @@ public class CollectorModule extends AbstractModule {
public CollectorModule() {
// Registers default collectors
registerCollector(LicensesCollector.class);
registerCollector(IndicesStatsCollector.class);
registerCollector(IndexStatsCollector.class);
registerCollector(ClusterStatsCollector.class);
registerCollector(ClusterStateCollector.class);

54
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsCollector.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.collector.indices;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterService;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.AbstractCollector;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.settings.MarvelSettings;
import org.elasticsearch.marvel.license.LicenseService;
import java.util.Collection;
import java.util.Collections;
/**
* Collector for indices statistics.
* <p/>
* This collector runs on the master node only and collect one {@link IndicesStatsMarvelDoc} document.
*/
public class IndicesStatsCollector extends AbstractCollector<IndicesStatsCollector> {
public static final String NAME = "indices-stats-collector";
public static final String TYPE = "marvel_indices_stats";
private final Client client;
@Inject
public IndicesStatsCollector(Settings settings, ClusterService clusterService, MarvelSettings marvelSettings, LicenseService licenseService,
Client client) {
super(settings, NAME, clusterService, marvelSettings, licenseService);
this.client = client;
}
@Override
protected boolean canCollect() {
return super.canCollect() && isLocalNodeMaster();
}
@Override
protected Collection<MarvelDoc> doCollect() throws Exception {
IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
.setRefresh(true)
.get(marvelSettings.indicesStatsTimeout());
MarvelDoc result = new IndicesStatsMarvelDoc(clusterUUID(), TYPE, System.currentTimeMillis(), indicesStats);
return Collections.singletonList(result);
}
}

23
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndicesStatsMarvelDoc.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.collector.indices;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
public class IndicesStatsMarvelDoc extends MarvelDoc {
private final IndicesStatsResponse indicesStats;
public IndicesStatsMarvelDoc(String clusterUUID, String type, long timestamp, IndicesStatsResponse indicesStats) {
super(clusterUUID, type, timestamp);
this.indicesStats = indicesStats;
}
public IndicesStatsResponse getIndicesStats() {
return indicesStats;
}
}

5
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java
View File

@ -11,12 +11,14 @@ import org.elasticsearch.marvel.agent.collector.cluster.ClusterStateCollector;
import org.elasticsearch.marvel.agent.collector.cluster.ClusterStatsCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndexRecoveryCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector;
import org.elasticsearch.marvel.agent.collector.licenses.LicensesCollector;
import org.elasticsearch.marvel.agent.collector.node.NodeStatsCollector;
import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStateRenderer;
import org.elasticsearch.marvel.agent.renderer.cluster.ClusterStatsRenderer;
import org.elasticsearch.marvel.agent.renderer.indices.IndexRecoveryRenderer;
import org.elasticsearch.marvel.agent.renderer.indices.IndexStatsRenderer;
import org.elasticsearch.marvel.agent.renderer.indices.IndicesStatsRenderer;
import org.elasticsearch.marvel.agent.renderer.licenses.LicensesRenderer;
import org.elasticsearch.marvel.agent.renderer.node.NodeStatsRenderer;
@ -39,6 +41,9 @@ public class RendererModule extends AbstractModule {
bind(LicensesRenderer.class).asEagerSingleton();
mbinder.addBinding(LicensesCollector.TYPE).to(LicensesRenderer.class);
bind(IndicesStatsRenderer.class).asEagerSingleton();
mbinder.addBinding(IndicesStatsCollector.TYPE).to(IndicesStatsRenderer.class);
bind(IndexStatsRenderer.class).asEagerSingleton();
mbinder.addBinding(IndexStatsCollector.TYPE).to(IndexStatsRenderer.class);

57
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRenderer.java Normal file
View File

@ -0,0 +1,57 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMarvelDoc;
import org.elasticsearch.marvel.agent.renderer.AbstractRenderer;
import java.io.IOException;
public class IndicesStatsRenderer extends AbstractRenderer<IndicesStatsMarvelDoc> {
public static final String[] FILTERS = {
"indices_stats._all.primaries.docs.count",
"indices_stats._all.primaries.indexing.index_time_in_millis",
"indices_stats._all.primaries.indexing.index_total",
"indices_stats._all.primaries.indexing.is_throttled",
"indices_stats._all.primaries.indexing.throttle_time_in_millis",
"indices_stats._all.primaries.search.query_time_in_millis",
"indices_stats._all.primaries.search.query_total",
"indices_stats._all.primaries.store.size_in_bytes",
"indices_stats._all.total.docs.count",
"indices_stats._all.total.indexing.index_time_in_millis",
"indices_stats._all.total.indexing.index_total",
"indices_stats._all.total.indexing.is_throttled",
"indices_stats._all.total.indexing.throttle_time_in_millis",
"indices_stats._all.total.search.query_time_in_millis",
"indices_stats._all.total.search.query_total",
"indices_stats._all.total.store.size_in_bytes",
};
public IndicesStatsRenderer() {
super(FILTERS, true);
}
@Override
protected void doRender(IndicesStatsMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException {
builder.startObject(Fields.INDICES_STATS);
IndicesStatsResponse indicesStats = marvelDoc.getIndicesStats();
if (indicesStats != null) {
indicesStats.toXContent(builder, params);
}
builder.endObject();
}
static final class Fields {
static final XContentBuilderString INDICES_STATS = new XContentBuilderString("indices_stats");
}
}

11
marvel/src/main/java/org/elasticsearch/marvel/agent/settings/MarvelSettings.java
View File

@ -27,6 +27,7 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
public static final String INTERVAL = PREFIX + "interval";
public static final String STARTUP_DELAY = PREFIX + "startup.delay";
public static final String INDEX_STATS_TIMEOUT = PREFIX + "index.stats.timeout";
public static final String INDICES_STATS_TIMEOUT = PREFIX + "indices.stats.timeout";
public static final String INDICES = PREFIX + "indices";
public static final String CLUSTER_STATE_TIMEOUT = PREFIX + "cluster.state.timeout";
public static final String CLUSTER_STATS_TIMEOUT = PREFIX + "cluster.stats.timeout";
@ -44,7 +45,9 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
map.put(STARTUP_DELAY, timeSetting(STARTUP_DELAY, null,
"Waiting time before the agent start to collect data (default to sampling interval)", false));
map.put(INDEX_STATS_TIMEOUT, timeoutSetting(INDEX_STATS_TIMEOUT, TimeValue.timeValueMinutes(10),
"Timeout value when collecting indices statistics (default to 10m)", true));
"Timeout value when collecting index statistics (default to 10m)", true));
map.put(INDICES_STATS_TIMEOUT, timeoutSetting(INDICES_STATS_TIMEOUT, TimeValue.timeValueMinutes(10),
"Timeout value when collecting total indices statistics (default to 10m)", true));
map.put(INDICES, arraySetting(INDICES, Strings.EMPTY_ARRAY,
"List of indices names whose stats will be exported (default to all indices)", true));
map.put(CLUSTER_STATE_TIMEOUT, timeoutSetting(CLUSTER_STATE_TIMEOUT, TimeValue.timeValueMinutes(10),
@ -54,7 +57,7 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
map.put(INDEX_RECOVERY_TIMEOUT, timeoutSetting(INDEX_RECOVERY_TIMEOUT, TimeValue.timeValueMinutes(10),
"Timeout value when collecting the recovery information (default to 10m)", true));
map.put(INDEX_RECOVERY_ACTIVE_ONLY, booleanSetting(INDEX_RECOVERY_ACTIVE_ONLY, Boolean.FALSE,
"INDEX_RECOVERY_ACTIVE_ONLY to indicate if only active recoveries should be collected (default to false: all recoveries are collected)", true));
"Flag to indicate if only active recoveries should be collected (default to false: all recoveries are collected)", true));
map.put(COLLECTORS, arraySetting(COLLECTORS, Strings.EMPTY_ARRAY,
"List of collectors allowed to collect data (default to all)", false));
map.put(LICENSE_GRACE_PERIOD, timeSetting(LICENSE_GRACE_PERIOD, MAX_LICENSE_GRACE_PERIOD,
@ -145,6 +148,10 @@ public class MarvelSettings extends AbstractComponent implements NodeSettingsSer
return getSettingValue(INDEX_STATS_TIMEOUT);
}
public TimeValue indicesStatsTimeout() {
return getSettingValue(INDICES_STATS_TIMEOUT);
}
public String[] indices() {
return getSettingValue(INDICES);
}

34
marvel/src/main/resources/marvel_index_template.json
View File

@ -25,6 +25,40 @@
}
}
},
"marvel_indices_stats": {
"properties": {
"indices_stats": {
"properties": {
"_all": {
"properties": {
"primaries": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
},
"total": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
}
}
}
}
}
}
},
"marvel_index_stats": {
"properties": {
"index_stats": {

75
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsIT.java Normal file
View File

@ -0,0 +1,75 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsCollector;
import org.elasticsearch.marvel.agent.renderer.AbstractRendererTestCase;
import org.elasticsearch.search.SearchHit;
import org.junit.Test;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import static org.hamcrest.Matchers.greaterThan;
public class IndicesStatsIT extends AbstractRendererTestCase {
@Override
protected Collection<String> collectors() {
return Collections.singletonList(IndicesStatsCollector.NAME);
}
@Test
public void testIndicesStats() throws Exception {
logger.debug("--> creating some indices for future indices stats");
final int nbIndices = randomIntBetween(1, 5);
for (int i = 0; i < nbIndices; i++) {
createIndex("stat" + i);
}
final long[] nbDocsPerIndex = new long[nbIndices];
for (int i = 0; i < nbIndices; i++) {
nbDocsPerIndex[i] = randomIntBetween(1, 50);
for (int j = 0; j < nbDocsPerIndex[i]; j++) {
client().prepareIndex("stat" + i, "type1").setSource("num", i).get();
}
}
waitForMarvelDocs(IndicesStatsCollector.TYPE);
logger.debug("--> wait for indicesx stats collector to collect global stat");
assertBusy(new Runnable() {
@Override
public void run() {
for (int i = 0; i < nbIndices; i++) {
CountResponse count = client().prepareCount()
.setTypes(IndicesStatsCollector.TYPE)
.get();
assertThat(count.getCount(), greaterThan(0L));
}
}
});
logger.debug("--> searching for marvel documents of type [{}]", IndicesStatsCollector.TYPE);
SearchResponse response = client().prepareSearch().setTypes(IndicesStatsCollector.TYPE).get();
assertThat(response.getHits().getTotalHits(), greaterThan(0L));
logger.debug("--> checking that every document contains the expected fields");
String[] filters = IndicesStatsRenderer.FILTERS;
for (SearchHit searchHit : response.getHits().getHits()) {
Map<String, Object> fields = searchHit.sourceAsMap();
for (String filter : filters) {
assertContains(filter, fields);
}
}
logger.debug("--> indices stats successfully collected");
}
}

42
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndicesStatsRendererTests.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.marvel.agent.collector.indices.IndicesStatsMarvelDoc;
import org.elasticsearch.marvel.agent.renderer.Renderer;
import org.elasticsearch.marvel.agent.renderer.RendererTestUtils;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.StreamsUtils;
import org.junit.Test;
public class IndicesStatsRendererTests extends ESSingleNodeTestCase {
private static final String SAMPLE_FILE = "/samples/marvel_indices_stats.json";
@Test
public void testIndexStatsRenderer() throws Exception {
createIndex("index-0");
logger.debug("--> retrieving indices stats response");
IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().get();
logger.debug("--> creating the indices stats marvel document");
IndicesStatsMarvelDoc marvelDoc = new IndicesStatsMarvelDoc("test", "marvel_indices_stats", 1437580442979L, indicesStats);
logger.debug("--> rendering the document");
Renderer renderer = new IndicesStatsRenderer();
String result = RendererTestUtils.renderAsJSON(marvelDoc, renderer);
logger.debug("--> loading sample document from file {}", SAMPLE_FILE);
String expected = StreamsUtils.copyToStringFromClasspath(SAMPLE_FILE);
logger.debug("--> comparing both documents, they must have the same structure");
RendererTestUtils.assertJSONStructure(result, expected);
}
}

44
marvel/src/test/resources/samples/marvel_indices_stats.json Normal file
View File

@ -0,0 +1,44 @@
{
"cluster_uuid": "dsFPzYRyQCe6cq48a0wxkQ",
"timestamp": "2015-07-22T15:54:02.979Z",
"indices_stats": {
"_all": {
"primaries": {
"docs": {
"count": 0
},
"store": {
"size_in_bytes": 127
},
"indexing": {
"index_total": 1,
"index_time_in_millis": 286,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"search": {
"query_total": 0,
"query_time_in_millis": 0
}
},
"total": {
"docs": {
"count": 0
},
"store": {
"size_in_bytes": 127
},
"indexing": {
"index_total": 1,
"index_time_in_millis": 286,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"search": {
"query_total": 0,
"query_time_in_millis": 0
}
}
}
}
}