honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Add additional stats to index/node stats 

Closes elastic/elasticsearch#397

Original commit: elastic/x-pack-elasticsearch@0ddaeb8bb1
This commit is contained in:
Tanguy Leroux 2015-08-11 19:09:57 +02:00
parent c8b83daf44
commit b3b7ca34a4
10 changed files with 221 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java
View File

@ -49,9 +49,6 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
ImmutableList.Builder<MarvelDoc> results = ImmutableList.builder();
IndicesStatsResponse indicesStats = client.admin().indices().prepareStats()
.setStore(true)
.setIndexing(true)
.setDocs(true)
.setIndices(marvelSettings.indices())
.get(marvelSettings.indexStatsTimeout());

2
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStateRenderer.java
View File

@ -20,7 +20,7 @@ import java.util.Locale;
public class ClusterStateRenderer extends AbstractRenderer<ClusterStateMarvelDoc> {
private static final String[] FILTERS = {
public static final String[] FILTERS = {
"cluster_state.version",
"cluster_state.master_node",
"cluster_state.status",

2
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/cluster/ClusterStatsRenderer.java
View File

@ -16,7 +16,7 @@ import java.io.IOException;
public class ClusterStatsRenderer extends AbstractRenderer<ClusterStatsMarvelDoc> {
private static final String[] FILTERS = {
public static final String[] FILTERS = {
"cluster_stats.nodes.count.total",
"cluster_stats.indices.shards.total",
"cluster_stats.indices.shards.index.replication.min",

19
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRenderer.java
View File

@ -16,12 +16,20 @@ import java.io.IOException;
public class IndexStatsRenderer extends AbstractRenderer<IndexStatsMarvelDoc> {
private static final String[] FILTERS = {
public static final String[] FILTERS = {
"index_stats.index",
"index_stats.primaries.docs.count",
"index_stats.total.docs.count",
"index_stats.total.indexing.index_total",
"index_stats.total.indexing.index_time_in_millis",
"index_stats.total.indexing.throttle_time_in_millis",
"index_stats.total.merges.total_size_in_bytes",
"index_stats.total.search.query_total",
"index_stats.total.search.query_time_in_millis",
"index_stats.total.segments.memory_in_bytes",
"index_stats.total.store.size_in_bytes",
"index_stats.total.store.throttle_time_in_millis",
"index_stats.total.indexing.throttle_time_in_millis",
"index_stats.total.refresh.total_time_in_millis",
};
public IndexStatsRenderer() {
@ -43,6 +51,12 @@ public class IndexStatsRenderer extends AbstractRenderer<IndexStatsMarvelDoc> {
indexStats.getTotal().toXContent(builder, params);
}
builder.endObject();
builder.startObject(Fields.PRIMARIES);
if (indexStats.getPrimaries() != null) {
indexStats.getPrimaries().toXContent(builder, params);
}
builder.endObject();
}
}
builder.endObject();
@ -52,5 +66,6 @@ public class IndexStatsRenderer extends AbstractRenderer<IndexStatsMarvelDoc> {
static final XContentBuilderString INDEX_STATS = new XContentBuilderString("index_stats");
static final XContentBuilderString INDEX = new XContentBuilderString("index");
static final XContentBuilderString TOTAL = new XContentBuilderString("total");
static final XContentBuilderString PRIMARIES = new XContentBuilderString("primaries");
}
}

3
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/node/NodeStatsRenderer.java
View File

@ -16,7 +16,7 @@ import java.io.IOException;
public class NodeStatsRenderer extends AbstractRenderer<NodeStatsMarvelDoc> {
private static final String[] FILTERS = {
public static final String[] FILTERS = {
// Extra information
"node_stats.node_id",
"node_stats.node_master",
@ -32,6 +32,7 @@ public class NodeStatsRenderer extends AbstractRenderer<NodeStatsMarvelDoc> {
"node_stats.fs.total.total_in_bytes",
"node_stats.fs.total.free_in_bytes",
"node_stats.fs.total.available_in_bytes",
"node_stats.os.load_average",
"node_stats.process.max_file_descriptors",
"node_stats.process.open_file_descriptors",
"node_stats.jvm.mem.heap_used_in_bytes",

48
marvel/src/main/resources/marvel_index_template.json
View File

@ -33,6 +33,17 @@
"type": "string",
"index": "not_analyzed"
},
"primaries": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
},
"total": {
"properties": {
"docs": {
@ -54,10 +65,47 @@
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
},
"index_time_in_millis": {
"type": "long"
},
"throttle_time_in_millis": {
"type": "long"
}
}
},
"merges": {
"properties": {
"total_size_in_bytes": {
"type": "long"
}
}
},
"search": {
"properties": {
"query_total": {
"type": "long"
},
"query_time_in_millis": {
"type": "long"
}
}
},
"segments": {
"properties": {
"memory_in_bytes": {
"type": "long"
}
}
},
"refresh": {
"properties": {
"total_time_in_millis": {
"type": "long"
}
}
}
}
}

114
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererIT.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.license.plugin.LicensePlugin;
import org.elasticsearch.marvel.MarvelPlugin;
import org.elasticsearch.marvel.agent.AgentService;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
import org.elasticsearch.node.Node;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
import org.junit.Test;
import java.util.Map;
import static org.hamcrest.Matchers.*;
@ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 1)
public class IndexStatsRendererIT extends ESIntegTestCase {
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
.put("plugin.types", MarvelPlugin.class.getName() + "," + LicensePlugin.class.getName())
.put(Node.HTTP_ENABLED, true)
.put(AgentService.SETTINGS_INTERVAL, "200ms")
.build();
}
@Test
public void testIndexStats() throws Exception {
final int nbIndices = randomIntBetween(1, 5);
for (int i = 0; i < nbIndices; i++) {
createIndex("test" + i);
}
final long[] nbDocsPerIndex = new long[nbIndices];
for (int i = 0; i < nbIndices; i++) {
nbDocsPerIndex[i] = randomIntBetween(1, 50);
for (int j = 0; j < nbDocsPerIndex[i]; j++) {
client().prepareIndex("test" + i, "type1").setSource("num", i).get();
}
}
refresh();
logger.debug("--> wait for index stats collector to collect stat for each index");
assertBusy(new Runnable() {
@Override
public void run() {
for (int i = 0; i < nbIndices; i++) {
CountResponse count = client().prepareCount()
.setTypes(IndexStatsCollector.TYPE)
.setQuery(QueryBuilders.termQuery("index_stats.index", "test" + i))
.get();
assertThat(count.getCount(), greaterThan(0L));
}
}
});
logger.debug("--> get the list of filters used by the renderer");
String[] filters = IndexStatsRenderer.FILTERS;
logger.debug("--> checking that every document contains the expected fields");
SearchResponse response = client().prepareSearch().setTypes(IndexStatsCollector.TYPE).get();
for (SearchHit searchHit : response.getHits().getHits()) {
Map<String, Object> fields = searchHit.sourceAsMap();
for (String filter : filters) {
assertContains(filter, fields);
}
}
}
/**
* Checks if a field exist in a map of values. If the field contains a dot like 'foo.bar'
* it checks that 'foo' exists in the map of values and that it points to a sub-map. Then
* it recurses to check if 'bar' exists in the sub-map.
*/
private void assertContains(String field, Map<String, Object> values) {
assertNotNull(field);
assertNotNull(values);
int point = field.indexOf('.');
if (point > -1) {
assertThat(point, allOf(greaterThan(0), lessThan(field.length())));
String segment = field.substring(0, point);
assertTrue(Strings.hasText(segment));
Object value = values.get(segment);
assertNotNull(value);
String next = field.substring(point + 1);
if (next.length() > 0) {
assertTrue(value instanceof Map);
assertContains(next, (Map<String, Object>) value);
} else {
assertFalse(value instanceof Map);
}
} else {
assertNotNull(values.get(field));
}
}
}

14
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererTests.java
View File

@ -8,7 +8,11 @@ package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.index.engine.SegmentsStats;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.refresh.RefreshStats;
import org.elasticsearch.index.search.stats.SearchStats;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMarvelDoc;
@ -32,7 +36,13 @@ public class IndexStatsRendererTests extends ESTestCase {
CommonStats stats = new CommonStats();
stats.docs = new DocsStats(345678L, 123L);
stats.store = new StoreStats(5761573L, 0L);
stats.indexing = new IndexingStats(new IndexingStats.Stats(0L, 0L, 0L, 0L, 0L, 0L, 0L, true, 302L), null);
stats.indexing = new IndexingStats(new IndexingStats.Stats(3L, 71L, 0L, 0L, 0L, 0L, 0L, true, 302L), null);
stats.search = new SearchStats(new SearchStats.Stats(1L, 7L, 0L, 0L, 0L, 0L, 0L, 0L, 0L), 0L, null);
stats.merge = new MergeStats();
stats.merge.add(0L, 0L, 0L, 42L, 0L, 0L, 0L, 0L, 0L, 0L);
stats.refresh = new RefreshStats(0L, 978L);
stats.segments = new SegmentsStats();
stats.segments.add(0, 87965412L);
return stats;
}
@ -40,7 +50,7 @@ public class IndexStatsRendererTests extends ESTestCase {
public CommonStats getPrimaries() {
// Primaries will be filtered out by the renderer
CommonStats stats = new CommonStats();
stats.docs = new DocsStats(randomLong(), randomLong());
stats.docs = new DocsStats(345678L, randomLong());
stats.store = new StoreStats(randomLong(), randomLong());
stats.indexing = new IndexingStats(new IndexingStats.Stats(0L, 0L, 0L, 0L, 0L, 0L, 0L, true, randomLong()), null);
return stats;

22
marvel/src/test/resources/samples/marvel_index_stats.json
View File

@ -12,8 +12,28 @@
"throttle_time_in_millis": 0
},
"indexing": {
"index_total" : 3,
"index_time_in_millis" : 71,
"throttle_time_in_millis": 302
},
"search" : {
"query_total" : 1,
"query_time_in_millis" : 7
},
"merges" : {
"total_size_in_bytes" : 42
},
"refresh" : {
"total_time_in_millis" : 978
},
"segments" : {
"memory_in_bytes" : 87965412
}
},
"primaries" : {
"docs" : {
"count" : 345678
}
}
}
}
}

7
marvel/src/test/resources/samples/marvel_node_stats.json
View File

@ -22,6 +22,9 @@
"count": 0
}
},
"os": {
"load_average" : 0.66
},
"process": {
"open_file_descriptors": 235,
"max_file_descriptors": 4096
@ -48,10 +51,10 @@
"index": {
"rejected": 0
},
"bulk": {
"search": {
"rejected": 0
},
"search": {
"bulk": {
"rejected": 0
}
},