honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Marvel: Refactor IndexStatsCollector to use Renderer/XContentFiltering (aka 'filter_path') feature 

Original commit: elastic/x-pack-elasticsearch@562ffea0b8
This commit is contained in:
Tanguy Leroux 2015-07-23 09:48:04 +02:00
parent bedab83929
commit 63ddbc9adb
17 changed files with 537 additions and 679 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
marvel/src/main/java/org/elasticsearch/marvel/MarvelModule.java
View File

@ -12,6 +12,7 @@ import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.marvel.agent.AgentService;
import org.elasticsearch.marvel.agent.collector.CollectorModule;
import org.elasticsearch.marvel.agent.exporter.ExporterModule;
import org.elasticsearch.marvel.agent.renderer.RendererModule;
import org.elasticsearch.marvel.agent.settings.MarvelSettingsModule;
import org.elasticsearch.marvel.license.LicenseModule;
@ -24,6 +25,11 @@ public class MarvelModule extends AbstractModule implements SpawnModules {
@Override
public Iterable<? extends Module> spawnModules() {
return ImmutableList.of(new MarvelSettingsModule(), new LicenseModule(), new CollectorModule(), new ExporterModule());
return ImmutableList.of(
new MarvelSettingsModule(),
new LicenseModule(),
new CollectorModule(),
new ExporterModule(),
new RendererModule());
}
}

8
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java
View File

@ -28,7 +28,7 @@ import java.util.Collection;
public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector> {
public static final String NAME = "index-stats-collector";
protected static final String TYPE = "marvel_index";
public static final String TYPE = "marvel_index_stats";
private final ClusterName clusterName;
private final Client client;
@ -66,10 +66,6 @@ public class IndexStatsCollector extends AbstractCollector<IndexStatsCollector>
}
protected MarvelDoc buildMarvelDoc(String clusterName, String type, long timestamp, IndexStats indexStats) {
return IndexStatsMarvelDoc.createMarvelDoc(clusterName, type, timestamp,
indexStats.getIndex(),
indexStats.getTotal().getDocs().getCount(),
indexStats.getTotal().getStore().sizeInBytes(), indexStats.getTotal().getStore().throttleTime().millis(),
indexStats.getTotal().getIndexing().getTotal().getThrottleTimeInMillis());
return IndexStatsMarvelDoc.createMarvelDoc(clusterName, type, timestamp, indexStats);
}
}

165
marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDoc.java
View File

@ -5,168 +5,37 @@
*/
package org.elasticsearch.marvel.agent.collector.indices;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
public class IndexStatsMarvelDoc extends MarvelDoc<IndexStatsMarvelDoc.Payload> {
public class IndexStatsMarvelDoc extends MarvelDoc<IndexStatsMarvelDoc> {
private final Payload payload;
private final String index;
private final Docs docs;
private final Store store;
private final Indexing indexing;
public IndexStatsMarvelDoc(String clusterName, String type, long timestamp,
String index, Docs docs, Store store, Indexing indexing) {
public IndexStatsMarvelDoc(String clusterName, String type, long timestamp, Payload payload) {
super(clusterName, type, timestamp);
this.index = index;
this.docs = docs;
this.store = store;
this.indexing = indexing;
this.payload = payload;
}
@Override
public IndexStatsMarvelDoc payload() {
return this;
public IndexStatsMarvelDoc.Payload payload() {
return payload;
}
public String getIndex() {
return index;
public static IndexStatsMarvelDoc createMarvelDoc(String clusterName, String type, long timestamp, IndexStats indexStats) {
return new IndexStatsMarvelDoc(clusterName, type, timestamp, new Payload(indexStats));
}
public Docs getDocs() {
return docs;
public static class Payload {
private final IndexStats indexStats;
Payload(IndexStats indexStats) {
this.indexStats = indexStats;
}
public Store getStore() {
return store;
}
public Indexing getIndexing() {
return indexing;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
super.toXContent(builder, params);
builder.field(Fields.INDEX, index);
if (docs != null) {
docs.toXContent(builder, params);
}
if (store != null) {
store.toXContent(builder, params);
}
if (indexing != null) {
indexing.toXContent(builder, params);
}
builder.endObject();
return builder;
}
public static IndexStatsMarvelDoc createMarvelDoc(String clusterName, String type, long timestamp,
String index, long docsCount, long storeSizeInBytes, long storeThrottleTimeInMillis, long indexingThrottleTimeInMillis) {
return new IndexStatsMarvelDoc(clusterName, type, timestamp, index,
new Docs(docsCount),
new Store(storeSizeInBytes, storeThrottleTimeInMillis),
new Indexing(indexingThrottleTimeInMillis));
}
static final class Fields {
static final XContentBuilderString INDEX = new XContentBuilderString("index");
}
static class Docs implements ToXContent {
private final long count;
Docs(long count) {
this.count = count;
}
public long getCount() {
return count;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.DOCS);
builder.field(Fields.COUNT, count);
builder.endObject();
return builder;
}
static final class Fields {
static final XContentBuilderString DOCS = new XContentBuilderString("docs");
static final XContentBuilderString COUNT = new XContentBuilderString("count");
}
}
static class Store implements ToXContent {
private final long sizeInBytes;
private final long throttleTimeInMillis;
public Store(long sizeInBytes, long throttleTimeInMillis) {
this.sizeInBytes = sizeInBytes;
this.throttleTimeInMillis = throttleTimeInMillis;
}
public long getSizeInBytes() {
return sizeInBytes;
}
public long getThrottleTimeInMillis() {
return throttleTimeInMillis;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.STORE);
builder.field(Fields.SIZE_IN_BYTES, sizeInBytes);
builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, new TimeValue(throttleTimeInMillis, TimeUnit.MILLISECONDS));
builder.endObject();
return builder;
}
static final class Fields {
static final XContentBuilderString STORE = new XContentBuilderString("store");
static final XContentBuilderString SIZE_IN_BYTES = new XContentBuilderString("size_in_bytes");
static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time");
static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis");
}
}
static class Indexing implements ToXContent {
private final long throttleTimeInMillis;
public Indexing(long throttleTimeInMillis) {
this.throttleTimeInMillis = throttleTimeInMillis;
}
public long getThrottleTimeInMillis() {
return throttleTimeInMillis;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.INDEXING);
builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, new TimeValue(throttleTimeInMillis, TimeUnit.MILLISECONDS));
builder.endObject();
return builder;
}
static final class Fields {
static final XContentBuilderString INDEXING = new XContentBuilderString("indexing");
static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time");
static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis");
public IndexStats getIndexStats() {
return indexStats;
}
}
}

84
marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/HttpESExporter.java
View File

@ -17,14 +17,18 @@ import org.elasticsearch.common.Strings;
import org.elasticsearch.common.component.Lifecycle;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.BoundTransportAddress;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.concurrent.EsExecutors;
import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.common.xcontent.smile.SmileXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.env.Environment;
import org.elasticsearch.http.HttpServer;
import org.elasticsearch.marvel.agent.renderer.Renderer;
import org.elasticsearch.marvel.agent.renderer.RendererRegistry;
import org.elasticsearch.marvel.agent.support.AgentUtils;
import org.elasticsearch.node.Node;
import org.elasticsearch.node.service.NodeService;
@ -81,6 +85,7 @@ public class HttpESExporter extends AbstractExporter<HttpESExporter> implements
final ClusterName clusterName;
final NodeService nodeService;
final Environment environment;
final RendererRegistry registry;
HttpServer httpServer;
final boolean httpEnabled;
@ -99,7 +104,8 @@ public class HttpESExporter extends AbstractExporter<HttpESExporter> implements
public HttpESExporter(Settings settings, ClusterService clusterService, ClusterName clusterName,
@ClusterDynamicSettings DynamicSettings dynamicSettings,
NodeSettingsService nodeSettingsService,
NodeService nodeService, Environment environment) {
NodeService nodeService, Environment environment,
RendererRegistry registry) {
super(settings, NAME, clusterService);
this.clusterService = clusterService;
@ -107,6 +113,7 @@ public class HttpESExporter extends AbstractExporter<HttpESExporter> implements
this.clusterName = clusterName;
this.nodeService = nodeService;
this.environment = environment;
this.registry = registry;
httpEnabled = settings.getAsBoolean(Node.HTTP_ENABLED, true);
@ -183,24 +190,35 @@ public class HttpESExporter extends AbstractExporter<HttpESExporter> implements
return conn;
}
private void addMarvelDocToConnection(HttpURLConnection conn,
MarvelDoc marvelDoc) throws IOException {
OutputStream os = conn.getOutputStream();
// TODO: find a way to disable builder's substream flushing or something neat solution
XContentBuilder builder = XContentFactory.smileBuilder();
builder.startObject().startObject("index")
.field("_type", marvelDoc.type())
.endObject().endObject();
builder.close();
builder.bytes().writeTo(os);
os.write(SmileXContent.smileXContent.streamSeparator());
private void render(OutputStream os, MarvelDoc marvelDoc) throws IOException {
final XContentType xContentType = XContentType.SMILE;
builder = XContentFactory.smileBuilder();
builder.humanReadable(false);
marvelDoc.toXContent(builder, ToXContent.EMPTY_PARAMS);
builder.close();
builder.bytes().writeTo(os);
os.write(SmileXContent.smileXContent.streamSeparator());
// Get the appropriate renderer in order to render the MarvelDoc
Renderer renderer = registry.renderer(marvelDoc.type());
try (XContentBuilder builder = new XContentBuilder(xContentType.xContent(), os)) {
// Builds the bulk action metadata line
builder.startObject();
builder.startObject("index").field("_type", marvelDoc.type()).endObject();
builder.endObject();
// Adds action metadata line bulk separator
renderBulkSeparator(builder);
// Render the MarvelDoc
renderer.render(marvelDoc,xContentType, os);
// Adds final bulk separator
renderBulkSeparator(builder);
}
}
private void renderBulkSeparator(XContentBuilder builder) throws IOException {
// Flush is needed here...
builder.flush();
//... because the separator is written directly in the builder's stream
builder.stream().write(builder.contentType().xContent().streamSeparator());
}
@SuppressWarnings("unchecked")
@ -233,17 +251,31 @@ public class HttpESExporter extends AbstractExporter<HttpESExporter> implements
@Override
protected void doExport(Collection<MarvelDoc> marvelDocs) throws Exception {
HttpURLConnection conn = openExportingConnection();
if (conn == null) {
HttpURLConnection connection = openExportingConnection();
if (connection == null) {
return;
}
try {
if ((marvelDocs != null) && (!marvelDocs.isEmpty())) {
OutputStream os = connection.getOutputStream();
// We need to use a buffer to render each Marvel document
// because the renderer might close the outputstream (ex: XContentBuilder)
try (BytesStreamOutput buffer = new BytesStreamOutput()) {
for (MarvelDoc marvelDoc : marvelDocs) {
addMarvelDocToConnection(conn, marvelDoc);
render(buffer, marvelDoc);
// write the result to the connection
os.write(buffer.bytes().toBytes());
buffer.reset();
}
sendCloseExportingConnection(conn);
} finally {
try {
sendCloseExportingConnection(connection);
} catch (IOException e) {
logger.error("error sending data to [{}]: {}", AgentUtils.santizeUrlPwds(conn.getURL()), AgentUtils.santizeUrlPwds(ExceptionsHelper.detailedMessage(e)));
logger.error("error sending data to [{}]: {}", AgentUtils.santizeUrlPwds(connection.getURL()), AgentUtils.santizeUrlPwds(ExceptionsHelper.detailedMessage(e)));
}
}
}
}

23
marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java
View File

@ -5,15 +5,7 @@
*/
package org.elasticsearch.marvel.agent.exporter;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
public abstract class MarvelDoc<T> implements ToXContent {
public abstract class MarvelDoc<T> {
private final String clusterName;
private final String type;
@ -38,17 +30,4 @@ public abstract class MarvelDoc<T> implements ToXContent {
}
public abstract T payload();
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.field(Fields.CLUSTER_NAME, clusterName());
DateTime timestampDateTime = new DateTime(timestamp(), DateTimeZone.UTC);
builder.field(Fields.TIMESTAMP, timestampDateTime.toString());
return builder;
}
static final class Fields {
static final XContentBuilderString CLUSTER_NAME = new XContentBuilderString("cluster_name");
static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp");
}
}

83
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/AbstractRenderer.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/**
* Renders Marvel documents using a XContentBuilder (potentially filtered)
*/
public abstract class AbstractRenderer<T extends MarvelDoc> implements Renderer<T> {
private static final String[] DEFAULT_FILTERS = {
Fields.CLUSTER_NAME.underscore().toString(),
Fields.TIMESTAMP.underscore().toString(),
};
private final String[] filters;
public AbstractRenderer(String[] filters, boolean additive) {
if (!additive) {
this.filters = filters;
} else {
if (CollectionUtils.isEmpty(filters)) {
this.filters = DEFAULT_FILTERS;
} else {
Set<String> additions = new HashSet<>();
Collections.addAll(additions, DEFAULT_FILTERS);
Collections.addAll(additions, filters);
this.filters = additions.toArray(new String[additions.size()]);
}
}
}
@Override
public void render(T marvelDoc, XContentType xContentType, OutputStream os) throws IOException {
if (marvelDoc != null) {
try (XContentBuilder builder = new XContentBuilder(xContentType.xContent(), os, filters())) {
builder.startObject();
// Add fields common to all Marvel documents
builder.field(Fields.CLUSTER_NAME, marvelDoc.clusterName());
DateTime timestampDateTime = new DateTime(marvelDoc.timestamp(), DateTimeZone.UTC);
builder.field(Fields.TIMESTAMP, timestampDateTime.toString());
// Render fields specific to the Marvel document
doRender(marvelDoc, builder, ToXContent.EMPTY_PARAMS);
builder.endObject();
}
}
}
protected abstract void doRender(T marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException;
/**
* Returns the list of filters used when rendering the document. If null,
* no filtering is applied.
*/
protected String[] filters() {
return filters;
}
static final class Fields {
static final XContentBuilderString CLUSTER_NAME = new XContentBuilderString("cluster_name");
static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp");
}
}

19
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/Renderer.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.IOException;
import java.io.OutputStream;
/**
* Renderers are used to render documents using a given OutputStream.
*/
public interface Renderer<T> {
void render(T document, XContentType xContentType, OutputStream os) throws IOException;
}

37
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.multibindings.MapBinder;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector;
import org.elasticsearch.marvel.agent.renderer.indices.IndexStatsRenderer;
import java.util.HashMap;
import java.util.Map;
public class RendererModule extends AbstractModule {
private Map<String, Class<? extends Renderer>> renderers = new HashMap<>();
public void registerRenderer(String payloadType, Class<? extends Renderer> renderer) {
renderers.put(payloadType, renderer);
}
@Override
protected void configure() {
MapBinder<String, Renderer> mbinder = MapBinder.newMapBinder(binder(), String.class, Renderer.class);
// Bind default renderers
bind(IndexStatsRenderer.class).asEagerSingleton();
mbinder.addBinding(IndexStatsCollector.TYPE).to(IndexStatsRenderer.class);
for (Map.Entry<String, Class<? extends Renderer>> entry : renderers.entrySet()) {
bind(entry.getValue()).asEagerSingleton();
mbinder.addBinding(entry.getKey()).to(entry.getValue());
}
}
}

25
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererRegistry.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.common.inject.Inject;
import java.util.Map;
public class RendererRegistry {
private final Map<String, Renderer> renderers;
@Inject
public RendererRegistry(Map<String, Renderer> renderers) {
this.renderers = ImmutableMap.copyOf(renderers);
}
public Renderer renderer(String type) {
return renderers.get(type);
}
}

56
marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRenderer.java Normal file
View File

@ -0,0 +1,56 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMarvelDoc;
import org.elasticsearch.marvel.agent.renderer.AbstractRenderer;
import java.io.IOException;
public class IndexStatsRenderer extends AbstractRenderer<IndexStatsMarvelDoc> {
private static final String[] FILTERS = {
"index_stats.index",
"index_stats.total.docs.count",
"index_stats.total.store.size_in_bytes",
"index_stats.total.store.throttle_time_in_millis",
"index_stats.total.indexing.throttle_time_in_millis",
};
public IndexStatsRenderer() {
super(FILTERS, true);
}
@Override
protected void doRender(IndexStatsMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException {
IndexStatsMarvelDoc.Payload payload = marvelDoc.payload();
builder.startObject(Fields.INDEX_STATS);
if (payload != null) {
IndexStats indexStats = payload.getIndexStats();
if (indexStats != null) {
builder.field(Fields.INDEX, indexStats.getIndex());
builder.startObject(Fields.TOTAL);
if (indexStats.getTotal() != null) {
indexStats.getTotal().toXContent(builder, params);
}
builder.endObject();
}
}
builder.endObject();
}
static final class Fields {
static final XContentBuilderString INDEX_STATS = new XContentBuilderString("index_stats");
static final XContentBuilderString INDEX = new XContentBuilderString("index");
static final XContentBuilderString TOTAL = new XContentBuilderString("total");
}
}

437
marvel/src/main/resources/marvel_index_template.json
View File

@ -1,292 +1,53 @@
{
"template": ".marvel*",
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1,
"analysis": {
"analyzer": {
"default": {
"type": "standard",
"stopwords": "_none_"
}
}
},
"mapper.dynamic": true,
"marvel.index_format": 6
"marvel.index_format": 7,
"marvel.version": "${project.version}",
"index.number_of_shards": 1,
"index.number_of_replicas": 1,
"index.codec": "best_compression",
"index.mapper.dynamic": false
},
"mappings": {
"_default_": {
"dynamic_templates": [
{
"string_fields": {
"match": "*",
"match_mapping_type": "string",
"mapping": {
"type": "multi_field",
"fields": {
"{name}": {
"_all": {
"enabled": false
},
"date_detection": false,
"properties": {
"cluster_name": {
"type": "string",
"index": "analyzed",
"omit_norms": true
"index": "not_analyzed"
},
"raw": {
"type": "string",
"index": "not_analyzed",
"ignore_above": 256
"timestamp": {
"type": "date",
"format": "date_time"
}
}
}
}
}
]
},
"node_stats": {
"marvel_index_stats": {
"properties": {
"breakers": {
"properties": {
"fielddata": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
},
"request": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
},
"parent": {
"properties": {
"estimated_size_in_bytes": {
"type": "long"
},
"tripped": {
"type": "long"
},
"limit_size_in_bytes": {
"type": "long"
}
}
}
}
},
"fs": {
"properties": {
"total": {
"properties": {
"disk_io_op": {
"type": "long"
},
"disk_reads": {
"type": "long"
},
"disk_writes": {
"type": "long"
},
"disk_io_size_in_bytes": {
"type": "long"
},
"disk_read_size_in_bytes": {
"type": "long"
},
"disk_write_size_in_bytes": {
"type": "long"
}
}
}
}
},
"jvm": {
"properties": {
"buffer_pools": {
"properties": {
"direct": {
"properties": {
"used_in_bytes": {
"type": "long"
}
}
},
"mapped": {
"properties": {
"used_in_bytes": {
"type": "long"
}
}
}
}
},
"gc": {
"properties": {
"collectors": {
"properties": {
"young": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
},
"old": {
"properties": {
"collection_count": {
"type": "long"
},
"collection_time_in_millis": {
"type": "long"
}
}
}
}
}
}
}
}
},
"indices": {
"properties": {
"indexing": {
"properties": {
"throttle_time_in_millis": {
"type": "long"
}
}
},
"percolate": {
"properties": {
"total": {
"type": "long"
},
"time_in_millis": {
"type": "long"
},
"queries": {
"type": "long"
},
"memory_size_in_bytes": {
"type": "long"
}
}
},
"segments": {
"properties": {
"index_writer_memory_in_bytes": {
"type": "long"
},
"version_map_memory_in_bytes": {
"type": "long"
},
"index_writer_max_memory_in_bytes": {
"type": "long"
}
}
},
"query_cache": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
},
"evictions": {
"type": "long"
},
"hit_count": {
"type": "long"
},
"miss_count": {
"type": "long"
}
}
}
}
},
"os": {
"properties": {
"load_average": {
"properties": {
"1m": {
"type": "float"
},
"5m": {
"type": "float"
},
"15m": {
"type": "float"
}
}
}
}
},
"thread_pool": {
"properties": {
"listener": {
"properties": {
"threads": {
"type": "long"
},
"rejected": {
"type": "long"
},
"completed": {
"type": "long"
},
"queue": {
"type": "long"
},
"largest": {
"type": "long"
}
}
}
}
}
}
},
"index_stats": {
"properties": {
"index": {
"type": "multi_field",
"fields": {
"index": {
"type": "string",
"norms": {
"enabled": false
}
},
"raw": {
"type": "string",
"index": "not_analyzed",
"norms": {
"enabled": false
},
"index_options": "docs",
"include_in_all": false,
"ignore_above": 256
}
}
"index": "not_analyzed"
},
"total": {
"properties": {
"fielddata": {
"docs": {
"properties": {
"memory_size_in_bytes": {
"count": {
"type": "long"
}
}
},
"store": {
"properties": {
"size_in_bytes": {
"type": "long"
},
"throttle_time_in_millis": {
"type": "long"
}
}
@ -297,148 +58,10 @@
"type": "long"
}
}
},
"merges": {
"properties": {
"total_size_in_bytes": {
"type": "long"
}
}
},
"percolate": {
"properties": {
"total": {
"type": "long"
},
"time_in_millis": {
"type": "long"
},
"queries": {
"type": "long"
},
"memory_size_in_bytes": {
"type": "long"
}
}
},
"search": {
"properties": {
"query": {
"properties": {
"query_total": {
"type": "long"
}
}
}
}
},
"segments": {
"properties": {
"index_writer_memory_in_bytes": {
"type": "long"
},
"version_map_memory_in_bytes": {
"type": "long"
},
"index_writer_max_memory_in_bytes": {
"type": "long"
}
}
},
"query_cache": {
"properties": {
"memory_size_in_bytes": {
"type": "long"
},
"evictions": {
"type": "long"
},
"hit_count": {
"type": "long"
},
"miss_count": {
"type": "long"
}
}
}
}
},
"primaries": {
"properties": {
"docs": {
"properties": {
"count": {
"type": "long"
}
}
},
"indexing": {
"properties": {
"index_total": {
"type": "long"
}
}
}
}
}
}
},
"cluster_event": {},
"shard_event": {},
"indices_stats": {
"properties": {
"primaries": {
"properties": {
"indexing": {
"properties": {
"index_total": {
"type": "long"
}
}
},
"docs": {
"properties": {
"count": {
"type": "long"
}
}
}
}
},
"total": {
"properties": {
"search": {
"properties": {
"query_total": {
"type": "long"
}
}
}
}
}
}
},
"cluster_stats": {},
"index_event": {},
"node_event": {},
"routing_event": {},
"cluster_state": {
"properties": {
"blocks": {
"type": "object",
"enabled": false
},
"nodes": {
"type": "object",
"enabled": false
},
"routing_nodes": {
"type": "object",
"enabled": false
},
"routing_table": {
"type": "object",
"enabled": false
}
}
}

40
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java
View File

@ -55,14 +55,17 @@ public class IndexStatsCollectorTests extends ElasticsearchSingleNodeTest {
assertThat(indexStatsMarvelDoc.timestamp(), greaterThan(0L));
assertThat(indexStatsMarvelDoc.type(), equalTo(IndexStatsCollector.TYPE));
assertThat(indexStatsMarvelDoc.getIndex(), equalTo("test"));
assertNotNull(indexStatsMarvelDoc.getDocs());
assertThat(indexStatsMarvelDoc.getDocs().getCount(), equalTo((long) nbDocs));
assertNotNull(indexStatsMarvelDoc.getStore());
assertThat(indexStatsMarvelDoc.getStore().getSizeInBytes(), greaterThan(0L));
assertThat(indexStatsMarvelDoc.getStore().getThrottleTimeInMillis(), equalTo(0L));
assertNotNull(indexStatsMarvelDoc.getIndexing());
assertThat(indexStatsMarvelDoc.getIndexing().getThrottleTimeInMillis(), equalTo(0L));
IndexStatsMarvelDoc.Payload payload = indexStatsMarvelDoc.payload();
assertNotNull(payload);
assertNotNull(payload.getIndexStats());
assertThat(payload.getIndexStats().getIndex(), equalTo("test"));
assertThat(payload.getIndexStats().getTotal().getDocs().getCount(), equalTo((long) nbDocs));
assertNotNull(payload.getIndexStats().getTotal().getStore());
assertThat(payload.getIndexStats().getTotal().getStore().getSizeInBytes(), greaterThan(0L));
assertThat(payload.getIndexStats().getTotal().getStore().getThrottleTime().millis(), equalTo(0L));
assertNotNull(payload.getIndexStats().getTotal().getIndexing());
assertThat(payload.getIndexStats().getTotal().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0L));
}
@Test
@ -97,18 +100,23 @@ public class IndexStatsCollectorTests extends ElasticsearchSingleNodeTest {
assertThat(marvelDoc, instanceOf(IndexStatsMarvelDoc.class));
IndexStatsMarvelDoc indexStatsMarvelDoc = (IndexStatsMarvelDoc) marvelDoc;
if (indexStatsMarvelDoc.getIndex().equals("test-" + i)) {
IndexStatsMarvelDoc.Payload payload = indexStatsMarvelDoc.payload();
assertNotNull(payload);
assertNotNull(payload.getIndexStats());
if (payload.getIndexStats().getIndex().equals("test-" + i)) {
assertThat(indexStatsMarvelDoc.clusterName(), equalTo(clusterName));
assertThat(indexStatsMarvelDoc.timestamp(), greaterThan(0L));
assertThat(indexStatsMarvelDoc.type(), equalTo(IndexStatsCollector.TYPE));
assertNotNull(indexStatsMarvelDoc.getDocs());
assertThat(indexStatsMarvelDoc.getDocs().getCount(), equalTo((long) docsPerIndex[i]));
assertNotNull(indexStatsMarvelDoc.getStore());
assertThat(indexStatsMarvelDoc.getStore().getSizeInBytes(), greaterThan(0L));
assertThat(indexStatsMarvelDoc.getStore().getThrottleTimeInMillis(), equalTo(0L));
assertNotNull(indexStatsMarvelDoc.getIndexing());
assertThat(indexStatsMarvelDoc.getIndexing().getThrottleTimeInMillis(), equalTo(0L));
assertNotNull(payload.getIndexStats().getTotal().getDocs());
assertThat(payload.getIndexStats().getTotal().getDocs().getCount(), equalTo((long) docsPerIndex[i]));
assertNotNull(payload.getIndexStats().getTotal().getStore());
assertThat(payload.getIndexStats().getTotal().getStore().getSizeInBytes(), greaterThan(0L));
assertThat(payload.getIndexStats().getTotal().getStore().getThrottleTime().millis(), equalTo(0L));
assertNotNull(payload.getIndexStats().getTotal().getIndexing());
assertThat(payload.getIndexStats().getTotal().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0L));
found = true;
}
}

42
marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDocTests.java
View File

@ -1,42 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.collector.indices;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import static org.hamcrest.Matchers.equalTo;
public class IndexStatsMarvelDocTests extends ElasticsearchTestCase {
@Test
public void testCreateMarvelDoc() {
String cluster = randomUnicodeOfLength(10);
String type = randomUnicodeOfLength(10);
long timestamp = randomLong();
String index = randomUnicodeOfLength(10);
long docsCount = randomLong();
long storeSize = randomLong();
long storeThrottle = randomLong();
long indexingThrottle = randomLong();
IndexStatsMarvelDoc marvelDoc = IndexStatsMarvelDoc.createMarvelDoc(cluster, type, timestamp,
index, docsCount, storeSize, storeThrottle, indexingThrottle);
assertNotNull(marvelDoc);
assertThat(marvelDoc.clusterName(), equalTo(cluster));
assertThat(marvelDoc.type(), equalTo(type));
assertThat(marvelDoc.timestamp(), equalTo(timestamp));
assertThat(marvelDoc.getIndex(), equalTo(index));
assertNotNull(marvelDoc.getDocs());
assertThat(marvelDoc.getDocs().getCount(), equalTo(docsCount));
assertNotNull(marvelDoc.getStore());
assertThat(marvelDoc.getStore().getSizeInBytes(), equalTo(storeSize));
assertThat(marvelDoc.getStore().getThrottleTimeInMillis(), equalTo(storeThrottle));
assertNotNull(marvelDoc.getIndexing());
assertThat(marvelDoc.getIndexing().getThrottleTimeInMillis(), equalTo(indexingThrottle));
}
}

3
marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/HttpESExporterTests.java
View File

@ -9,6 +9,7 @@ import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
@ -221,7 +222,7 @@ public class HttpESExporterTests extends ElasticsearchIntegrationTest {
private MarvelDoc newRandomMarvelDoc() {
return IndexStatsMarvelDoc.createMarvelDoc(internalCluster().getClusterName(), "test_marvelDoc", timeStampGenerator.incrementAndGet(),
"test_index", randomInt(), randomLong(), randomLong(), randomLong());
new IndexStats("test_index", null));
}
private void assertMarvelTemplate() {

92
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/AbstractRendererTests.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.hamcrest.Matchers;
import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
import static org.hamcrest.Matchers.nullValue;
@Ignore
public abstract class AbstractRendererTests extends ElasticsearchTestCase {
@Test
public void testSample() throws IOException {
logger.debug("--> creating the sample document");
MarvelDoc marvelDoc = newMarvelDoc();
assertNotNull(marvelDoc);
logger.debug("--> creating the renderer");
Renderer renderer = newRenderer();
assertNotNull(renderer);
try (BytesStreamOutput os = new BytesStreamOutput()) {
logger.debug("--> rendering the document");
renderer.render(marvelDoc, XContentType.JSON, os);
String sample = sampleFilePath();
assertTrue(Strings.hasText(sample));
logger.debug("--> loading expected document from file {}", sample);
String expected = Streams.copyToStringFromClasspath(sample);
logger.debug("--> comparing both document, they must be identical");
assertContent(os.bytes(), expected);
}
}
protected abstract Renderer newRenderer();
protected abstract MarvelDoc newMarvelDoc();
protected abstract String sampleFilePath();
protected void assertContent(BytesReference result, String expected) {
assertNotNull(result);
assertNotNull(expected);
try {
XContentParser resultParser = XContentFactory.xContent(result).createParser(result);
XContentParser expectedParser = XContentFactory.xContent(expected).createParser(expected);
while (true) {
XContentParser.Token token1 = resultParser.nextToken();
XContentParser.Token token2 = expectedParser.nextToken();
if (token1 == null) {
assertThat(token2, nullValue());
return;
}
assertThat(token1, Matchers.equalTo(token2));
switch (token1) {
case FIELD_NAME:
assertThat(resultParser.currentName(), Matchers.equalTo(expectedParser.currentName()));
break;
case VALUE_STRING:
assertThat(resultParser.text(), Matchers.equalTo(expectedParser.text()));
break;
case VALUE_NUMBER:
assertThat(resultParser.numberType(), Matchers.equalTo(expectedParser.numberType()));
assertThat(resultParser.numberValue(), Matchers.equalTo(expectedParser.numberValue()));
break;
}
}
} catch (Exception e) {
fail("Fail to verify the result of the renderer: " + e.getMessage());
}
}
}

55
marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererTests.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.marvel.agent.renderer.indices;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndexStats;
import org.elasticsearch.action.admin.indices.stats.ShardStats;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.index.store.StoreStats;
import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMarvelDoc;
import org.elasticsearch.marvel.agent.exporter.MarvelDoc;
import org.elasticsearch.marvel.agent.renderer.AbstractRendererTests;
import org.elasticsearch.marvel.agent.renderer.Renderer;
public class IndexStatsRendererTests extends AbstractRendererTests {
@Override
protected Renderer newRenderer() {
return new IndexStatsRenderer();
}
@Override
protected MarvelDoc newMarvelDoc() {
return IndexStatsMarvelDoc.createMarvelDoc("test", "marvel_index_stats", 1437580442979L,
new IndexStats("index-0", new ShardStats[0]) {
@Override
public CommonStats getTotal() {
CommonStats stats = new CommonStats();
stats.docs = new DocsStats(345678L, 123L);
stats.store = new StoreStats(5761573L, 0L);
stats.indexing = new IndexingStats(new IndexingStats.Stats(0L, 0L, 0L, 0L, 0L, 0L, 0L, true, 302L), null);
return stats;
}
@Override
public CommonStats getPrimaries() {
// Primaries will be filtered out by the renderer
CommonStats stats = new CommonStats();
stats.docs = new DocsStats(randomLong(), randomLong());
stats.store = new StoreStats(randomLong(), randomLong());
stats.indexing = new IndexingStats(new IndexingStats.Stats(0L, 0L, 0L, 0L, 0L, 0L, 0L, true, randomLong()), null);
return stats;
}
});
}
@Override
protected String sampleFilePath() {
return "/samples/index_stats.json";
}
}

19
marvel/src/test/resources/samples/index_stats.json Normal file
View File

@ -0,0 +1,19 @@
{
"cluster_name": "test",
"timestamp": "2015-07-22T15:54:02.979Z",
"index_stats": {
"index": "index-0",
"total": {
"docs": {
"count": 345678
},
"store": {
"size_in_bytes": 5761573,
"throttle_time_in_millis": 0
},
"indexing": {
"throttle_time_in_millis": 302
}
}
}
}