diff --git a/marvel/src/main/java/org/elasticsearch/marvel/MarvelModule.java b/marvel/src/main/java/org/elasticsearch/marvel/MarvelModule.java index 8eb191b2583..5de06d388c3 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/MarvelModule.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/MarvelModule.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.inject.SpawnModules; import org.elasticsearch.marvel.agent.AgentService; import org.elasticsearch.marvel.agent.collector.CollectorModule; import org.elasticsearch.marvel.agent.exporter.ExporterModule; +import org.elasticsearch.marvel.agent.renderer.RendererModule; import org.elasticsearch.marvel.agent.settings.MarvelSettingsModule; import org.elasticsearch.marvel.license.LicenseModule; @@ -24,6 +25,11 @@ public class MarvelModule extends AbstractModule implements SpawnModules { @Override public Iterable spawnModules() { - return ImmutableList.of(new MarvelSettingsModule(), new LicenseModule(), new CollectorModule(), new ExporterModule()); + return ImmutableList.of( + new MarvelSettingsModule(), + new LicenseModule(), + new CollectorModule(), + new ExporterModule(), + new RendererModule()); } } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java index a6104b2219f..442b120efed 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollector.java @@ -28,7 +28,7 @@ import java.util.Collection; public class IndexStatsCollector extends AbstractCollector { public static final String NAME = "index-stats-collector"; - protected static final String TYPE = "marvel_index"; + public static final String TYPE = "marvel_index_stats"; private final ClusterName clusterName; private final Client client; @@ -66,10 +66,6 @@ public class IndexStatsCollector extends AbstractCollector } protected MarvelDoc buildMarvelDoc(String clusterName, String type, long timestamp, IndexStats indexStats) { - return IndexStatsMarvelDoc.createMarvelDoc(clusterName, type, timestamp, - indexStats.getIndex(), - indexStats.getTotal().getDocs().getCount(), - indexStats.getTotal().getStore().sizeInBytes(), indexStats.getTotal().getStore().throttleTime().millis(), - indexStats.getTotal().getIndexing().getTotal().getThrottleTimeInMillis()); + return IndexStatsMarvelDoc.createMarvelDoc(clusterName, type, timestamp, indexStats); } } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDoc.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDoc.java index 9cae619e181..164e5a70e85 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDoc.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDoc.java @@ -5,168 +5,37 @@ */ package org.elasticsearch.marvel.agent.collector.indices; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.marvel.agent.exporter.MarvelDoc; -import java.io.IOException; -import java.util.concurrent.TimeUnit; +public class IndexStatsMarvelDoc extends MarvelDoc { -public class IndexStatsMarvelDoc extends MarvelDoc { + private final Payload payload; - private final String index; - private final Docs docs; - private final Store store; - private final Indexing indexing; - - public IndexStatsMarvelDoc(String clusterName, String type, long timestamp, - String index, Docs docs, Store store, Indexing indexing) { + public IndexStatsMarvelDoc(String clusterName, String type, long timestamp, Payload payload) { super(clusterName, type, timestamp); - this.index = index; - this.docs = docs; - this.store = store; - this.indexing = indexing; + this.payload = payload; } @Override - public IndexStatsMarvelDoc payload() { - return this; + public IndexStatsMarvelDoc.Payload payload() { + return payload; } - public String getIndex() { - return index; + public static IndexStatsMarvelDoc createMarvelDoc(String clusterName, String type, long timestamp, IndexStats indexStats) { + return new IndexStatsMarvelDoc(clusterName, type, timestamp, new Payload(indexStats)); } - public Docs getDocs() { - return docs; - } + public static class Payload { - public Store getStore() { - return store; - } + private final IndexStats indexStats; - public Indexing getIndexing() { - return indexing; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - super.toXContent(builder, params); - builder.field(Fields.INDEX, index); - if (docs != null) { - docs.toXContent(builder, params); - } - if (store != null) { - store.toXContent(builder, params); - } - if (indexing != null) { - indexing.toXContent(builder, params); - } - builder.endObject(); - return builder; - } - - public static IndexStatsMarvelDoc createMarvelDoc(String clusterName, String type, long timestamp, - String index, long docsCount, long storeSizeInBytes, long storeThrottleTimeInMillis, long indexingThrottleTimeInMillis) { - return new IndexStatsMarvelDoc(clusterName, type, timestamp, index, - new Docs(docsCount), - new Store(storeSizeInBytes, storeThrottleTimeInMillis), - new Indexing(indexingThrottleTimeInMillis)); - } - - static final class Fields { - static final XContentBuilderString INDEX = new XContentBuilderString("index"); - } - - static class Docs implements ToXContent { - - private final long count; - - Docs(long count) { - this.count = count; + Payload(IndexStats indexStats) { + this.indexStats = indexStats; } - public long getCount() { - return count; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.DOCS); - builder.field(Fields.COUNT, count); - builder.endObject(); - return builder; - } - - static final class Fields { - static final XContentBuilderString DOCS = new XContentBuilderString("docs"); - static final XContentBuilderString COUNT = new XContentBuilderString("count"); - } - } - - static class Store implements ToXContent { - - private final long sizeInBytes; - private final long throttleTimeInMillis; - - public Store(long sizeInBytes, long throttleTimeInMillis) { - this.sizeInBytes = sizeInBytes; - this.throttleTimeInMillis = throttleTimeInMillis; - } - - public long getSizeInBytes() { - return sizeInBytes; - } - - public long getThrottleTimeInMillis() { - return throttleTimeInMillis; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.STORE); - builder.field(Fields.SIZE_IN_BYTES, sizeInBytes); - builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, new TimeValue(throttleTimeInMillis, TimeUnit.MILLISECONDS)); - builder.endObject(); - return builder; - } - - static final class Fields { - static final XContentBuilderString STORE = new XContentBuilderString("store"); - static final XContentBuilderString SIZE_IN_BYTES = new XContentBuilderString("size_in_bytes"); - static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time"); - static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis"); - } - } - - static class Indexing implements ToXContent { - - private final long throttleTimeInMillis; - - public Indexing(long throttleTimeInMillis) { - this.throttleTimeInMillis = throttleTimeInMillis; - } - - public long getThrottleTimeInMillis() { - return throttleTimeInMillis; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(Fields.INDEXING); - builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, new TimeValue(throttleTimeInMillis, TimeUnit.MILLISECONDS)); - builder.endObject(); - return builder; - } - - static final class Fields { - static final XContentBuilderString INDEXING = new XContentBuilderString("indexing"); - static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time"); - static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis"); + public IndexStats getIndexStats() { + return indexStats; } } } - diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/HttpESExporter.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/HttpESExporter.java index 208a7539f30..0ff002df91c 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/HttpESExporter.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/HttpESExporter.java @@ -17,14 +17,18 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.EsExecutors; -import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.common.xcontent.smile.SmileXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.env.Environment; import org.elasticsearch.http.HttpServer; +import org.elasticsearch.marvel.agent.renderer.Renderer; +import org.elasticsearch.marvel.agent.renderer.RendererRegistry; import org.elasticsearch.marvel.agent.support.AgentUtils; import org.elasticsearch.node.Node; import org.elasticsearch.node.service.NodeService; @@ -81,6 +85,7 @@ public class HttpESExporter extends AbstractExporter implements final ClusterName clusterName; final NodeService nodeService; final Environment environment; + final RendererRegistry registry; HttpServer httpServer; final boolean httpEnabled; @@ -99,7 +104,8 @@ public class HttpESExporter extends AbstractExporter implements public HttpESExporter(Settings settings, ClusterService clusterService, ClusterName clusterName, @ClusterDynamicSettings DynamicSettings dynamicSettings, NodeSettingsService nodeSettingsService, - NodeService nodeService, Environment environment) { + NodeService nodeService, Environment environment, + RendererRegistry registry) { super(settings, NAME, clusterService); this.clusterService = clusterService; @@ -107,6 +113,7 @@ public class HttpESExporter extends AbstractExporter implements this.clusterName = clusterName; this.nodeService = nodeService; this.environment = environment; + this.registry = registry; httpEnabled = settings.getAsBoolean(Node.HTTP_ENABLED, true); @@ -183,24 +190,35 @@ public class HttpESExporter extends AbstractExporter implements return conn; } - private void addMarvelDocToConnection(HttpURLConnection conn, - MarvelDoc marvelDoc) throws IOException { - OutputStream os = conn.getOutputStream(); - // TODO: find a way to disable builder's substream flushing or something neat solution - XContentBuilder builder = XContentFactory.smileBuilder(); - builder.startObject().startObject("index") - .field("_type", marvelDoc.type()) - .endObject().endObject(); - builder.close(); - builder.bytes().writeTo(os); - os.write(SmileXContent.smileXContent.streamSeparator()); + private void render(OutputStream os, MarvelDoc marvelDoc) throws IOException { + final XContentType xContentType = XContentType.SMILE; - builder = XContentFactory.smileBuilder(); - builder.humanReadable(false); - marvelDoc.toXContent(builder, ToXContent.EMPTY_PARAMS); - builder.close(); - builder.bytes().writeTo(os); - os.write(SmileXContent.smileXContent.streamSeparator()); + // Get the appropriate renderer in order to render the MarvelDoc + Renderer renderer = registry.renderer(marvelDoc.type()); + + try (XContentBuilder builder = new XContentBuilder(xContentType.xContent(), os)) { + + // Builds the bulk action metadata line + builder.startObject(); + builder.startObject("index").field("_type", marvelDoc.type()).endObject(); + builder.endObject(); + + // Adds action metadata line bulk separator + renderBulkSeparator(builder); + + // Render the MarvelDoc + renderer.render(marvelDoc,xContentType, os); + + // Adds final bulk separator + renderBulkSeparator(builder); + } + } + + private void renderBulkSeparator(XContentBuilder builder) throws IOException { + // Flush is needed here... + builder.flush(); + //... because the separator is written directly in the builder's stream + builder.stream().write(builder.contentType().xContent().streamSeparator()); } @SuppressWarnings("unchecked") @@ -233,17 +251,31 @@ public class HttpESExporter extends AbstractExporter implements @Override protected void doExport(Collection marvelDocs) throws Exception { - HttpURLConnection conn = openExportingConnection(); - if (conn == null) { + HttpURLConnection connection = openExportingConnection(); + if (connection == null) { return; } - try { - for (MarvelDoc marvelDoc : marvelDocs) { - addMarvelDocToConnection(conn, marvelDoc); + + if ((marvelDocs != null) && (!marvelDocs.isEmpty())) { + OutputStream os = connection.getOutputStream(); + + // We need to use a buffer to render each Marvel document + // because the renderer might close the outputstream (ex: XContentBuilder) + try (BytesStreamOutput buffer = new BytesStreamOutput()) { + for (MarvelDoc marvelDoc : marvelDocs) { + render(buffer, marvelDoc); + + // write the result to the connection + os.write(buffer.bytes().toBytes()); + buffer.reset(); + } + } finally { + try { + sendCloseExportingConnection(connection); + } catch (IOException e) { + logger.error("error sending data to [{}]: {}", AgentUtils.santizeUrlPwds(connection.getURL()), AgentUtils.santizeUrlPwds(ExceptionsHelper.detailedMessage(e))); + } } - sendCloseExportingConnection(conn); - } catch (IOException e) { - logger.error("error sending data to [{}]: {}", AgentUtils.santizeUrlPwds(conn.getURL()), AgentUtils.santizeUrlPwds(ExceptionsHelper.detailedMessage(e))); } } diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java index 27b0c11482e..1374259d691 100644 --- a/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/exporter/MarvelDoc.java @@ -5,15 +5,7 @@ */ package org.elasticsearch.marvel.agent.exporter; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentBuilderString; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; - -import java.io.IOException; - -public abstract class MarvelDoc implements ToXContent { +public abstract class MarvelDoc { private final String clusterName; private final String type; @@ -38,17 +30,4 @@ public abstract class MarvelDoc implements ToXContent { } public abstract T payload(); - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(Fields.CLUSTER_NAME, clusterName()); - DateTime timestampDateTime = new DateTime(timestamp(), DateTimeZone.UTC); - builder.field(Fields.TIMESTAMP, timestampDateTime.toString()); - return builder; - } - - static final class Fields { - static final XContentBuilderString CLUSTER_NAME = new XContentBuilderString("cluster_name"); - static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); - } } \ No newline at end of file diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/AbstractRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/AbstractRenderer.java new file mode 100644 index 00000000000..9c1c75f18ab --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/AbstractRenderer.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer; + +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Renders Marvel documents using a XContentBuilder (potentially filtered) + */ +public abstract class AbstractRenderer implements Renderer { + + private static final String[] DEFAULT_FILTERS = { + Fields.CLUSTER_NAME.underscore().toString(), + Fields.TIMESTAMP.underscore().toString(), + }; + + private final String[] filters; + + public AbstractRenderer(String[] filters, boolean additive) { + if (!additive) { + this.filters = filters; + } else { + if (CollectionUtils.isEmpty(filters)) { + this.filters = DEFAULT_FILTERS; + } else { + Set additions = new HashSet<>(); + Collections.addAll(additions, DEFAULT_FILTERS); + Collections.addAll(additions, filters); + this.filters = additions.toArray(new String[additions.size()]); + } + } + } + + @Override + public void render(T marvelDoc, XContentType xContentType, OutputStream os) throws IOException { + if (marvelDoc != null) { + try (XContentBuilder builder = new XContentBuilder(xContentType.xContent(), os, filters())) { + builder.startObject(); + + // Add fields common to all Marvel documents + builder.field(Fields.CLUSTER_NAME, marvelDoc.clusterName()); + DateTime timestampDateTime = new DateTime(marvelDoc.timestamp(), DateTimeZone.UTC); + builder.field(Fields.TIMESTAMP, timestampDateTime.toString()); + + // Render fields specific to the Marvel document + doRender(marvelDoc, builder, ToXContent.EMPTY_PARAMS); + + builder.endObject(); + } + } + } + + protected abstract void doRender(T marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException; + + /** + * Returns the list of filters used when rendering the document. If null, + * no filtering is applied. + */ + protected String[] filters() { + return filters; + } + + static final class Fields { + static final XContentBuilderString CLUSTER_NAME = new XContentBuilderString("cluster_name"); + static final XContentBuilderString TIMESTAMP = new XContentBuilderString("timestamp"); + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/Renderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/Renderer.java new file mode 100644 index 00000000000..caa49e4eac3 --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/Renderer.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer; + +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Renderers are used to render documents using a given OutputStream. + */ +public interface Renderer { + + void render(T document, XContentType xContentType, OutputStream os) throws IOException; +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java new file mode 100644 index 00000000000..fbc070ee5d1 --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererModule.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer; + +import org.elasticsearch.common.inject.AbstractModule; +import org.elasticsearch.common.inject.multibindings.MapBinder; +import org.elasticsearch.marvel.agent.collector.indices.IndexStatsCollector; +import org.elasticsearch.marvel.agent.renderer.indices.IndexStatsRenderer; + +import java.util.HashMap; +import java.util.Map; + +public class RendererModule extends AbstractModule { + + private Map> renderers = new HashMap<>(); + + public void registerRenderer(String payloadType, Class renderer) { + renderers.put(payloadType, renderer); + } + + @Override + protected void configure() { + MapBinder mbinder = MapBinder.newMapBinder(binder(), String.class, Renderer.class); + + // Bind default renderers + bind(IndexStatsRenderer.class).asEagerSingleton(); + mbinder.addBinding(IndexStatsCollector.TYPE).to(IndexStatsRenderer.class); + + for (Map.Entry> entry : renderers.entrySet()) { + bind(entry.getValue()).asEagerSingleton(); + mbinder.addBinding(entry.getKey()).to(entry.getValue()); + } + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererRegistry.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererRegistry.java new file mode 100644 index 00000000000..5c0108cdc36 --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/RendererRegistry.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer; + +import com.google.common.collect.ImmutableMap; +import org.elasticsearch.common.inject.Inject; + +import java.util.Map; + +public class RendererRegistry { + + private final Map renderers; + + @Inject + public RendererRegistry(Map renderers) { + this.renderers = ImmutableMap.copyOf(renderers); + } + + public Renderer renderer(String type) { + return renderers.get(type); + } +} diff --git a/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRenderer.java b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRenderer.java new file mode 100644 index 00000000000..0414420231e --- /dev/null +++ b/marvel/src/main/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRenderer.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer.indices; + +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMarvelDoc; +import org.elasticsearch.marvel.agent.renderer.AbstractRenderer; + +import java.io.IOException; + +public class IndexStatsRenderer extends AbstractRenderer { + + private static final String[] FILTERS = { + "index_stats.index", + "index_stats.total.docs.count", + "index_stats.total.store.size_in_bytes", + "index_stats.total.store.throttle_time_in_millis", + "index_stats.total.indexing.throttle_time_in_millis", + }; + + public IndexStatsRenderer() { + super(FILTERS, true); + } + + @Override + protected void doRender(IndexStatsMarvelDoc marvelDoc, XContentBuilder builder, ToXContent.Params params) throws IOException { + IndexStatsMarvelDoc.Payload payload = marvelDoc.payload(); + + builder.startObject(Fields.INDEX_STATS); + if (payload != null) { + IndexStats indexStats = payload.getIndexStats(); + if (indexStats != null) { + builder.field(Fields.INDEX, indexStats.getIndex()); + + builder.startObject(Fields.TOTAL); + if (indexStats.getTotal() != null) { + indexStats.getTotal().toXContent(builder, params); + } + builder.endObject(); + } + } + builder.endObject(); + } + + static final class Fields { + static final XContentBuilderString INDEX_STATS = new XContentBuilderString("index_stats"); + static final XContentBuilderString INDEX = new XContentBuilderString("index"); + static final XContentBuilderString TOTAL = new XContentBuilderString("total"); + } +} diff --git a/marvel/src/main/resources/marvel_index_template.json b/marvel/src/main/resources/marvel_index_template.json index 1fa44c9115c..2e6c160d78a 100644 --- a/marvel/src/main/resources/marvel_index_template.json +++ b/marvel/src/main/resources/marvel_index_template.json @@ -1,444 +1,67 @@ { "template": ".marvel*", "settings": { - "number_of_shards": 1, - "number_of_replicas": 1, - "analysis": { - "analyzer": { - "default": { - "type": "standard", - "stopwords": "_none_" - } - } - }, - "mapper.dynamic": true, - "marvel.index_format": 6 + "marvel.index_format": 7, + "marvel.version": "${project.version}", + "index.number_of_shards": 1, + "index.number_of_replicas": 1, + "index.codec": "best_compression", + "index.mapper.dynamic": false }, "mappings": { "_default_": { - "dynamic_templates": [ - { - "string_fields": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "multi_field", - "fields": { - "{name}": { - "type": "string", - "index": "analyzed", - "omit_norms": true - }, - "raw": { - "type": "string", - "index": "not_analyzed", - "ignore_above": 256 - } - } - } - } - } - ] - }, - "node_stats": { + "_all": { + "enabled": false + }, + "date_detection": false, "properties": { - "breakers": { - "properties": { - "fielddata": { - "properties": { - "estimated_size_in_bytes": { - "type": "long" - }, - "tripped": { - "type": "long" - }, - "limit_size_in_bytes": { - "type": "long" - } - } - }, - "request": { - "properties": { - "estimated_size_in_bytes": { - "type": "long" - }, - "tripped": { - "type": "long" - }, - "limit_size_in_bytes": { - "type": "long" - } - } - }, - "parent": { - "properties": { - "estimated_size_in_bytes": { - "type": "long" - }, - "tripped": { - "type": "long" - }, - "limit_size_in_bytes": { - "type": "long" - } - } - } - } + "cluster_name": { + "type": "string", + "index": "not_analyzed" }, - "fs": { - "properties": { - "total": { - "properties": { - "disk_io_op": { - "type": "long" - }, - "disk_reads": { - "type": "long" - }, - "disk_writes": { - "type": "long" - }, - "disk_io_size_in_bytes": { - "type": "long" - }, - "disk_read_size_in_bytes": { - "type": "long" - }, - "disk_write_size_in_bytes": { - "type": "long" - } - } - } - } - }, - "jvm": { - "properties": { - "buffer_pools": { - "properties": { - "direct": { - "properties": { - "used_in_bytes": { - "type": "long" - } - } - }, - "mapped": { - "properties": { - "used_in_bytes": { - "type": "long" - } - } - } - } - }, - "gc": { - "properties": { - "collectors": { - "properties": { - "young": { - "properties": { - "collection_count": { - "type": "long" - }, - "collection_time_in_millis": { - "type": "long" - } - } - }, - "old": { - "properties": { - "collection_count": { - "type": "long" - }, - "collection_time_in_millis": { - "type": "long" - } - } - } - } - } - } - } - } - }, - "indices": { - "properties": { - "indexing": { - "properties": { - "throttle_time_in_millis": { - "type": "long" - } - } - }, - "percolate": { - "properties": { - "total": { - "type": "long" - }, - "time_in_millis": { - "type": "long" - }, - "queries": { - "type": "long" - }, - "memory_size_in_bytes": { - "type": "long" - } - } - }, - "segments": { - "properties": { - "index_writer_memory_in_bytes": { - "type": "long" - }, - "version_map_memory_in_bytes": { - "type": "long" - }, - "index_writer_max_memory_in_bytes": { - "type": "long" - } - } - }, - "query_cache": { - "properties": { - "memory_size_in_bytes": { - "type": "long" - }, - "evictions": { - "type": "long" - }, - "hit_count": { - "type": "long" - }, - "miss_count": { - "type": "long" - } - } - } - } - }, - "os": { - "properties": { - "load_average": { - "properties": { - "1m": { - "type": "float" - }, - "5m": { - "type": "float" - }, - "15m": { - "type": "float" - } - } - } - } - }, - "thread_pool": { - "properties": { - "listener": { - "properties": { - "threads": { - "type": "long" - }, - "rejected": { - "type": "long" - }, - "completed": { - "type": "long" - }, - "queue": { - "type": "long" - }, - "largest": { - "type": "long" - } - } - } - } + "timestamp": { + "type": "date", + "format": "date_time" } } }, - "index_stats": { + "marvel_index_stats": { "properties": { - "index": { - "type": "multi_field", - "fields": { + "index_stats": { + "properties": { "index": { "type": "string", - "norms": { - "enabled": false - } + "index": "not_analyzed" }, - "raw": { - "type": "string", - "index": "not_analyzed", - "norms": { - "enabled": false - }, - "index_options": "docs", - "include_in_all": false, - "ignore_above": 256 - } - } - }, - "total": { - "properties": { - "fielddata": { + "total": { "properties": { - "memory_size_in_bytes": { - "type": "long" - } - } - }, - "indexing": { - "properties": { - "throttle_time_in_millis": { - "type": "long" - } - } - }, - "merges": { - "properties": { - "total_size_in_bytes": { - "type": "long" - } - } - }, - "percolate": { - "properties": { - "total": { - "type": "long" - }, - "time_in_millis": { - "type": "long" - }, - "queries": { - "type": "long" - }, - "memory_size_in_bytes": { - "type": "long" - } - } - }, - "search": { - "properties": { - "query": { + "docs": { "properties": { - "query_total": { + "count": { + "type": "long" + } + } + }, + "store": { + "properties": { + "size_in_bytes": { + "type": "long" + }, + "throttle_time_in_millis": { + "type": "long" + } + } + }, + "indexing": { + "properties": { + "throttle_time_in_millis": { "type": "long" } } } } - }, - "segments": { - "properties": { - "index_writer_memory_in_bytes": { - "type": "long" - }, - "version_map_memory_in_bytes": { - "type": "long" - }, - "index_writer_max_memory_in_bytes": { - "type": "long" - } - } - }, - "query_cache": { - "properties": { - "memory_size_in_bytes": { - "type": "long" - }, - "evictions": { - "type": "long" - }, - "hit_count": { - "type": "long" - }, - "miss_count": { - "type": "long" - } - } } } - }, - "primaries": { - "properties": { - "docs": { - "properties": { - "count": { - "type": "long" - } - } - }, - "indexing": { - "properties": { - "index_total": { - "type": "long" - } - } - } - } - } - } - }, - "cluster_event": {}, - "shard_event": {}, - "indices_stats": { - "properties": { - "primaries": { - "properties": { - "indexing": { - "properties": { - "index_total": { - "type": "long" - } - } - }, - "docs": { - "properties": { - "count": { - "type": "long" - } - } - } - } - }, - "total": { - "properties": { - "search": { - "properties": { - "query_total": { - "type": "long" - } - } - } - } - } - } - }, - "cluster_stats": {}, - "index_event": {}, - "node_event": {}, - "routing_event": {}, - "cluster_state": { - "properties": { - "blocks": { - "type": "object", - "enabled": false - }, - "nodes": { - "type": "object", - "enabled": false - }, - "routing_nodes": { - "type": "object", - "enabled": false - }, - "routing_table": { - "type": "object", - "enabled": false } } } diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java index e8a5d8bc6cf..7d314e87a45 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsCollectorTests.java @@ -55,14 +55,17 @@ public class IndexStatsCollectorTests extends ElasticsearchSingleNodeTest { assertThat(indexStatsMarvelDoc.timestamp(), greaterThan(0L)); assertThat(indexStatsMarvelDoc.type(), equalTo(IndexStatsCollector.TYPE)); - assertThat(indexStatsMarvelDoc.getIndex(), equalTo("test")); - assertNotNull(indexStatsMarvelDoc.getDocs()); - assertThat(indexStatsMarvelDoc.getDocs().getCount(), equalTo((long) nbDocs)); - assertNotNull(indexStatsMarvelDoc.getStore()); - assertThat(indexStatsMarvelDoc.getStore().getSizeInBytes(), greaterThan(0L)); - assertThat(indexStatsMarvelDoc.getStore().getThrottleTimeInMillis(), equalTo(0L)); - assertNotNull(indexStatsMarvelDoc.getIndexing()); - assertThat(indexStatsMarvelDoc.getIndexing().getThrottleTimeInMillis(), equalTo(0L)); + IndexStatsMarvelDoc.Payload payload = indexStatsMarvelDoc.payload(); + assertNotNull(payload); + assertNotNull(payload.getIndexStats()); + + assertThat(payload.getIndexStats().getIndex(), equalTo("test")); + assertThat(payload.getIndexStats().getTotal().getDocs().getCount(), equalTo((long) nbDocs)); + assertNotNull(payload.getIndexStats().getTotal().getStore()); + assertThat(payload.getIndexStats().getTotal().getStore().getSizeInBytes(), greaterThan(0L)); + assertThat(payload.getIndexStats().getTotal().getStore().getThrottleTime().millis(), equalTo(0L)); + assertNotNull(payload.getIndexStats().getTotal().getIndexing()); + assertThat(payload.getIndexStats().getTotal().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0L)); } @Test @@ -97,18 +100,23 @@ public class IndexStatsCollectorTests extends ElasticsearchSingleNodeTest { assertThat(marvelDoc, instanceOf(IndexStatsMarvelDoc.class)); IndexStatsMarvelDoc indexStatsMarvelDoc = (IndexStatsMarvelDoc) marvelDoc; - if (indexStatsMarvelDoc.getIndex().equals("test-" + i)) { + + IndexStatsMarvelDoc.Payload payload = indexStatsMarvelDoc.payload(); + assertNotNull(payload); + assertNotNull(payload.getIndexStats()); + + if (payload.getIndexStats().getIndex().equals("test-" + i)) { assertThat(indexStatsMarvelDoc.clusterName(), equalTo(clusterName)); assertThat(indexStatsMarvelDoc.timestamp(), greaterThan(0L)); assertThat(indexStatsMarvelDoc.type(), equalTo(IndexStatsCollector.TYPE)); - assertNotNull(indexStatsMarvelDoc.getDocs()); - assertThat(indexStatsMarvelDoc.getDocs().getCount(), equalTo((long) docsPerIndex[i])); - assertNotNull(indexStatsMarvelDoc.getStore()); - assertThat(indexStatsMarvelDoc.getStore().getSizeInBytes(), greaterThan(0L)); - assertThat(indexStatsMarvelDoc.getStore().getThrottleTimeInMillis(), equalTo(0L)); - assertNotNull(indexStatsMarvelDoc.getIndexing()); - assertThat(indexStatsMarvelDoc.getIndexing().getThrottleTimeInMillis(), equalTo(0L)); + assertNotNull(payload.getIndexStats().getTotal().getDocs()); + assertThat(payload.getIndexStats().getTotal().getDocs().getCount(), equalTo((long) docsPerIndex[i])); + assertNotNull(payload.getIndexStats().getTotal().getStore()); + assertThat(payload.getIndexStats().getTotal().getStore().getSizeInBytes(), greaterThan(0L)); + assertThat(payload.getIndexStats().getTotal().getStore().getThrottleTime().millis(), equalTo(0L)); + assertNotNull(payload.getIndexStats().getTotal().getIndexing()); + assertThat(payload.getIndexStats().getTotal().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0L)); found = true; } } diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDocTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDocTests.java deleted file mode 100644 index 07739ee0e6f..00000000000 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/collector/indices/IndexStatsMarvelDocTests.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.marvel.agent.collector.indices; - -import org.elasticsearch.test.ElasticsearchTestCase; -import org.junit.Test; - -import static org.hamcrest.Matchers.equalTo; - -public class IndexStatsMarvelDocTests extends ElasticsearchTestCase { - - @Test - public void testCreateMarvelDoc() { - String cluster = randomUnicodeOfLength(10); - String type = randomUnicodeOfLength(10); - long timestamp = randomLong(); - String index = randomUnicodeOfLength(10); - long docsCount = randomLong(); - long storeSize = randomLong(); - long storeThrottle = randomLong(); - long indexingThrottle = randomLong(); - - IndexStatsMarvelDoc marvelDoc = IndexStatsMarvelDoc.createMarvelDoc(cluster, type, timestamp, - index, docsCount, storeSize, storeThrottle, indexingThrottle); - - assertNotNull(marvelDoc); - assertThat(marvelDoc.clusterName(), equalTo(cluster)); - assertThat(marvelDoc.type(), equalTo(type)); - assertThat(marvelDoc.timestamp(), equalTo(timestamp)); - assertThat(marvelDoc.getIndex(), equalTo(index)); - assertNotNull(marvelDoc.getDocs()); - assertThat(marvelDoc.getDocs().getCount(), equalTo(docsCount)); - assertNotNull(marvelDoc.getStore()); - assertThat(marvelDoc.getStore().getSizeInBytes(), equalTo(storeSize)); - assertThat(marvelDoc.getStore().getThrottleTimeInMillis(), equalTo(storeThrottle)); - assertNotNull(marvelDoc.getIndexing()); - assertThat(marvelDoc.getIndexing().getThrottleTimeInMillis(), equalTo(indexingThrottle)); - } -} diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/HttpESExporterTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/HttpESExporterTests.java index 1a5c3fea230..17fa6f20b2f 100644 --- a/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/HttpESExporterTests.java +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/exporter/HttpESExporterTests.java @@ -9,6 +9,7 @@ import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -221,7 +222,7 @@ public class HttpESExporterTests extends ElasticsearchIntegrationTest { private MarvelDoc newRandomMarvelDoc() { return IndexStatsMarvelDoc.createMarvelDoc(internalCluster().getClusterName(), "test_marvelDoc", timeStampGenerator.incrementAndGet(), - "test_index", randomInt(), randomLong(), randomLong(), randomLong()); + new IndexStats("test_index", null)); } private void assertMarvelTemplate() { diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/AbstractRendererTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/AbstractRendererTests.java new file mode 100644 index 00000000000..675eabc5c13 --- /dev/null +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/AbstractRendererTests.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.test.ElasticsearchTestCase; +import org.hamcrest.Matchers; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; + +import static org.hamcrest.Matchers.nullValue; + +@Ignore +public abstract class AbstractRendererTests extends ElasticsearchTestCase { + + @Test + public void testSample() throws IOException { + logger.debug("--> creating the sample document"); + MarvelDoc marvelDoc = newMarvelDoc(); + assertNotNull(marvelDoc); + + logger.debug("--> creating the renderer"); + Renderer renderer = newRenderer(); + assertNotNull(renderer); + + try (BytesStreamOutput os = new BytesStreamOutput()) { + logger.debug("--> rendering the document"); + renderer.render(marvelDoc, XContentType.JSON, os); + + String sample = sampleFilePath(); + assertTrue(Strings.hasText(sample)); + + logger.debug("--> loading expected document from file {}", sample); + String expected = Streams.copyToStringFromClasspath(sample); + + logger.debug("--> comparing both document, they must be identical"); + assertContent(os.bytes(), expected); + } + } + + protected abstract Renderer newRenderer(); + + protected abstract MarvelDoc newMarvelDoc(); + + protected abstract String sampleFilePath(); + + protected void assertContent(BytesReference result, String expected) { + assertNotNull(result); + assertNotNull(expected); + + try { + XContentParser resultParser = XContentFactory.xContent(result).createParser(result); + XContentParser expectedParser = XContentFactory.xContent(expected).createParser(expected); + + while (true) { + XContentParser.Token token1 = resultParser.nextToken(); + XContentParser.Token token2 = expectedParser.nextToken(); + if (token1 == null) { + assertThat(token2, nullValue()); + return; + } + assertThat(token1, Matchers.equalTo(token2)); + switch (token1) { + case FIELD_NAME: + assertThat(resultParser.currentName(), Matchers.equalTo(expectedParser.currentName())); + break; + case VALUE_STRING: + assertThat(resultParser.text(), Matchers.equalTo(expectedParser.text())); + break; + case VALUE_NUMBER: + assertThat(resultParser.numberType(), Matchers.equalTo(expectedParser.numberType())); + assertThat(resultParser.numberValue(), Matchers.equalTo(expectedParser.numberValue())); + break; + } + } + } catch (Exception e) { + fail("Fail to verify the result of the renderer: " + e.getMessage()); + } + } +} diff --git a/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererTests.java b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererTests.java new file mode 100644 index 00000000000..f9757908b1e --- /dev/null +++ b/marvel/src/test/java/org/elasticsearch/marvel/agent/renderer/indices/IndexStatsRendererTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.marvel.agent.renderer.indices; + +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndexStats; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.index.indexing.IndexingStats; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.store.StoreStats; +import org.elasticsearch.marvel.agent.collector.indices.IndexStatsMarvelDoc; +import org.elasticsearch.marvel.agent.exporter.MarvelDoc; +import org.elasticsearch.marvel.agent.renderer.AbstractRendererTests; +import org.elasticsearch.marvel.agent.renderer.Renderer; + +public class IndexStatsRendererTests extends AbstractRendererTests { + + @Override + protected Renderer newRenderer() { + return new IndexStatsRenderer(); + } + + @Override + protected MarvelDoc newMarvelDoc() { + return IndexStatsMarvelDoc.createMarvelDoc("test", "marvel_index_stats", 1437580442979L, + new IndexStats("index-0", new ShardStats[0]) { + @Override + public CommonStats getTotal() { + CommonStats stats = new CommonStats(); + stats.docs = new DocsStats(345678L, 123L); + stats.store = new StoreStats(5761573L, 0L); + stats.indexing = new IndexingStats(new IndexingStats.Stats(0L, 0L, 0L, 0L, 0L, 0L, 0L, true, 302L), null); + return stats; + } + + @Override + public CommonStats getPrimaries() { + // Primaries will be filtered out by the renderer + CommonStats stats = new CommonStats(); + stats.docs = new DocsStats(randomLong(), randomLong()); + stats.store = new StoreStats(randomLong(), randomLong()); + stats.indexing = new IndexingStats(new IndexingStats.Stats(0L, 0L, 0L, 0L, 0L, 0L, 0L, true, randomLong()), null); + return stats; + } + }); + } + + @Override + protected String sampleFilePath() { + return "/samples/index_stats.json"; + } +} diff --git a/marvel/src/test/resources/samples/index_stats.json b/marvel/src/test/resources/samples/index_stats.json new file mode 100644 index 00000000000..c9751a9de18 --- /dev/null +++ b/marvel/src/test/resources/samples/index_stats.json @@ -0,0 +1,19 @@ +{ + "cluster_name": "test", + "timestamp": "2015-07-22T15:54:02.979Z", + "index_stats": { + "index": "index-0", + "total": { + "docs": { + "count": 345678 + }, + "store": { + "size_in_bytes": 5761573, + "throttle_time_in_millis": 0 + }, + "indexing": { + "throttle_time_in_millis": 302 + } + } + } +} \ No newline at end of file