From 268e561287d188c430f88a054cf2ce9268023a27 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 30 Sep 2013 12:22:25 +0200 Subject: [PATCH] Introduced shardStats on InternalIndicesService to expose shard level statistics. Added a load_average_format=array|hash param to OsStats to allow serializing load averages into a hash (with keys 1m,5m, 15m). Added node_info_format to NodeStats, allowing to suppress node info output --- .../admin/cluster/node/stats/NodeStats.java | 24 +-- .../admin/indices/stats/CommonStats.java | 112 ++++++++++++ .../admin/indices/stats/CommonStatsFlags.java | 8 +- .../indices/stats/IndicesStatsResponse.java | 16 +- .../admin/indices/stats/ShardStats.java | 36 +++- .../stats/TransportIndicesStatsAction.java | 36 ++-- .../indices/InternalIndicesService.java | 169 +++++------------- .../org/elasticsearch/monitor/os/OsStats.java | 19 +- 8 files changed, 247 insertions(+), 173 deletions(-) diff --git a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java index 1c709508002..149fdc5fda8 100644 --- a/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java +++ b/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java @@ -281,19 +281,21 @@ public class NodeStats extends NodeOperationResponse implements ToXContent { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field("name", getNode().name(), XContentBuilder.FieldCaseConversion.NONE); - builder.field("transport_address", getNode().address().toString(), XContentBuilder.FieldCaseConversion.NONE); + if (!params.param("node_info_format", "default").equals("none")) { + builder.field("name", getNode().name(), XContentBuilder.FieldCaseConversion.NONE); + builder.field("transport_address", getNode().address().toString(), XContentBuilder.FieldCaseConversion.NONE); - if (getHostname() != null) { - builder.field("hostname", getHostname(), XContentBuilder.FieldCaseConversion.NONE); - } - - if (!getNode().attributes().isEmpty()) { - builder.startObject("attributes"); - for (Map.Entry attr : getNode().attributes().entrySet()) { - builder.field(attr.getKey(), attr.getValue(), XContentBuilder.FieldCaseConversion.NONE); + if (getHostname() != null) { + builder.field("hostname", getHostname(), XContentBuilder.FieldCaseConversion.NONE); + } + + if (!getNode().attributes().isEmpty()) { + builder.startObject("attributes"); + for (Map.Entry attr : getNode().attributes().entrySet()) { + builder.field(attr.getKey(), attr.getValue(), XContentBuilder.FieldCaseConversion.NONE); + } + builder.endObject(); } - builder.endObject(); } if (getIndices() != null) { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 1afc5eeac6c..43969b296e5 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.percolator.stats.PercolateStats; import org.elasticsearch.index.refresh.RefreshStats; import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.search.suggest.completion.CompletionStats; @@ -47,6 +48,117 @@ import java.io.IOException; */ public class CommonStats implements Streamable, ToXContent { + public CommonStats() { + this(CommonStatsFlags.NONE); + } + + public CommonStats(CommonStatsFlags flags) { + CommonStatsFlags.Flag[] setFlags = flags.getFlags(); + + for (CommonStatsFlags.Flag flag : setFlags) { + switch (flag) { + case Docs: + docs = new DocsStats(); + break; + case Store: + store = new StoreStats(); + break; + case Indexing: + indexing = new IndexingStats(); + break; + case Get: + get = new GetStats(); + break; + case Search: + search = new SearchStats(); + break; + case Merge: + merge = new MergeStats(); + break; + case Refresh: + refresh = new RefreshStats(); + break; + case Flush: + flush = new FlushStats(); + break; + case Warmer: + warmer = new WarmerStats(); + break; + case FilterCache: + filterCache = new FilterCacheStats(); + break; + case IdCache: + idCache = new IdCacheStats(); + break; + case FieldData: + fieldData = new FieldDataStats(); + break; + case Completion: + completion = new CompletionStats(); + break; + case Percolate: + percolate = new PercolateStats(); + break; + default: + throw new IllegalStateException("Unknown Flag: " + flag); + } + } + } + + + public CommonStats(IndexShard indexShard, CommonStatsFlags flags) { + CommonStatsFlags.Flag[] setFlags = flags.getFlags(); + + for (CommonStatsFlags.Flag flag : setFlags) { + switch (flag) { + case Docs: + docs = indexShard.docStats(); + break; + case Store: + store = indexShard.storeStats(); + break; + case Indexing: + indexing = indexShard.indexingStats(flags.types()); + break; + case Get: + get = indexShard.getStats(); + break; + case Search: + search = indexShard.searchStats(flags.groups()); + break; + case Merge: + merge = indexShard.mergeStats(); + break; + case Refresh: + refresh = indexShard.refreshStats(); + break; + case Flush: + flush = indexShard.flushStats(); + break; + case Warmer: + warmer = indexShard.warmerStats(); + break; + case FilterCache: + filterCache = indexShard.filterCacheStats(); + break; + case IdCache: + idCache = indexShard.idCacheStats(); + break; + case FieldData: + fieldData = indexShard.fieldDataStats(flags.fieldDataFields()); + break; + case Completion: + completion = indexShard.completionStats(flags.completionDataFields()); + break; + case Percolate: + percolate = indexShard.shardPercolateService().stats(); + break; + default: + throw new IllegalStateException("Unknown Flag: " + flag); + } + } + } + @Nullable public DocsStats docs; diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index e0209793a01..0efa0894134 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -30,6 +30,10 @@ import java.util.EnumSet; /** */ public class CommonStatsFlags implements Streamable, Cloneable { + + public final static CommonStatsFlags ALL = new CommonStatsFlags().all(); + public final static CommonStatsFlags NONE = new CommonStatsFlags().clear(); + private EnumSet flags = EnumSet.of(Flag.Docs, Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Percolate); private String[] types = null; private String[] groups = null; @@ -69,7 +73,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { } /** - * Document types to return stats for. Mainly affects {@link #indexing(boolean)} when + * Document types to return stats for. Mainly affects {@link Flag#Indexing} when * enabled, returning specific indexing stats for those types. */ public CommonStatsFlags types(String... types) { @@ -78,7 +82,7 @@ public class CommonStatsFlags implements Streamable, Cloneable { } /** - * Document types to return stats for. Mainly affects {@link #indexing(boolean)} when + * Document types to return stats for. Mainly affects {@link Flag#Indexing} when * enabled, returning specific indexing stats for those types. */ public String[] types() { diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java index a1afde168e6..c043ab1d9c4 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java @@ -186,16 +186,7 @@ public class IndicesStatsResponse extends BroadcastOperationResponse implements builder.startArray(Integer.toString(indexShardStats.getShardId().id())); for (ShardStats shardStats : indexShardStats) { builder.startObject(); - - builder.startObject(Fields.ROUTING) - .field(Fields.STATE, shardStats.getShardRouting().state()) - .field(Fields.PRIMARY, shardStats.getShardRouting().primary()) - .field(Fields.NODE, shardStats.getShardRouting().currentNodeId()) - .field(Fields.RELOCATING_NODE, shardStats.getShardRouting().relocatingNodeId()) - .endObject(); - - shardStats.getStats().toXContent(builder, params); - + shardStats.toXContent(builder, params); builder.endObject(); } builder.endArray(); @@ -213,11 +204,6 @@ public class IndicesStatsResponse extends BroadcastOperationResponse implements static final class Fields { static final XContentBuilderString INDICES = new XContentBuilderString("indices"); static final XContentBuilderString SHARDS = new XContentBuilderString("shards"); - static final XContentBuilderString ROUTING = new XContentBuilderString("routing"); - static final XContentBuilderString STATE = new XContentBuilderString("state"); - static final XContentBuilderString PRIMARY = new XContentBuilderString("primary"); - static final XContentBuilderString NODE = new XContentBuilderString("node"); - static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node"); } @Override diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java index 9e9bee4b13c..8a29290bd9d 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java @@ -23,6 +23,10 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRespons import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentBuilderString; +import org.elasticsearch.index.shard.service.IndexShard; import java.io.IOException; @@ -30,7 +34,7 @@ import static org.elasticsearch.cluster.routing.ImmutableShardRouting.readShardR /** */ -public class ShardStats extends BroadcastShardOperationResponse { +public class ShardStats extends BroadcastShardOperationResponse implements ToXContent { private ShardRouting shardRouting; @@ -39,10 +43,10 @@ public class ShardStats extends BroadcastShardOperationResponse { ShardStats() { } - ShardStats(ShardRouting shardRouting) { - super(shardRouting.index(), shardRouting.id()); - this.shardRouting = shardRouting; - this.stats = new CommonStats(); + public ShardStats(IndexShard indexShard, CommonStatsFlags flags) { + super(indexShard.routingEntry().index(), indexShard.routingEntry().id()); + this.shardRouting = indexShard.routingEntry(); + this.stats = new CommonStats(indexShard, flags); } /** @@ -75,4 +79,26 @@ public class ShardStats extends BroadcastShardOperationResponse { shardRouting.writeTo(out); stats.writeTo(out); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.ROUTING) + .field(Fields.STATE, shardRouting.state()) + .field(Fields.PRIMARY, shardRouting.primary()) + .field(Fields.NODE, shardRouting.currentNodeId()) + .field(Fields.RELOCATING_NODE, shardRouting.relocatingNodeId()) + .endObject(); + + stats.toXContent(builder, params); + return builder; + } + + static final class Fields { + static final XContentBuilderString ROUTING = new XContentBuilderString("routing"); + static final XContentBuilderString STATE = new XContentBuilderString("state"); + static final XContentBuilderString PRIMARY = new XContentBuilderString("primary"); + static final XContentBuilderString NODE = new XContentBuilderString("node"); + static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node"); + } + } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java b/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java index b0087eb6857..f6ed0da5f25 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java @@ -138,51 +138,57 @@ public class TransportIndicesStatsAction extends TransportBroadcastOperationActi protected ShardStats shardOperation(IndexShardStatsRequest request) throws ElasticSearchException { InternalIndexService indexService = (InternalIndexService) indicesService.indexServiceSafe(request.index()); InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId()); - ShardStats stats = new ShardStats(indexShard.routingEntry()); + + CommonStatsFlags flags = new CommonStatsFlags().clear(); if (request.request.docs()) { - stats.stats.docs = indexShard.docStats(); + flags.set(CommonStatsFlags.Flag.Docs); } if (request.request.store()) { - stats.stats.store = indexShard.storeStats(); + flags.set(CommonStatsFlags.Flag.Store); } if (request.request.indexing()) { - stats.stats.indexing = indexShard.indexingStats(request.request.types()); + flags.set(CommonStatsFlags.Flag.Indexing); + flags.types(request.request.types()); } if (request.request.get()) { - stats.stats.get = indexShard.getStats(); + flags.set(CommonStatsFlags.Flag.Get); } if (request.request.search()) { - stats.getStats().search = indexShard.searchStats(request.request.groups()); + flags.set(CommonStatsFlags.Flag.Search); + flags.groups(request.request.groups()); } if (request.request.merge()) { - stats.stats.merge = indexShard.mergeStats(); + flags.set(CommonStatsFlags.Flag.Merge); } if (request.request.refresh()) { - stats.stats.refresh = indexShard.refreshStats(); + flags.set(CommonStatsFlags.Flag.Refresh); } if (request.request.flush()) { - stats.stats.flush = indexShard.flushStats(); + flags.set(CommonStatsFlags.Flag.Flush); } if (request.request.warmer()) { - stats.stats.warmer = indexShard.warmerStats(); + flags.set(CommonStatsFlags.Flag.Warmer); } if (request.request.filterCache()) { - stats.stats.filterCache = indexShard.filterCacheStats(); + flags.set(CommonStatsFlags.Flag.FilterCache); } if (request.request.idCache()) { - stats.stats.idCache = indexShard.idCacheStats(); + flags.set(CommonStatsFlags.Flag.IdCache); } if (request.request.fieldData()) { - stats.stats.fieldData = indexShard.fieldDataStats(request.request.fieldDataFields()); + flags.set(CommonStatsFlags.Flag.FieldData); + flags.fieldDataFields(request.request.fieldDataFields()); } if (request.request.percolate()) { - stats.stats.percolate = indexShard.shardPercolateService().stats(); + flags.set(CommonStatsFlags.Flag.Percolate); } if (request.request.completion()) { - stats.stats.completion = indexShard.completionStats(request.request.completionFields()); + flags.set(CommonStatsFlags.Flag.Completion); + flags.completionDataFields(request.request.completionFields()); } + ShardStats stats = new ShardStats(indexShard, flags); return stats; } diff --git a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java index bcf1047575e..d1942c8ab5d 100644 --- a/src/main/java/org/elasticsearch/indices/InternalIndicesService.java +++ b/src/main/java/org/elasticsearch/indices/InternalIndicesService.java @@ -21,12 +21,14 @@ package org.elasticsearch.indices; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import com.google.common.collect.UnmodifiableIterator; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag; +import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.inject.*; @@ -39,12 +41,9 @@ import org.elasticsearch.index.analysis.AnalysisModule; import org.elasticsearch.index.analysis.AnalysisService; import org.elasticsearch.index.cache.IndexCache; import org.elasticsearch.index.cache.IndexCacheModule; -import org.elasticsearch.index.cache.filter.FilterCacheStats; -import org.elasticsearch.index.cache.id.IdCacheStats; import org.elasticsearch.index.codec.CodecModule; import org.elasticsearch.index.engine.IndexEngine; import org.elasticsearch.index.engine.IndexEngineModule; -import org.elasticsearch.index.fielddata.FieldDataStats; import org.elasticsearch.index.fielddata.IndexFieldDataModule; import org.elasticsearch.index.fielddata.IndexFieldDataService; import org.elasticsearch.index.flush.FlushStats; @@ -55,7 +54,6 @@ import org.elasticsearch.index.indexing.IndexingStats; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceModule; import org.elasticsearch.index.merge.MergeStats; -import org.elasticsearch.index.percolator.stats.PercolateStats; import org.elasticsearch.index.query.IndexQueryParserModule; import org.elasticsearch.index.query.IndexQueryParserService; import org.elasticsearch.index.refresh.RefreshStats; @@ -63,22 +61,19 @@ import org.elasticsearch.index.search.stats.SearchStats; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.index.service.InternalIndexService; import org.elasticsearch.index.settings.IndexSettingsModule; -import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.index.similarity.SimilarityModule; import org.elasticsearch.index.store.IndexStore; import org.elasticsearch.index.store.IndexStoreModule; -import org.elasticsearch.index.store.StoreStats; -import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.indices.analysis.IndicesAnalysisService; import org.elasticsearch.indices.recovery.RecoverySettings; import org.elasticsearch.indices.store.IndicesStore; import org.elasticsearch.plugins.IndexPluginsModule; import org.elasticsearch.plugins.PluginsService; -import org.elasticsearch.search.suggest.completion.CompletionStats; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -182,131 +177,63 @@ public class InternalIndicesService extends AbstractLifecycleComponent shardStats = Lists.newArrayList(); + for (String index : indices()) { + IndexService indexService = indexService(index); + if (indexService == null) { + continue; // something changed, move along + } + for (int shardId : indexService.shardIds()) { + IndexShard indexShard = indexService.shard(shardId); + if (indexShard == null) { + continue; + } + shardStats.add(new ShardStats(indexShard, flags)); + } + } + return shardStats.toArray(new ShardStats[shardStats.size()]); + } + /** * Returns true if changes (adding / removing) indices, shards and so on are allowed. */ diff --git a/src/main/java/org/elasticsearch/monitor/os/OsStats.java b/src/main/java/org/elasticsearch/monitor/os/OsStats.java index 60c80c5d523..a83edaed073 100644 --- a/src/main/java/org/elasticsearch/monitor/os/OsStats.java +++ b/src/main/java/org/elasticsearch/monitor/os/OsStats.java @@ -109,6 +109,9 @@ public class OsStats implements Streamable, Serializable, ToXContent { static final XContentBuilderString UPTIME = new XContentBuilderString("uptime"); static final XContentBuilderString UPTIME_IN_MILLIS = new XContentBuilderString("uptime_in_millis"); static final XContentBuilderString LOAD_AVERAGE = new XContentBuilderString("load_average"); + static final XContentBuilderString LOAD_AVERAGE_1m = new XContentBuilderString("1m"); + static final XContentBuilderString LOAD_AVERAGE_5m = new XContentBuilderString("5m"); + static final XContentBuilderString LOAD_AVERAGE_15m = new XContentBuilderString("15m"); static final XContentBuilderString CPU = new XContentBuilderString("cpu"); static final XContentBuilderString SYS = new XContentBuilderString("sys"); @@ -142,11 +145,19 @@ public class OsStats implements Streamable, Serializable, ToXContent { } if (loadAverage.length > 0) { - builder.startArray(Fields.LOAD_AVERAGE); - for (double value : loadAverage) { - builder.value(value); + if (params.param("load_average_format", "array").equals("hash")) { + builder.startObject(Fields.LOAD_AVERAGE); + builder.field(Fields.LOAD_AVERAGE_1m, loadAverage[0]); + builder.field(Fields.LOAD_AVERAGE_5m, loadAverage[1]); + builder.field(Fields.LOAD_AVERAGE_15m, loadAverage[2]); + builder.endObject(); + } else { + builder.startArray(Fields.LOAD_AVERAGE); + for (double value : loadAverage) { + builder.value(value); + } + builder.endArray(); } - builder.endArray(); } if (cpu != null) {