Introduced shardStats on InternalIndicesService to expose shard level statistics.

Added a load_average_format=array|hash param to OsStats to allow serializing load averages into a hash (with keys 1m,5m, 15m).
Added node_info_format to NodeStats, allowing to suppress node info output

src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStats.java

@@ -281,6 +281,7 @@ public class NodeStats extends NodeOperationResponse implements ToXContent {
 
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        if (!params.param("node_info_format", "default").equals("none")) {
             builder.field("name", getNode().name(), XContentBuilder.FieldCaseConversion.NONE);
             builder.field("transport_address", getNode().address().toString(), XContentBuilder.FieldCaseConversion.NONE);
 
@@ -295,6 +296,7 @@ public class NodeStats extends NodeOperationResponse implements ToXContent {
                 }
                 builder.endObject();
             }
+        }
 
         if (getIndices() != null) {
             getIndices().toXContent(builder, params);

src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java

@@ -37,6 +37,7 @@ import org.elasticsearch.index.percolator.stats.PercolateStats;
 import org.elasticsearch.index.refresh.RefreshStats;
 import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.shard.DocsStats;
+import org.elasticsearch.index.shard.service.IndexShard;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
@@ -47,6 +48,117 @@ import java.io.IOException;
  */
 public class CommonStats implements Streamable, ToXContent {
 
+    public CommonStats() {
+        this(CommonStatsFlags.NONE);
+    }
+
+    public CommonStats(CommonStatsFlags flags) {
+        CommonStatsFlags.Flag[] setFlags = flags.getFlags();
+
+        for (CommonStatsFlags.Flag flag : setFlags) {
+            switch (flag) {
+                case Docs:
+                    docs = new DocsStats();
+                    break;
+                case Store:
+                    store = new StoreStats();
+                    break;
+                case Indexing:
+                    indexing = new IndexingStats();
+                    break;
+                case Get:
+                    get = new GetStats();
+                    break;
+                case Search:
+                    search = new SearchStats();
+                    break;
+                case Merge:
+                    merge = new MergeStats();
+                    break;
+                case Refresh:
+                    refresh = new RefreshStats();
+                    break;
+                case Flush:
+                    flush = new FlushStats();
+                    break;
+                case Warmer:
+                    warmer = new WarmerStats();
+                    break;
+                case FilterCache:
+                    filterCache = new FilterCacheStats();
+                    break;
+                case IdCache:
+                    idCache = new IdCacheStats();
+                    break;
+                case FieldData:
+                    fieldData = new FieldDataStats();
+                    break;
+                case Completion:
+                    completion = new CompletionStats();
+                    break;
+                case Percolate:
+                    percolate = new PercolateStats();
+                    break;
+                default:
+                    throw new IllegalStateException("Unknown Flag: " + flag);
+            }
+        }
+    }
+
+
+    public CommonStats(IndexShard indexShard, CommonStatsFlags flags) {
+        CommonStatsFlags.Flag[] setFlags = flags.getFlags();
+
+        for (CommonStatsFlags.Flag flag : setFlags) {
+            switch (flag) {
+                case Docs:
+                    docs = indexShard.docStats();
+                    break;
+                case Store:
+                    store = indexShard.storeStats();
+                    break;
+                case Indexing:
+                    indexing = indexShard.indexingStats(flags.types());
+                    break;
+                case Get:
+                    get = indexShard.getStats();
+                    break;
+                case Search:
+                    search = indexShard.searchStats(flags.groups());
+                    break;
+                case Merge:
+                    merge = indexShard.mergeStats();
+                    break;
+                case Refresh:
+                    refresh = indexShard.refreshStats();
+                    break;
+                case Flush:
+                    flush = indexShard.flushStats();
+                    break;
+                case Warmer:
+                    warmer = indexShard.warmerStats();
+                    break;
+                case FilterCache:
+                    filterCache = indexShard.filterCacheStats();
+                    break;
+                case IdCache:
+                    idCache = indexShard.idCacheStats();
+                    break;
+                case FieldData:
+                    fieldData = indexShard.fieldDataStats(flags.fieldDataFields());
+                    break;
+                case Completion:
+                    completion = indexShard.completionStats(flags.completionDataFields());
+                    break;
+                case Percolate:
+                    percolate = indexShard.shardPercolateService().stats();
+                    break;
+                default:
+                    throw new IllegalStateException("Unknown Flag: " + flag);
+            }
+        }
+    }
+
     @Nullable
     public DocsStats docs;
 

src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java

@@ -30,6 +30,10 @@ import java.util.EnumSet;
 /**
  */
 public class CommonStatsFlags implements Streamable, Cloneable {
+
+    public final static CommonStatsFlags ALL = new CommonStatsFlags().all();
+    public final static CommonStatsFlags NONE = new CommonStatsFlags().clear();
+
     private EnumSet<Flag> flags = EnumSet.of(Flag.Docs, Flag.Store, Flag.Indexing, Flag.Get, Flag.Search, Flag.Percolate);
     private String[] types = null;
     private String[] groups = null;
@@ -69,7 +73,7 @@ public class CommonStatsFlags implements Streamable, Cloneable {
     }
 
     /**
-     * Document types to return stats for. Mainly affects {@link #indexing(boolean)} when
+     * Document types to return stats for. Mainly affects {@link Flag#Indexing} when
      * enabled, returning specific indexing stats for those types.
      */
     public CommonStatsFlags types(String... types) {
@@ -78,7 +82,7 @@ public class CommonStatsFlags implements Streamable, Cloneable {
     }
 
     /**
-     * Document types to return stats for. Mainly affects {@link #indexing(boolean)} when
+     * Document types to return stats for. Mainly affects {@link Flag#Indexing} when
      * enabled, returning specific indexing stats for those types.
      */
     public String[] types() {

src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponse.java

@@ -186,16 +186,7 @@ public class IndicesStatsResponse extends BroadcastOperationResponse implements
                     builder.startArray(Integer.toString(indexShardStats.getShardId().id()));
                     for (ShardStats shardStats : indexShardStats) {
                         builder.startObject();
-
+                        shardStats.toXContent(builder, params);
-                        builder.startObject(Fields.ROUTING)
-                                .field(Fields.STATE, shardStats.getShardRouting().state())
-                                .field(Fields.PRIMARY, shardStats.getShardRouting().primary())
-                                .field(Fields.NODE, shardStats.getShardRouting().currentNodeId())
-                                .field(Fields.RELOCATING_NODE, shardStats.getShardRouting().relocatingNodeId())
-                                .endObject();
-
-                        shardStats.getStats().toXContent(builder, params);
-
                         builder.endObject();
                     }
                     builder.endArray();
@@ -213,11 +204,6 @@ public class IndicesStatsResponse extends BroadcastOperationResponse implements
     static final class Fields {
         static final XContentBuilderString INDICES = new XContentBuilderString("indices");
         static final XContentBuilderString SHARDS = new XContentBuilderString("shards");
-        static final XContentBuilderString ROUTING = new XContentBuilderString("routing");
-        static final XContentBuilderString STATE = new XContentBuilderString("state");
-        static final XContentBuilderString PRIMARY = new XContentBuilderString("primary");
-        static final XContentBuilderString NODE = new XContentBuilderString("node");
-        static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node");
     }
 
     @Override

src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java

@@ -23,6 +23,10 @@ import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRespons
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+import org.elasticsearch.index.shard.service.IndexShard;
 
 import java.io.IOException;
 
@@ -30,7 +34,7 @@ import static org.elasticsearch.cluster.routing.ImmutableShardRouting.readShardR
 
 /**
  */
-public class ShardStats extends BroadcastShardOperationResponse {
+public class ShardStats extends BroadcastShardOperationResponse implements ToXContent {
 
     private ShardRouting shardRouting;
 
@@ -39,10 +43,10 @@ public class ShardStats extends BroadcastShardOperationResponse {
     ShardStats() {
     }
 
-    ShardStats(ShardRouting shardRouting) {
+    public ShardStats(IndexShard indexShard, CommonStatsFlags flags) {
-        super(shardRouting.index(), shardRouting.id());
+        super(indexShard.routingEntry().index(), indexShard.routingEntry().id());
-        this.shardRouting = shardRouting;
+        this.shardRouting = indexShard.routingEntry();
-        this.stats = new CommonStats();
+        this.stats = new CommonStats(indexShard, flags);
     }
 
     /**
@@ -75,4 +79,26 @@ public class ShardStats extends BroadcastShardOperationResponse {
         shardRouting.writeTo(out);
         stats.writeTo(out);
     }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.ROUTING)
+                .field(Fields.STATE, shardRouting.state())
+                .field(Fields.PRIMARY, shardRouting.primary())
+                .field(Fields.NODE, shardRouting.currentNodeId())
+                .field(Fields.RELOCATING_NODE, shardRouting.relocatingNodeId())
+                .endObject();
+
+        stats.toXContent(builder, params);
+        return builder;
+    }
+
+    static final class Fields {
+        static final XContentBuilderString ROUTING = new XContentBuilderString("routing");
+        static final XContentBuilderString STATE = new XContentBuilderString("state");
+        static final XContentBuilderString PRIMARY = new XContentBuilderString("primary");
+        static final XContentBuilderString NODE = new XContentBuilderString("node");
+        static final XContentBuilderString RELOCATING_NODE = new XContentBuilderString("relocating_node");
+    }
+
 }

src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java

@@ -138,51 +138,57 @@ public class TransportIndicesStatsAction extends TransportBroadcastOperationActi
     protected ShardStats shardOperation(IndexShardStatsRequest request) throws ElasticSearchException {
         InternalIndexService indexService = (InternalIndexService) indicesService.indexServiceSafe(request.index());
         InternalIndexShard indexShard = (InternalIndexShard) indexService.shardSafe(request.shardId());
-        ShardStats stats = new ShardStats(indexShard.routingEntry());
+
+        CommonStatsFlags flags = new CommonStatsFlags().clear();
 
         if (request.request.docs()) {
-            stats.stats.docs = indexShard.docStats();
+            flags.set(CommonStatsFlags.Flag.Docs);
         }
         if (request.request.store()) {
-            stats.stats.store = indexShard.storeStats();
+            flags.set(CommonStatsFlags.Flag.Store);
         }
         if (request.request.indexing()) {
-            stats.stats.indexing = indexShard.indexingStats(request.request.types());
+            flags.set(CommonStatsFlags.Flag.Indexing);
+            flags.types(request.request.types());
         }
         if (request.request.get()) {
-            stats.stats.get = indexShard.getStats();
+            flags.set(CommonStatsFlags.Flag.Get);
         }
         if (request.request.search()) {
-            stats.getStats().search = indexShard.searchStats(request.request.groups());
+            flags.set(CommonStatsFlags.Flag.Search);
+            flags.groups(request.request.groups());
         }
         if (request.request.merge()) {
-            stats.stats.merge = indexShard.mergeStats();
+            flags.set(CommonStatsFlags.Flag.Merge);
         }
         if (request.request.refresh()) {
-            stats.stats.refresh = indexShard.refreshStats();
+            flags.set(CommonStatsFlags.Flag.Refresh);
         }
         if (request.request.flush()) {
-            stats.stats.flush = indexShard.flushStats();
+            flags.set(CommonStatsFlags.Flag.Flush);
         }
         if (request.request.warmer()) {
-            stats.stats.warmer = indexShard.warmerStats();
+            flags.set(CommonStatsFlags.Flag.Warmer);
         }
         if (request.request.filterCache()) {
-            stats.stats.filterCache = indexShard.filterCacheStats();
+            flags.set(CommonStatsFlags.Flag.FilterCache);
         }
         if (request.request.idCache()) {
-            stats.stats.idCache = indexShard.idCacheStats();
+            flags.set(CommonStatsFlags.Flag.IdCache);
         }
         if (request.request.fieldData()) {
-            stats.stats.fieldData = indexShard.fieldDataStats(request.request.fieldDataFields());
+            flags.set(CommonStatsFlags.Flag.FieldData);
+            flags.fieldDataFields(request.request.fieldDataFields());
         }
         if (request.request.percolate()) {
-            stats.stats.percolate = indexShard.shardPercolateService().stats();
+            flags.set(CommonStatsFlags.Flag.Percolate);
         }
         if (request.request.completion()) {
-            stats.stats.completion = indexShard.completionStats(request.request.completionFields());
+            flags.set(CommonStatsFlags.Flag.Completion);
+            flags.completionDataFields(request.request.completionFields());
         }
 
+        ShardStats stats = new ShardStats(indexShard, flags);
         return stats;
     }
 

src/main/java/org/elasticsearch/indices/InternalIndicesService.java

@@ -21,12 +21,14 @@ package org.elasticsearch.indices;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
 import com.google.common.collect.UnmodifiableIterator;
 import org.elasticsearch.ElasticSearchException;
 import org.elasticsearch.ElasticSearchIllegalStateException;
 import org.elasticsearch.action.admin.indices.stats.CommonStats;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags;
 import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags.Flag;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.component.AbstractLifecycleComponent;
 import org.elasticsearch.common.inject.*;
@@ -39,12 +41,9 @@ import org.elasticsearch.index.analysis.AnalysisModule;
 import org.elasticsearch.index.analysis.AnalysisService;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.cache.IndexCacheModule;
-import org.elasticsearch.index.cache.filter.FilterCacheStats;
-import org.elasticsearch.index.cache.id.IdCacheStats;
 import org.elasticsearch.index.codec.CodecModule;
 import org.elasticsearch.index.engine.IndexEngine;
 import org.elasticsearch.index.engine.IndexEngineModule;
-import org.elasticsearch.index.fielddata.FieldDataStats;
 import org.elasticsearch.index.fielddata.IndexFieldDataModule;
 import org.elasticsearch.index.fielddata.IndexFieldDataService;
 import org.elasticsearch.index.flush.FlushStats;
@@ -55,7 +54,6 @@ import org.elasticsearch.index.indexing.IndexingStats;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceModule;
 import org.elasticsearch.index.merge.MergeStats;
-import org.elasticsearch.index.percolator.stats.PercolateStats;
 import org.elasticsearch.index.query.IndexQueryParserModule;
 import org.elasticsearch.index.query.IndexQueryParserService;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -63,22 +61,19 @@ import org.elasticsearch.index.search.stats.SearchStats;
 import org.elasticsearch.index.service.IndexService;
 import org.elasticsearch.index.service.InternalIndexService;
 import org.elasticsearch.index.settings.IndexSettingsModule;
-import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.index.shard.service.IndexShard;
 import org.elasticsearch.index.similarity.SimilarityModule;
 import org.elasticsearch.index.store.IndexStore;
 import org.elasticsearch.index.store.IndexStoreModule;
-import org.elasticsearch.index.store.StoreStats;
-import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.indices.analysis.IndicesAnalysisService;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.indices.store.IndicesStore;
 import org.elasticsearch.plugins.IndexPluginsModule;
 import org.elasticsearch.plugins.PluginsService;
-import org.elasticsearch.search.suggest.completion.CompletionStats;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
@@ -182,131 +177,63 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
 
     @Override
     public NodeIndicesStats stats(boolean includePrevious, CommonStatsFlags flags) {
-        CommonStats stats = new CommonStats();
+        CommonStats stats = new CommonStats(flags);
+
+        if (includePrevious) {
             Flag[] setFlags = flags.getFlags();
             for (Flag flag : setFlags) {
                 switch (flag) {
-                case Docs:
-                    stats.docs = new DocsStats();
-                    break;
-                case Store:
-                    stats.store = new StoreStats();
-                    break;
-                case Warmer:
-                    stats.warmer = new WarmerStats();
-                    break;
                     case Get:
-                    stats.get = new GetStats();
-                    if (includePrevious) {
                         stats.get.add(oldShardsStats.getStats);
-                    }
                         break;
                     case Indexing:
-                    stats.indexing = new IndexingStats();
-                    if (includePrevious) {
                         stats.indexing.add(oldShardsStats.indexingStats);
-                    }
                         break;
                     case Search:
-                    stats.search = new SearchStats();
-                    if (includePrevious) {
                         stats.search.add(oldShardsStats.searchStats);
-                    }
                         break;
                     case Merge:
-                    stats.merge = new MergeStats();
-                    if (includePrevious) {
                         stats.merge.add(oldShardsStats.mergeStats);
-                    }
                         break;
                     case Refresh:
-                    stats.refresh = new RefreshStats();
-                    if (includePrevious) {
                         stats.refresh.add(oldShardsStats.refreshStats);
-                    }
                         break;
                     case Flush:
-                    stats.flush = new FlushStats();
-                    if (includePrevious) {
                         stats.flush.add(oldShardsStats.flushStats);
-                    }
                         break;
-                case FieldData:
-                    stats.fieldData = new FieldDataStats();
-                    break;
-                case IdCache:
-                    stats.idCache = new IdCacheStats();
-                    break;
-                case FilterCache:
-                    stats.filterCache = new FilterCacheStats();
-                    break;
-                case Percolate:
-                    stats.percolate = new PercolateStats();
-                    break;
-                case Completion:
-                    stats.completion = new CompletionStats();
-                    break;
-                default:
-                    throw new IllegalStateException("Unknown Flag: " + flag);
                 }
             }
-
+        }
 
         for (IndexService indexService : indices.values()) {
             for (IndexShard indexShard : indexService) {
-                for (Flag flag : setFlags) {
+                CommonStats indexStas = new CommonStats(indexShard, flags);
-                    switch (flag) {
+                stats.add(indexStas);
-                        case Store:
-                            stats.store.add(indexShard.storeStats());
-                            break;
-                        case Docs:
-                            stats.docs.add(indexShard.docStats());
-                            break;
-                        case Get:
-                            stats.get.add(indexShard.getStats());
-                            break;
-                        case Indexing:
-                            stats.indexing.add(indexShard.indexingStats());
-                            break;
-                        case Search:
-                            stats.search.add(indexShard.searchStats());
-                            break;
-                        case Merge:
-                            stats.merge.add(indexShard.mergeStats());
-                            break;
-                        case Refresh:
-                            stats.refresh.add(indexShard.refreshStats());
-                            break;
-                        case Flush:
-                            stats.flush.add(indexShard.flushStats());
-                            break;
-                        case FilterCache:
-                            stats.filterCache.add(indexShard.filterCacheStats());
-                            break;
-                        case IdCache:
-                            stats.idCache.add(indexShard.idCacheStats());
-                            break;
-                        case FieldData:
-                            stats.fieldData.add(indexShard.fieldDataStats(flags.fieldDataFields()));
-                            break;
-                        case Warmer:
-                            stats.warmer.add(indexShard.warmerStats());
-                            break;
-                        case Percolate:
-                            stats.percolate.add(indexShard.shardPercolateService().stats());
-                            break;
-                        case Completion:
-                            stats.completion.add(indexShard.completionStats(flags.completionDataFields()));
-                            break;
-                        default:
-                            throw new IllegalStateException("Unknown Flag: " + flag);
-                    }
-                }
             }
         }
         return new NodeIndicesStats(stats);
     }
 
+
+    public ShardStats[] shardStats(CommonStatsFlags flags) {
+        // TODO: Do we want to upgrade this to the IndicesService level
+        List<ShardStats> shardStats = Lists.newArrayList();
+        for (String index : indices()) {
+            IndexService indexService = indexService(index);
+            if (indexService == null) {
+                continue; // something changed, move along
+            }
+            for (int shardId : indexService.shardIds()) {
+                IndexShard indexShard = indexService.shard(shardId);
+                if (indexShard == null) {
+                    continue;
+                }
+                shardStats.add(new ShardStats(indexShard, flags));
+            }
+        }
+        return shardStats.toArray(new ShardStats[shardStats.size()]);
+    }
+
     /**
      * Returns <tt>true</tt> if changes (adding / removing) indices, shards and so on are allowed.
      */

src/main/java/org/elasticsearch/monitor/os/OsStats.java

@@ -109,6 +109,9 @@ public class OsStats implements Streamable, Serializable, ToXContent {
         static final XContentBuilderString UPTIME = new XContentBuilderString("uptime");
         static final XContentBuilderString UPTIME_IN_MILLIS = new XContentBuilderString("uptime_in_millis");
         static final XContentBuilderString LOAD_AVERAGE = new XContentBuilderString("load_average");
+        static final XContentBuilderString LOAD_AVERAGE_1m = new XContentBuilderString("1m");
+        static final XContentBuilderString LOAD_AVERAGE_5m = new XContentBuilderString("5m");
+        static final XContentBuilderString LOAD_AVERAGE_15m = new XContentBuilderString("15m");
 
         static final XContentBuilderString CPU = new XContentBuilderString("cpu");
         static final XContentBuilderString SYS = new XContentBuilderString("sys");
@@ -142,12 +145,20 @@ public class OsStats implements Streamable, Serializable, ToXContent {
         }
 
         if (loadAverage.length > 0) {
+            if (params.param("load_average_format", "array").equals("hash")) {
+                builder.startObject(Fields.LOAD_AVERAGE);
+                builder.field(Fields.LOAD_AVERAGE_1m, loadAverage[0]);
+                builder.field(Fields.LOAD_AVERAGE_5m, loadAverage[1]);
+                builder.field(Fields.LOAD_AVERAGE_15m, loadAverage[2]);
+                builder.endObject();
+            } else {
                 builder.startArray(Fields.LOAD_AVERAGE);
                 for (double value : loadAverage) {
                     builder.value(value);
                 }
                 builder.endArray();
             }
+        }
 
         if (cpu != null) {
             builder.startObject(Fields.CPU);