Node / Indices Stats: Add get stats (including missing / exists), closes #1288.

modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Streamable;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.flush.FlushStats;
+import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.indexing.IndexingStats;
 import org.elasticsearch.index.merge.MergeStats;
 import org.elasticsearch.index.refresh.RefreshStats;
@@ -44,6 +45,8 @@ public class CommonStats implements Streamable, ToXContent {
 
     @Nullable IndexingStats indexing;
 
+    @Nullable GetStats get;
+
     @Nullable MergeStats merge;
 
     @Nullable RefreshStats refresh;
@@ -75,6 +78,14 @@ public class CommonStats implements Streamable, ToXContent {
         } else {
             indexing.add(stats.indexing());
         }
+        if (get == null) {
+            if (stats.get() != null) {
+                get = new GetStats();
+                get.add(stats.get());
+            }
+        } else {
+            get.add(stats.get());
+        }
         if (merge == null) {
             if (stats.merge() != null) {
                 merge = new MergeStats();
@@ -125,6 +136,14 @@ public class CommonStats implements Streamable, ToXContent {
         return indexing;
     }
 
+    @Nullable public GetStats get() {
+        return get;
+    }
+
+    @Nullable public GetStats getGet() {
+        return get;
+    }
+
     @Nullable public MergeStats merge() {
         return merge;
     }
@@ -165,6 +184,9 @@ public class CommonStats implements Streamable, ToXContent {
         if (in.readBoolean()) {
             indexing = IndexingStats.readIndexingStats(in);
         }
+        if (in.readBoolean()) {
+            get = GetStats.readGetStats(in);
+        }
         if (in.readBoolean()) {
             merge = MergeStats.readMergeStats(in);
         }
@@ -195,6 +217,12 @@ public class CommonStats implements Streamable, ToXContent {
             out.writeBoolean(true);
             indexing.writeTo(out);
         }
+        if (get == null) {
+            out.writeBoolean(false);
+        } else {
+            out.writeBoolean(true);
+            get.writeTo(out);
+        }
         if (merge == null) {
             out.writeBoolean(false);
         } else {
@@ -226,6 +254,9 @@ public class CommonStats implements Streamable, ToXContent {
         if (indexing != null) {
             indexing.toXContent(builder, params);
         }
+        if (get != null) {
+            get.toXContent(builder, params);
+        }
         if (merge != null) {
             merge.toXContent(builder, params);
         }

modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java

@@ -39,6 +39,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
     private boolean docs = true;
     private boolean store = true;
     private boolean indexing = true;
+    private boolean get = true;
     private boolean merge = false;
     private boolean refresh = false;
     private boolean flush = false;
@@ -55,6 +56,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
     public IndicesStatsRequest clear() {
         docs = false;
         store = false;
+        get = false;
         indexing = false;
         merge = false;
         refresh = false;
@@ -106,6 +108,15 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
         return this.indexing;
     }
 
+    public IndicesStatsRequest get(boolean get) {
+        this.get = get;
+        return this;
+    }
+
+    public boolean get() {
+        return this.get;
+    }
+
     public IndicesStatsRequest merge(boolean merge) {
         this.merge = merge;
         return this;
@@ -138,6 +149,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
         out.writeBoolean(docs);
         out.writeBoolean(store);
         out.writeBoolean(indexing);
+        out.writeBoolean(get);
         out.writeBoolean(merge);
         out.writeBoolean(flush);
         out.writeBoolean(refresh);
@@ -156,6 +168,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
         docs = in.readBoolean();
         store = in.readBoolean();
         indexing = in.readBoolean();
+        get = in.readBoolean();
         merge = in.readBoolean();
         flush = in.readBoolean();
         refresh = in.readBoolean();

modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java

@@ -135,6 +135,9 @@ public class TransportIndicesStatsAction extends TransportBroadcastOperationActi
         if (request.request.indexing()) {
             stats.stats.indexing = indexShard.indexingStats(request.request.types());
         }
+        if (request.request.get()) {
+            stats.stats.get = indexShard.getStats();
+        }
         if (request.request.merge()) {
             stats.stats.merge = indexShard.mergeStats();
         }

modules/elasticsearch/src/main/java/org/elasticsearch/index/get/GetStats.java

@@ -0,0 +1,180 @@
+/*
+ * Licensed to Elastic Search and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Elastic Search licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.get;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Streamable;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentBuilderString;
+
+import java.io.IOException;
+
+/**
+ */
+public class GetStats implements Streamable, ToXContent {
+
+    private long existsCount;
+    private long existsTimeInMillis;
+    private long missingCount;
+    private long missingTimeInMillis;
+
+    public GetStats() {
+    }
+
+    public GetStats(long existsCount, long existsTimeInMillis, long missingCount, long missingTimeInMillis) {
+        this.existsCount = existsCount;
+        this.existsTimeInMillis = existsTimeInMillis;
+        this.missingCount = missingCount;
+        this.missingTimeInMillis = missingTimeInMillis;
+    }
+
+    public void add(GetStats stats) {
+        if (stats == null) {
+            return;
+        }
+        existsCount += stats.existsCount;
+        existsTimeInMillis += stats.existsTimeInMillis;
+        missingCount += stats.missingCount;
+        missingTimeInMillis += stats.missingTimeInMillis;
+    }
+
+    public long count() {
+        return existsCount + missingCount;
+    }
+
+    public long getCount() {
+        return count();
+    }
+
+    public long timeInMillis() {
+        return existsTimeInMillis + missingTimeInMillis;
+    }
+
+    public long getTimeInMillis() {
+        return timeInMillis();
+    }
+
+    public TimeValue time() {
+        return new TimeValue(timeInMillis());
+    }
+
+    public TimeValue getTime() {
+        return time();
+    }
+
+    public long existsCount() {
+        return this.existsCount;
+    }
+
+    public long getExistsCount() {
+        return this.existsCount;
+    }
+
+    public long existsTimeInMillis() {
+        return this.existsTimeInMillis;
+    }
+
+    public long getExistsTimeInMillis() {
+        return this.existsTimeInMillis;
+    }
+
+    public TimeValue existsTime() {
+        return new TimeValue(existsTimeInMillis);
+    }
+
+    public TimeValue getExistsTime() {
+        return existsTime();
+    }
+
+    public long missingCount() {
+        return this.missingCount;
+    }
+
+    public long getMissingCount() {
+        return this.missingCount;
+    }
+
+    public long missingTimeInMillis() {
+        return this.missingTimeInMillis;
+    }
+
+    public long getMissingTimeInMillis() {
+        return this.missingTimeInMillis;
+    }
+
+    public TimeValue missingTime() {
+        return new TimeValue(missingTimeInMillis);
+    }
+
+    public TimeValue getMissingTime() {
+        return missingTime();
+    }
+
+    @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject(Fields.GET);
+        builder.field(Fields.TOTAL, count());
+        builder.field(Fields.TIME, time().toString());
+        builder.field(Fields.TIME_IN_MILLIS, timeInMillis());
+        builder.field(Fields.EXISTS_TOTAL, existsCount);
+        builder.field(Fields.EXISTS_TIME, existsTime().toString());
+        builder.field(Fields.EXISTS_TIME_IN_MILLIS, existsTimeInMillis);
+        builder.field(Fields.MISSING_TOTAL, missingCount);
+        builder.field(Fields.MISSING_TIME, missingTime().toString());
+        builder.field(Fields.MISSING_TIME_IN_MILLIS, missingTimeInMillis);
+        builder.endObject();
+        return builder;
+    }
+
+    static final class Fields {
+        static final XContentBuilderString GET = new XContentBuilderString("get");
+        static final XContentBuilderString TOTAL = new XContentBuilderString("total");
+        static final XContentBuilderString TIME = new XContentBuilderString("time");
+        static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis");
+        static final XContentBuilderString EXISTS_TOTAL = new XContentBuilderString("exists_total");
+        static final XContentBuilderString EXISTS_TIME = new XContentBuilderString("exists_time");
+        static final XContentBuilderString EXISTS_TIME_IN_MILLIS = new XContentBuilderString("exists_time_in_millis");
+        static final XContentBuilderString MISSING_TOTAL = new XContentBuilderString("missing_total");
+        static final XContentBuilderString MISSING_TIME = new XContentBuilderString("missing_time");
+        static final XContentBuilderString MISSING_TIME_IN_MILLIS = new XContentBuilderString("missing_time_in_millis");
+    }
+
+    public static GetStats readGetStats(StreamInput in) throws IOException {
+        GetStats stats = new GetStats();
+        stats.readFrom(in);
+        return stats;
+    }
+
+    @Override public void readFrom(StreamInput in) throws IOException {
+        existsCount = in.readVLong();
+        existsTimeInMillis = in.readVLong();
+        missingCount = in.readVLong();
+        missingTimeInMillis = in.readVLong();
+    }
+
+    @Override public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(existsCount);
+        out.writeVLong(existsTimeInMillis);
+        out.writeVLong(missingCount);
+        out.writeVLong(missingTimeInMillis);
+    }
+}

modules/elasticsearch/src/main/java/org/elasticsearch/index/get/ShardGetService.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.BytesHolder;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.lucene.document.ResetFieldSelector;
 import org.elasticsearch.common.lucene.uid.UidField;
+import org.elasticsearch.common.metrics.MeanMetric;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.cache.IndexCache;
 import org.elasticsearch.index.engine.Engine;
@@ -63,6 +64,9 @@ public class ShardGetService extends AbstractIndexShardComponent {
 
     private IndexShard indexShard;
 
+    private final MeanMetric existsMetric = new MeanMetric();
+    private final MeanMetric missingMetric = new MeanMetric();
+
     @Inject public ShardGetService(ShardId shardId, @IndexSettings Settings indexSettings, ScriptService scriptService,
                                    MapperService mapperService, IndexCache indexCache) {
         super(shardId, indexSettings);
@@ -71,6 +75,10 @@ public class ShardGetService extends AbstractIndexShardComponent {
         this.indexCache = indexCache;
     }
 
+    public GetStats stats() {
+        return new GetStats(existsMetric.count(), existsMetric.sum(), missingMetric.count(), missingMetric.sum());
+    }
+
     // sadly, to overcome cyclic dep, we need to do this and inject it ourselves...
     public ShardGetService setIndexShard(IndexShard indexShard) {
         this.indexShard = indexShard;
@@ -78,6 +86,17 @@ public class ShardGetService extends AbstractIndexShardComponent {
     }
 
     public GetResult get(String type, String id, String[] gFields, boolean realtime) throws ElasticSearchException {
+        long now = System.nanoTime();
+        GetResult getResult = innerGet(type, id, gFields, realtime);
+        if (getResult.exists()) {
+            existsMetric.inc(System.nanoTime() - now);
+        } else {
+            missingMetric.inc(System.nanoTime() - now);
+        }
+        return getResult;
+    }
+
+    public GetResult innerGet(String type, String id, String[] gFields, boolean realtime) throws ElasticSearchException {
         boolean loadSource = gFields == null || gFields.length > 0;
         Engine.GetResult get = null;
         if (type == null || type.equals("_all")) {

modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java

@@ -26,6 +26,7 @@ import org.elasticsearch.common.util.concurrent.ThreadSafe;
 import org.elasticsearch.index.engine.Engine;
 import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.flush.FlushStats;
+import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.get.ShardGetService;
 import org.elasticsearch.index.indexing.IndexingStats;
 import org.elasticsearch.index.indexing.ShardIndexingService;
@@ -56,6 +57,8 @@ public interface IndexShard extends IndexShardComponent {
 
     IndexingStats indexingStats(String... types);
 
+    GetStats getStats();
+
     MergeStats mergeStats();
 
     RefreshStats refreshStats();

modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java

@@ -47,6 +47,7 @@ import org.elasticsearch.index.engine.EngineException;
 import org.elasticsearch.index.engine.OptimizeFailedEngineException;
 import org.elasticsearch.index.engine.RefreshFailedEngineException;
 import org.elasticsearch.index.flush.FlushStats;
+import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.get.ShardGetService;
 import org.elasticsearch.index.indexing.IndexingStats;
 import org.elasticsearch.index.indexing.ShardIndexingService;
@@ -412,6 +413,10 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
         return indexingService.stats(types);
     }
 
+    @Override public GetStats getStats() {
+        return getService.stats();
+    }
+
     @Override public StoreStats storeStats() {
         try {
             return store.stats();

modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java

@@ -54,6 +54,7 @@ import org.elasticsearch.index.engine.IndexEngineModule;
 import org.elasticsearch.index.flush.FlushStats;
 import org.elasticsearch.index.gateway.IndexGateway;
 import org.elasticsearch.index.gateway.IndexGatewayModule;
+import org.elasticsearch.index.get.GetStats;
 import org.elasticsearch.index.indexing.IndexingStats;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MapperServiceModule;
@@ -178,12 +179,14 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
         DocsStats docsStats = new DocsStats();
         StoreStats storeStats = new StoreStats();
         IndexingStats indexingStats = new IndexingStats();
+        GetStats getStats = new GetStats();
         CacheStats cacheStats = new CacheStats();
         MergeStats mergeStats = new MergeStats();
         RefreshStats refreshStats = new RefreshStats();
         FlushStats flushStats = new FlushStats();
 
         if (includePrevious) {
+            getStats.add(oldShardsStats.getStats);
             indexingStats.add(oldShardsStats.indexingStats);
             mergeStats.add(oldShardsStats.mergeStats);
             refreshStats.add(oldShardsStats.refreshStats);
@@ -194,6 +197,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
             for (IndexShard indexShard : indexService) {
                 storeStats.add(indexShard.storeStats());
                 docsStats.add(indexShard.docStats());
+                getStats.add(indexShard.getStats());
                 indexingStats.add(indexShard.indexingStats());
                 mergeStats.add(indexShard.mergeStats());
                 refreshStats.add(indexShard.refreshStats());
@@ -348,6 +352,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
 
     static class OldShardsStats extends IndicesLifecycle.Listener {
 
+        final GetStats getStats = new GetStats();
         final IndexingStats indexingStats = new IndexingStats();
         final MergeStats mergeStats = new MergeStats();
         final RefreshStats refreshStats = new RefreshStats();
@@ -355,6 +360,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
 
         @Override public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, boolean delete) {
             if (indexShard != null) {
+                getStats.add(indexShard.getStats());
                 indexingStats.add(indexShard.indexingStats(), false);
                 mergeStats.add(indexShard.mergeStats());
                 refreshStats.add(indexShard.refreshStats());

modules/test/integration/src/test/java/org/elasticsearch/test/integration/indices/stats/SimpleIndexStatsTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.test.integration.indices.stats;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.indices.stats.IndicesStats;
+import org.elasticsearch.action.get.GetResponse;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.test.integration.AbstractNodesTests;
 import org.testng.annotations.AfterClass;
@@ -52,7 +53,7 @@ public class SimpleIndexStatsTests extends AbstractNodesTests {
         return client("node2");
     }
 
-    @Test public void simpleIndexTemplateTests() throws Exception {
+    @Test public void simpleStats() throws Exception {
         // rely on 1 replica for this tests
         client.admin().indices().prepareCreate("test1").execute().actionGet();
         client.admin().indices().prepareCreate("test2").execute().actionGet();
@@ -109,5 +110,24 @@ public class SimpleIndexStatsTests extends AbstractNodesTests {
         assertThat(stats.primaries().indexing().typeStats().get("type1").indexCount(), equalTo(1l));
         assertThat(stats.primaries().indexing().typeStats().get("type").indexCount(), equalTo(1l));
         assertThat(stats.primaries().indexing().typeStats().get("type2"), nullValue());
+
+        assertThat(stats.total().get().count(), equalTo(0l));
+        // check get
+        GetResponse getResponse = client.prepareGet("test1", "type1", "1").execute().actionGet();
+        assertThat(getResponse.exists(), equalTo(true));
+
+        stats = client.admin().indices().prepareStats().execute().actionGet();
+        assertThat(stats.total().get().count(), equalTo(1l));
+        assertThat(stats.total().get().existsCount(), equalTo(1l));
+        assertThat(stats.total().get().missingCount(), equalTo(0l));
+
+        // missing get
+        getResponse = client.prepareGet("test1", "type1", "2").execute().actionGet();
+        assertThat(getResponse.exists(), equalTo(false));
+
+        stats = client.admin().indices().prepareStats().execute().actionGet();
+        assertThat(stats.total().get().count(), equalTo(2l));
+        assertThat(stats.total().get().existsCount(), equalTo(1l));
+        assertThat(stats.total().get().missingCount(), equalTo(1l));
     }
 }