[Stats] Adds counter for failed indexing requests

Adds the stats to the _cat/indices, _cat/shards, and the _stats apis.

Closes #8938

core/src/main/java/org/elasticsearch/index/indexing/IndexingStats.java

@@ -42,6 +42,7 @@ public class IndexingStats implements Streamable, ToXContent {
         private long indexCount;
         private long indexTimeInMillis;
         private long indexCurrent;
+        private long indexFailedCount;
 
         private long deleteCount;
         private long deleteTimeInMillis;
@@ -56,10 +57,11 @@ public class IndexingStats implements Streamable, ToXContent {
 
         }
 
-        public Stats(long indexCount, long indexTimeInMillis, long indexCurrent, long deleteCount, long deleteTimeInMillis, long deleteCurrent, long noopUpdateCount, boolean isThrottled, long throttleTimeInMillis) {
+        public Stats(long indexCount, long indexTimeInMillis, long indexCurrent, long indexFailedCount, long deleteCount, long deleteTimeInMillis, long deleteCurrent, long noopUpdateCount, boolean isThrottled, long throttleTimeInMillis) {
             this.indexCount = indexCount;
             this.indexTimeInMillis = indexTimeInMillis;
             this.indexCurrent = indexCurrent;
+            this.indexFailedCount = indexFailedCount;
             this.deleteCount = deleteCount;
             this.deleteTimeInMillis = deleteTimeInMillis;
             this.deleteCurrent = deleteCurrent;
@@ -72,6 +74,7 @@ public class IndexingStats implements Streamable, ToXContent {
             indexCount += stats.indexCount;
             indexTimeInMillis += stats.indexTimeInMillis;
             indexCurrent += stats.indexCurrent;
+            indexFailedCount += stats.indexFailedCount;
 
             deleteCount += stats.deleteCount;
             deleteTimeInMillis += stats.deleteTimeInMillis;
@@ -88,6 +91,10 @@ public class IndexingStats implements Streamable, ToXContent {
             return indexCount;
         }
 
+        public long getIndexFailedCount() {
+            return indexFailedCount;
+        }
+
         public TimeValue getIndexTime() {
             return new TimeValue(indexTimeInMillis);
         }
@@ -156,6 +163,10 @@ public class IndexingStats implements Streamable, ToXContent {
             indexTimeInMillis = in.readVLong();
             indexCurrent = in.readVLong();
 
+            if(in.getVersion().onOrAfter(Version.V_2_1_0)){
+                indexFailedCount = in.readVLong();
+            }
+
             deleteCount = in.readVLong();
             deleteTimeInMillis = in.readVLong();
             deleteCurrent = in.readVLong();
@@ -170,6 +181,10 @@ public class IndexingStats implements Streamable, ToXContent {
             out.writeVLong(indexTimeInMillis);
             out.writeVLong(indexCurrent);
 
+            if(out.getVersion().onOrAfter(Version.V_2_1_0)) {
+                out.writeVLong(indexFailedCount);
+            }
+
             out.writeVLong(deleteCount);
             out.writeVLong(deleteTimeInMillis);
             out.writeVLong(deleteCurrent);
@@ -184,6 +199,7 @@ public class IndexingStats implements Streamable, ToXContent {
             builder.field(Fields.INDEX_TOTAL, indexCount);
             builder.timeValueField(Fields.INDEX_TIME_IN_MILLIS, Fields.INDEX_TIME, indexTimeInMillis);
             builder.field(Fields.INDEX_CURRENT, indexCurrent);
+            builder.field(Fields.INDEX_FAILED, indexFailedCount);
 
             builder.field(Fields.DELETE_TOTAL, deleteCount);
             builder.timeValueField(Fields.DELETE_TIME_IN_MILLIS, Fields.DELETE_TIME, deleteTimeInMillis);
@@ -268,6 +284,7 @@ public class IndexingStats implements Streamable, ToXContent {
         static final XContentBuilderString INDEX_TIME = new XContentBuilderString("index_time");
         static final XContentBuilderString INDEX_TIME_IN_MILLIS = new XContentBuilderString("index_time_in_millis");
         static final XContentBuilderString INDEX_CURRENT = new XContentBuilderString("index_current");
+        static final XContentBuilderString INDEX_FAILED = new XContentBuilderString("index_failed");
         static final XContentBuilderString DELETE_TOTAL = new XContentBuilderString("delete_total");
         static final XContentBuilderString DELETE_TIME = new XContentBuilderString("delete_time");
         static final XContentBuilderString DELETE_TIME_IN_MILLIS = new XContentBuilderString("delete_time_in_millis");

core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java

@@ -171,6 +171,8 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
     public void postIndex(Engine.Index index, Throwable ex) {
         totalStats.indexCurrent.dec();
         typeStats(index.type()).indexCurrent.dec();
+        totalStats.indexFailed.inc();
+        typeStats(index.type()).indexFailed.inc();
     }
 
     public Engine.Delete preDelete(Engine.Delete delete) {
@@ -256,6 +258,7 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
         public final MeanMetric indexMetric = new MeanMetric();
         public final MeanMetric deleteMetric = new MeanMetric();
         public final CounterMetric indexCurrent = new CounterMetric();
+        public final CounterMetric indexFailed = new CounterMetric();
         public final CounterMetric deleteCurrent = new CounterMetric();
         public final CounterMetric noopUpdates = new CounterMetric();
         public final CounterMetric throttleTimeMillisMetric = new CounterMetric();
@@ -272,7 +275,7 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
                 }
             }
             return new IndexingStats.Stats(
-                    indexMetric.count(), TimeUnit.NANOSECONDS.toMillis(indexMetric.sum()), indexCurrent.count(),
+                    indexMetric.count(), TimeUnit.NANOSECONDS.toMillis(indexMetric.sum()), indexCurrent.count(), indexFailed.count(),
                     deleteMetric.count(), TimeUnit.NANOSECONDS.toMillis(deleteMetric.sum()), deleteCurrent.count(),
                     noopUpdates.count(), isThrottled, TimeUnit.MILLISECONDS.toMillis(throttleTimeMillisMetric.count() + TimeValue.nsecToMSec(currentThrottleNS)));
         }

core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -192,6 +192,9 @@ public class RestIndicesAction extends AbstractCatAction {
         table.addCell("indexing.index_total", "sibling:pri;alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
         table.addCell("pri.indexing.index_total", "default:false;text-align:right;desc:number of indexing ops");
 
+        table.addCell("indexing.index_failed", "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
+        table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops");
+
         table.addCell("merges.current", "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
         table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges");
 
@@ -403,6 +406,9 @@ public class RestIndicesAction extends AbstractCatAction {
             table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexCount());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexCount());
 
+            table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexFailedCount());
+            table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexFailedCount());
+
             table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrent());
             table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentSize());
 

core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java

@@ -166,6 +166,7 @@ public class RestNodesAction extends AbstractCatAction {
         table.addCell("indexing.index_current", "alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops");
         table.addCell("indexing.index_time", "alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing");
         table.addCell("indexing.index_total", "alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
+        table.addCell("indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
 
         table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
         table.addCell("merges.current_docs", "alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs");
@@ -300,6 +301,7 @@ public class RestNodesAction extends AbstractCatAction {
             table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCurrent());
             table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexTime());
             table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCount());
+            table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedCount());
 
             MergeStats mergeStats = indicesStats == null ? null : indicesStats.getMerge();
             table.addCell(mergeStats == null ? null : mergeStats.getCurrent());

core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java

@@ -121,6 +121,7 @@ public class RestShardsAction extends AbstractCatAction {
         table.addCell("indexing.index_current", "alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops");
         table.addCell("indexing.index_time", "alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing");
         table.addCell("indexing.index_total", "alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
+        table.addCell("indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
 
         table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
         table.addCell("merges.current_docs", "alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs");
@@ -255,6 +256,7 @@ public class RestShardsAction extends AbstractCatAction {
             table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexCurrent());
             table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexTime());
             table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexCount());
+            table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexFailedCount());
 
             table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrent());
             table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrentNumDocs());

core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java

@@ -19,7 +19,9 @@
 
 package org.elasticsearch.indices.stats;
 
+import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.cache.IndexCacheModule;
+import org.elasticsearch.index.engine.VersionConflictEngineException;
 import org.elasticsearch.index.shard.MergeSchedulerConfig;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util.Version;
@@ -41,7 +43,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.cache.query.QueryCacheStats;
-import org.elasticsearch.index.cache.query.index.IndexQueryCache;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.shard.MergePolicyConfig;
 import org.elasticsearch.index.store.IndexStore;
@@ -368,7 +369,6 @@ public class IndexStatsIT extends ESIntegTestCase {
         client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").execute().actionGet();
         client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").execute().actionGet();
         client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").execute().actionGet();
-
         refresh();
 
         NumShards test1 = getNumShards("test1");
@@ -381,6 +381,7 @@ public class IndexStatsIT extends ESIntegTestCase {
         assertThat(stats.getPrimaries().getDocs().getCount(), equalTo(3l));
         assertThat(stats.getTotal().getDocs().getCount(), equalTo(totalExpectedWrites));
         assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexCount(), equalTo(3l));
+        assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(0l));
         assertThat(stats.getPrimaries().getIndexing().getTotal().isThrottled(), equalTo(false));
         assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0l));
         assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(totalExpectedWrites));
@@ -423,10 +424,12 @@ public class IndexStatsIT extends ESIntegTestCase {
         stats = client().admin().indices().prepareStats().setTypes("type1", "type").execute().actionGet();
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCount(), equalTo(1l));
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type").getIndexCount(), equalTo(1l));
+        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(0l));
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2"), nullValue());
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCurrent(), equalTo(0l));
         assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getDeleteCurrent(), equalTo(0l));
 
+
         assertThat(stats.getTotal().getGet().getCount(), equalTo(0l));
         // check get
         GetResponse getResponse = client().prepareGet("test1", "type1", "1").execute().actionGet();
@@ -462,6 +465,30 @@ public class IndexStatsIT extends ESIntegTestCase {
         assertThat(stats.getTotal().getIndexing(), nullValue());
         assertThat(stats.getTotal().getGet(), nullValue());
         assertThat(stats.getTotal().getSearch(), nullValue());
+
+        // index failed
+        try {
+            client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").setVersion(1)
+                    .setVersionType(VersionType.EXTERNAL).execute().actionGet();
+            fail("Expected a version conflict");
+        } catch (VersionConflictEngineException e) {}
+        try {
+            client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").setVersion(1)
+                    .setVersionType(VersionType.EXTERNAL).execute().actionGet();
+            fail("Expected a version conflict");
+        } catch (VersionConflictEngineException e) {}
+        try {
+            client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").setVersion(1)
+                    .setVersionType(VersionType.EXTERNAL).execute().actionGet();
+            fail("Expected a version conflict");
+        } catch (VersionConflictEngineException e) {}
+
+        stats = client().admin().indices().prepareStats().setTypes("type1", "type2").execute().actionGet();
+        assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(2l));
+        assertThat(stats.getIndex("test2").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(1l));
+        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(1L));
+        assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2").getIndexFailedCount(), equalTo(1L));
+        assertThat(stats.getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(3L));
     }
 
     @Test