honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Stats] Adds counter for failed indexing requests 

Adds the stats to the _cat/indices, _cat/shards, and the _stats apis.

Closes #8938
This commit is contained in:
André Carvalho 2015-08-25 23:33:19 -03:00 committed by Nik Everett
parent e8834cc78c
commit 9802c38fa7
6 changed files with 61 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
core/src/main/java/org/elasticsearch/index/indexing/IndexingStats.java
View File

@ -42,6 +42,7 @@ public class IndexingStats implements Streamable, ToXContent {
private long indexCount;
private long indexTimeInMillis;
private long indexCurrent;
private long indexFailedCount;
private long deleteCount;
private long deleteTimeInMillis;
@ -56,10 +57,11 @@ public class IndexingStats implements Streamable, ToXContent {
}
public Stats(long indexCount, long indexTimeInMillis, long indexCurrent, long deleteCount, long deleteTimeInMillis, long deleteCurrent, long noopUpdateCount, boolean isThrottled, long throttleTimeInMillis) {
public Stats(long indexCount, long indexTimeInMillis, long indexCurrent, long indexFailedCount, long deleteCount, long deleteTimeInMillis, long deleteCurrent, long noopUpdateCount, boolean isThrottled, long throttleTimeInMillis) {
this.indexCount = indexCount;
this.indexTimeInMillis = indexTimeInMillis;
this.indexCurrent = indexCurrent;
this.indexFailedCount = indexFailedCount;
this.deleteCount = deleteCount;
this.deleteTimeInMillis = deleteTimeInMillis;
this.deleteCurrent = deleteCurrent;
@ -72,6 +74,7 @@ public class IndexingStats implements Streamable, ToXContent {
indexCount += stats.indexCount;
indexTimeInMillis += stats.indexTimeInMillis;
indexCurrent += stats.indexCurrent;
indexFailedCount += stats.indexFailedCount;
deleteCount += stats.deleteCount;
deleteTimeInMillis += stats.deleteTimeInMillis;
@ -88,6 +91,10 @@ public class IndexingStats implements Streamable, ToXContent {
return indexCount;
}
public long getIndexFailedCount() {
return indexFailedCount;
}
public TimeValue getIndexTime() {
return new TimeValue(indexTimeInMillis);
}
@ -156,6 +163,10 @@ public class IndexingStats implements Streamable, ToXContent {
indexTimeInMillis = in.readVLong();
indexCurrent = in.readVLong();
if(in.getVersion().onOrAfter(Version.V_2_1_0)){
indexFailedCount = in.readVLong();
}
deleteCount = in.readVLong();
deleteTimeInMillis = in.readVLong();
deleteCurrent = in.readVLong();
@ -170,6 +181,10 @@ public class IndexingStats implements Streamable, ToXContent {
out.writeVLong(indexTimeInMillis);
out.writeVLong(indexCurrent);
if(out.getVersion().onOrAfter(Version.V_2_1_0)) {
out.writeVLong(indexFailedCount);
}
out.writeVLong(deleteCount);
out.writeVLong(deleteTimeInMillis);
out.writeVLong(deleteCurrent);
@ -184,6 +199,7 @@ public class IndexingStats implements Streamable, ToXContent {
builder.field(Fields.INDEX_TOTAL, indexCount);
builder.timeValueField(Fields.INDEX_TIME_IN_MILLIS, Fields.INDEX_TIME, indexTimeInMillis);
builder.field(Fields.INDEX_CURRENT, indexCurrent);
builder.field(Fields.INDEX_FAILED, indexFailedCount);
builder.field(Fields.DELETE_TOTAL, deleteCount);
builder.timeValueField(Fields.DELETE_TIME_IN_MILLIS, Fields.DELETE_TIME, deleteTimeInMillis);
@ -268,6 +284,7 @@ public class IndexingStats implements Streamable, ToXContent {
static final XContentBuilderString INDEX_TIME = new XContentBuilderString("index_time");
static final XContentBuilderString INDEX_TIME_IN_MILLIS = new XContentBuilderString("index_time_in_millis");
static final XContentBuilderString INDEX_CURRENT = new XContentBuilderString("index_current");
static final XContentBuilderString INDEX_FAILED = new XContentBuilderString("index_failed");
static final XContentBuilderString DELETE_TOTAL = new XContentBuilderString("delete_total");
static final XContentBuilderString DELETE_TIME = new XContentBuilderString("delete_time");
static final XContentBuilderString DELETE_TIME_IN_MILLIS = new XContentBuilderString("delete_time_in_millis");

5
core/src/main/java/org/elasticsearch/index/indexing/ShardIndexingService.java
View File

@ -171,6 +171,8 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
public void postIndex(Engine.Index index, Throwable ex) {
totalStats.indexCurrent.dec();
typeStats(index.type()).indexCurrent.dec();
totalStats.indexFailed.inc();
typeStats(index.type()).indexFailed.inc();
}
public Engine.Delete preDelete(Engine.Delete delete) {
@ -256,6 +258,7 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
public final MeanMetric indexMetric = new MeanMetric();
public final MeanMetric deleteMetric = new MeanMetric();
public final CounterMetric indexCurrent = new CounterMetric();
public final CounterMetric indexFailed = new CounterMetric();
public final CounterMetric deleteCurrent = new CounterMetric();
public final CounterMetric noopUpdates = new CounterMetric();
public final CounterMetric throttleTimeMillisMetric = new CounterMetric();
@ -272,7 +275,7 @@ public class ShardIndexingService extends AbstractIndexShardComponent {
}
}
return new IndexingStats.Stats(
indexMetric.count(), TimeUnit.NANOSECONDS.toMillis(indexMetric.sum()), indexCurrent.count(),
indexMetric.count(), TimeUnit.NANOSECONDS.toMillis(indexMetric.sum()), indexCurrent.count(), indexFailed.count(),
deleteMetric.count(), TimeUnit.NANOSECONDS.toMillis(deleteMetric.sum()), deleteCurrent.count(),
noopUpdates.count(), isThrottled, TimeUnit.MILLISECONDS.toMillis(throttleTimeMillisMetric.count() + TimeValue.nsecToMSec(currentThrottleNS)));
}

6
core/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java
View File

@ -192,6 +192,9 @@ public class RestIndicesAction extends AbstractCatAction {
table.addCell("indexing.index_total", "sibling:pri;alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
table.addCell("pri.indexing.index_total", "default:false;text-align:right;desc:number of indexing ops");
table.addCell("indexing.index_failed", "sibling:pri;alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
table.addCell("pri.indexing.index_failed", "default:false;text-align:right;desc:number of failed indexing ops");
table.addCell("merges.current", "sibling:pri;alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
table.addCell("pri.merges.current", "default:false;text-align:right;desc:number of current merges");
@ -403,6 +406,9 @@ public class RestIndicesAction extends AbstractCatAction {
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getIndexing().getTotal().getIndexFailedCount());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getIndexing().getTotal().getIndexFailedCount());
table.addCell(indexStats == null ? null : indexStats.getTotal().getMerge().getCurrent());
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getMerge().getCurrentSize());

2
core/src/main/java/org/elasticsearch/rest/action/cat/RestNodesAction.java
View File

@ -166,6 +166,7 @@ public class RestNodesAction extends AbstractCatAction {
table.addCell("indexing.index_current", "alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops");
table.addCell("indexing.index_time", "alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing");
table.addCell("indexing.index_total", "alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
table.addCell("indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
table.addCell("merges.current_docs", "alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs");
@ -300,6 +301,7 @@ public class RestNodesAction extends AbstractCatAction {
table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCurrent());
table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexTime());
table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCount());
table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedCount());
MergeStats mergeStats = indicesStats == null ? null : indicesStats.getMerge();
table.addCell(mergeStats == null ? null : mergeStats.getCurrent());

2
core/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java
View File

@ -121,6 +121,7 @@ public class RestShardsAction extends AbstractCatAction {
table.addCell("indexing.index_current", "alias:iic,indexingIndexCurrent;default:false;text-align:right;desc:number of current indexing ops");
table.addCell("indexing.index_time", "alias:iiti,indexingIndexTime;default:false;text-align:right;desc:time spent in indexing");
table.addCell("indexing.index_total", "alias:iito,indexingIndexTotal;default:false;text-align:right;desc:number of indexing ops");
table.addCell("indexing.index_failed", "alias:iif,indexingIndexFailed;default:false;text-align:right;desc:number of failed indexing ops");
table.addCell("merges.current", "alias:mc,mergesCurrent;default:false;text-align:right;desc:number of current merges");
table.addCell("merges.current_docs", "alias:mcd,mergesCurrentDocs;default:false;text-align:right;desc:number of current merging docs");
@ -255,6 +256,7 @@ public class RestShardsAction extends AbstractCatAction {
table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexCurrent());
table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexTime());
table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexCount());
table.addCell(shardStats == null ? null : shardStats.getIndexing().getTotal().getIndexFailedCount());
table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrent());
table.addCell(shardStats == null ? null : shardStats.getMerge().getCurrentNumDocs());

31
core/src/test/java/org/elasticsearch/indices/stats/IndexStatsIT.java
View File

@ -19,7 +19,9 @@
package org.elasticsearch.indices.stats;
import org.elasticsearch.index.VersionType;
import org.elasticsearch.index.cache.IndexCacheModule;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.shard.MergeSchedulerConfig;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util.Version;
@ -41,7 +43,6 @@ import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.query.QueryCacheStats;
import org.elasticsearch.index.cache.query.index.IndexQueryCache;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.shard.MergePolicyConfig;
import org.elasticsearch.index.store.IndexStore;
@ -368,7 +369,6 @@ public class IndexStatsIT extends ESIntegTestCase {
client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").execute().actionGet();
client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").execute().actionGet();
client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").execute().actionGet();
refresh();
NumShards test1 = getNumShards("test1");
@ -381,6 +381,7 @@ public class IndexStatsIT extends ESIntegTestCase {
assertThat(stats.getPrimaries().getDocs().getCount(), equalTo(3l));
assertThat(stats.getTotal().getDocs().getCount(), equalTo(totalExpectedWrites));
assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexCount(), equalTo(3l));
assertThat(stats.getPrimaries().getIndexing().getTotal().getIndexFailedCount(), equalTo(0l));
assertThat(stats.getPrimaries().getIndexing().getTotal().isThrottled(), equalTo(false));
assertThat(stats.getPrimaries().getIndexing().getTotal().getThrottleTimeInMillis(), equalTo(0l));
assertThat(stats.getTotal().getIndexing().getTotal().getIndexCount(), equalTo(totalExpectedWrites));
@ -423,10 +424,12 @@ public class IndexStatsIT extends ESIntegTestCase {
stats = client().admin().indices().prepareStats().setTypes("type1", "type").execute().actionGet();
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCount(), equalTo(1l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type").getIndexCount(), equalTo(1l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(0l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2"), nullValue());
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexCurrent(), equalTo(0l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getDeleteCurrent(), equalTo(0l));
assertThat(stats.getTotal().getGet().getCount(), equalTo(0l));
// check get
GetResponse getResponse = client().prepareGet("test1", "type1", "1").execute().actionGet();
@ -462,6 +465,30 @@ public class IndexStatsIT extends ESIntegTestCase {
assertThat(stats.getTotal().getIndexing(), nullValue());
assertThat(stats.getTotal().getGet(), nullValue());
assertThat(stats.getTotal().getSearch(), nullValue());
// index failed
try {
client().prepareIndex("test1", "type1", Integer.toString(1)).setSource("field", "value").setVersion(1)
.setVersionType(VersionType.EXTERNAL).execute().actionGet();
fail("Expected a version conflict");
} catch (VersionConflictEngineException e) {}
try {
client().prepareIndex("test1", "type2", Integer.toString(1)).setSource("field", "value").setVersion(1)
.setVersionType(VersionType.EXTERNAL).execute().actionGet();
fail("Expected a version conflict");
} catch (VersionConflictEngineException e) {}
try {
client().prepareIndex("test2", "type", Integer.toString(1)).setSource("field", "value").setVersion(1)
.setVersionType(VersionType.EXTERNAL).execute().actionGet();
fail("Expected a version conflict");
} catch (VersionConflictEngineException e) {}
stats = client().admin().indices().prepareStats().setTypes("type1", "type2").execute().actionGet();
assertThat(stats.getIndex("test1").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(2l));
assertThat(stats.getIndex("test2").getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(1l));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type1").getIndexFailedCount(), equalTo(1L));
assertThat(stats.getPrimaries().getIndexing().getTypeStats().get("type2").getIndexFailedCount(), equalTo(1L));
assertThat(stats.getTotal().getIndexing().getTotal().getIndexFailedCount(), equalTo(3L));
}
@Test