honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Node / Indices Stats: Add get stats (including missing / exists), closes #1288.

This commit is contained in:
Shay Banon 2011-08-30 20:56:39 +03:00
parent 32b64fc9a3
commit 1add5ce566
9 changed files with 281 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.merge.MergeStats;
import org.elasticsearch.index.refresh.RefreshStats;
@ -44,6 +45,8 @@ public class CommonStats implements Streamable, ToXContent {
@Nullable IndexingStats indexing;
@Nullable GetStats get;
@Nullable MergeStats merge;
@Nullable RefreshStats refresh;
@ -75,6 +78,14 @@ public class CommonStats implements Streamable, ToXContent {
} else {
indexing.add(stats.indexing());
}
if (get == null) {
if (stats.get() != null) {
get = new GetStats();
get.add(stats.get());
}
} else {
get.add(stats.get());
}
if (merge == null) {
if (stats.merge() != null) {
merge = new MergeStats();
@ -125,6 +136,14 @@ public class CommonStats implements Streamable, ToXContent {
return indexing;
}
@Nullable public GetStats get() {
return get;
}
@Nullable public GetStats getGet() {
return get;
}
@Nullable public MergeStats merge() {
return merge;
}
@ -165,6 +184,9 @@ public class CommonStats implements Streamable, ToXContent {
if (in.readBoolean()) {
indexing = IndexingStats.readIndexingStats(in);
}
if (in.readBoolean()) {
get = GetStats.readGetStats(in);
}
if (in.readBoolean()) {
merge = MergeStats.readMergeStats(in);
}
@ -195,6 +217,12 @@ public class CommonStats implements Streamable, ToXContent {
out.writeBoolean(true);
indexing.writeTo(out);
}
if (get == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
get.writeTo(out);
}
if (merge == null) {
out.writeBoolean(false);
} else {
@ -226,6 +254,9 @@ public class CommonStats implements Streamable, ToXContent {
if (indexing != null) {
indexing.toXContent(builder, params);
}
if (get != null) {
get.toXContent(builder, params);
}
if (merge != null) {
merge.toXContent(builder, params);
}

13
modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java
View File

@ -39,6 +39,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
private boolean docs = true;
private boolean store = true;
private boolean indexing = true;
private boolean get = true;
private boolean merge = false;
private boolean refresh = false;
private boolean flush = false;
@ -55,6 +56,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
public IndicesStatsRequest clear() {
docs = false;
store = false;
get = false;
indexing = false;
merge = false;
refresh = false;
@ -106,6 +108,15 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
return this.indexing;
}
public IndicesStatsRequest get(boolean get) {
this.get = get;
return this;
}
public boolean get() {
return this.get;
}
public IndicesStatsRequest merge(boolean merge) {
this.merge = merge;
return this;
@ -138,6 +149,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
out.writeBoolean(docs);
out.writeBoolean(store);
out.writeBoolean(indexing);
out.writeBoolean(get);
out.writeBoolean(merge);
out.writeBoolean(flush);
out.writeBoolean(refresh);
@ -156,6 +168,7 @@ public class IndicesStatsRequest extends BroadcastOperationRequest {
docs = in.readBoolean();
store = in.readBoolean();
indexing = in.readBoolean();
get = in.readBoolean();
merge = in.readBoolean();
flush = in.readBoolean();
refresh = in.readBoolean();

3
modules/elasticsearch/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java
View File

@ -135,6 +135,9 @@ public class TransportIndicesStatsAction extends TransportBroadcastOperationActi
if (request.request.indexing()) {
stats.stats.indexing = indexShard.indexingStats(request.request.types());
}
if (request.request.get()) {
stats.stats.get = indexShard.getStats();
}
if (request.request.merge()) {
stats.stats.merge = indexShard.mergeStats();
}

180
modules/elasticsearch/src/main/java/org/elasticsearch/index/get/GetStats.java Normal file
View File

@ -0,0 +1,180 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.get;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentBuilderString;
import java.io.IOException;
/**
*/
public class GetStats implements Streamable, ToXContent {
private long existsCount;
private long existsTimeInMillis;
private long missingCount;
private long missingTimeInMillis;
public GetStats() {
}
public GetStats(long existsCount, long existsTimeInMillis, long missingCount, long missingTimeInMillis) {
this.existsCount = existsCount;
this.existsTimeInMillis = existsTimeInMillis;
this.missingCount = missingCount;
this.missingTimeInMillis = missingTimeInMillis;
}
public void add(GetStats stats) {
if (stats == null) {
return;
}
existsCount += stats.existsCount;
existsTimeInMillis += stats.existsTimeInMillis;
missingCount += stats.missingCount;
missingTimeInMillis += stats.missingTimeInMillis;
}
public long count() {
return existsCount + missingCount;
}
public long getCount() {
return count();
}
public long timeInMillis() {
return existsTimeInMillis + missingTimeInMillis;
}
public long getTimeInMillis() {
return timeInMillis();
}
public TimeValue time() {
return new TimeValue(timeInMillis());
}
public TimeValue getTime() {
return time();
}
public long existsCount() {
return this.existsCount;
}
public long getExistsCount() {
return this.existsCount;
}
public long existsTimeInMillis() {
return this.existsTimeInMillis;
}
public long getExistsTimeInMillis() {
return this.existsTimeInMillis;
}
public TimeValue existsTime() {
return new TimeValue(existsTimeInMillis);
}
public TimeValue getExistsTime() {
return existsTime();
}
public long missingCount() {
return this.missingCount;
}
public long getMissingCount() {
return this.missingCount;
}
public long missingTimeInMillis() {
return this.missingTimeInMillis;
}
public long getMissingTimeInMillis() {
return this.missingTimeInMillis;
}
public TimeValue missingTime() {
return new TimeValue(missingTimeInMillis);
}
public TimeValue getMissingTime() {
return missingTime();
}
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.GET);
builder.field(Fields.TOTAL, count());
builder.field(Fields.TIME, time().toString());
builder.field(Fields.TIME_IN_MILLIS, timeInMillis());
builder.field(Fields.EXISTS_TOTAL, existsCount);
builder.field(Fields.EXISTS_TIME, existsTime().toString());
builder.field(Fields.EXISTS_TIME_IN_MILLIS, existsTimeInMillis);
builder.field(Fields.MISSING_TOTAL, missingCount);
builder.field(Fields.MISSING_TIME, missingTime().toString());
builder.field(Fields.MISSING_TIME_IN_MILLIS, missingTimeInMillis);
builder.endObject();
return builder;
}
static final class Fields {
static final XContentBuilderString GET = new XContentBuilderString("get");
static final XContentBuilderString TOTAL = new XContentBuilderString("total");
static final XContentBuilderString TIME = new XContentBuilderString("time");
static final XContentBuilderString TIME_IN_MILLIS = new XContentBuilderString("time_in_millis");
static final XContentBuilderString EXISTS_TOTAL = new XContentBuilderString("exists_total");
static final XContentBuilderString EXISTS_TIME = new XContentBuilderString("exists_time");
static final XContentBuilderString EXISTS_TIME_IN_MILLIS = new XContentBuilderString("exists_time_in_millis");
static final XContentBuilderString MISSING_TOTAL = new XContentBuilderString("missing_total");
static final XContentBuilderString MISSING_TIME = new XContentBuilderString("missing_time");
static final XContentBuilderString MISSING_TIME_IN_MILLIS = new XContentBuilderString("missing_time_in_millis");
}
public static GetStats readGetStats(StreamInput in) throws IOException {
GetStats stats = new GetStats();
stats.readFrom(in);
return stats;
}
@Override public void readFrom(StreamInput in) throws IOException {
existsCount = in.readVLong();
existsTimeInMillis = in.readVLong();
missingCount = in.readVLong();
missingTimeInMillis = in.readVLong();
}
@Override public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(existsCount);
out.writeVLong(existsTimeInMillis);
out.writeVLong(missingCount);
out.writeVLong(missingTimeInMillis);
}
}

19
modules/elasticsearch/src/main/java/org/elasticsearch/index/get/ShardGetService.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.BytesHolder;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.document.ResetFieldSelector;
import org.elasticsearch.common.lucene.uid.UidField;
import org.elasticsearch.common.metrics.MeanMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.cache.IndexCache;
import org.elasticsearch.index.engine.Engine;
@ -63,6 +64,9 @@ public class ShardGetService extends AbstractIndexShardComponent {
private IndexShard indexShard;
private final MeanMetric existsMetric = new MeanMetric();
private final MeanMetric missingMetric = new MeanMetric();
@Inject public ShardGetService(ShardId shardId, @IndexSettings Settings indexSettings, ScriptService scriptService,
MapperService mapperService, IndexCache indexCache) {
super(shardId, indexSettings);
@ -71,6 +75,10 @@ public class ShardGetService extends AbstractIndexShardComponent {
this.indexCache = indexCache;
}
public GetStats stats() {
return new GetStats(existsMetric.count(), existsMetric.sum(), missingMetric.count(), missingMetric.sum());
}
// sadly, to overcome cyclic dep, we need to do this and inject it ourselves...
public ShardGetService setIndexShard(IndexShard indexShard) {
this.indexShard = indexShard;
@ -78,6 +86,17 @@ public class ShardGetService extends AbstractIndexShardComponent {
}
public GetResult get(String type, String id, String[] gFields, boolean realtime) throws ElasticSearchException {
long now = System.nanoTime();
GetResult getResult = innerGet(type, id, gFields, realtime);
if (getResult.exists()) {
existsMetric.inc(System.nanoTime() - now);
} else {
missingMetric.inc(System.nanoTime() - now);
}
return getResult;
}
public GetResult innerGet(String type, String id, String[] gFields, boolean realtime) throws ElasticSearchException {
boolean loadSource = gFields == null || gFields.length > 0;
Engine.GetResult get = null;
if (type == null || type.equals("_all")) {

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/service/IndexShard.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.common.util.concurrent.ThreadSafe;
import org.elasticsearch.index.engine.Engine;
import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.get.ShardGetService;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.indexing.ShardIndexingService;
@ -56,6 +57,8 @@ public interface IndexShard extends IndexShardComponent {
IndexingStats indexingStats(String... types);
GetStats getStats();
MergeStats mergeStats();
RefreshStats refreshStats();

5
modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/service/InternalIndexShard.java
View File

@ -47,6 +47,7 @@ import org.elasticsearch.index.engine.EngineException;
import org.elasticsearch.index.engine.OptimizeFailedEngineException;
import org.elasticsearch.index.engine.RefreshFailedEngineException;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.get.ShardGetService;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.indexing.ShardIndexingService;
@ -412,6 +413,10 @@ public class InternalIndexShard extends AbstractIndexShardComponent implements I
return indexingService.stats(types);
}
@Override public GetStats getStats() {
return getService.stats();
}
@Override public StoreStats storeStats() {
try {
return store.stats();

6
modules/elasticsearch/src/main/java/org/elasticsearch/indices/InternalIndicesService.java
View File

@ -54,6 +54,7 @@ import org.elasticsearch.index.engine.IndexEngineModule;
import org.elasticsearch.index.flush.FlushStats;
import org.elasticsearch.index.gateway.IndexGateway;
import org.elasticsearch.index.gateway.IndexGatewayModule;
import org.elasticsearch.index.get.GetStats;
import org.elasticsearch.index.indexing.IndexingStats;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MapperServiceModule;
@ -178,12 +179,14 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
DocsStats docsStats = new DocsStats();
StoreStats storeStats = new StoreStats();
IndexingStats indexingStats = new IndexingStats();
GetStats getStats = new GetStats();
CacheStats cacheStats = new CacheStats();
MergeStats mergeStats = new MergeStats();
RefreshStats refreshStats = new RefreshStats();
FlushStats flushStats = new FlushStats();
if (includePrevious) {
getStats.add(oldShardsStats.getStats);
indexingStats.add(oldShardsStats.indexingStats);
mergeStats.add(oldShardsStats.mergeStats);
refreshStats.add(oldShardsStats.refreshStats);
@ -194,6 +197,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
for (IndexShard indexShard : indexService) {
storeStats.add(indexShard.storeStats());
docsStats.add(indexShard.docStats());
getStats.add(indexShard.getStats());
indexingStats.add(indexShard.indexingStats());
mergeStats.add(indexShard.mergeStats());
refreshStats.add(indexShard.refreshStats());
@ -348,6 +352,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
static class OldShardsStats extends IndicesLifecycle.Listener {
final GetStats getStats = new GetStats();
final IndexingStats indexingStats = new IndexingStats();
final MergeStats mergeStats = new MergeStats();
final RefreshStats refreshStats = new RefreshStats();
@ -355,6 +360,7 @@ public class InternalIndicesService extends AbstractLifecycleComponent<IndicesSe
@Override public synchronized void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, boolean delete) {
if (indexShard != null) {
getStats.add(indexShard.getStats());
indexingStats.add(indexShard.indexingStats(), false);
mergeStats.add(indexShard.mergeStats());
refreshStats.add(indexShard.refreshStats());

22
modules/test/integration/src/test/java/org/elasticsearch/test/integration/indices/stats/SimpleIndexStatsTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.test.integration.indices.stats;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.stats.IndicesStats;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.test.integration.AbstractNodesTests;
import org.testng.annotations.AfterClass;
@ -52,7 +53,7 @@ public class SimpleIndexStatsTests extends AbstractNodesTests {
return client("node2");
}
@Test public void simpleIndexTemplateTests() throws Exception {
@Test public void simpleStats() throws Exception {
// rely on 1 replica for this tests
client.admin().indices().prepareCreate("test1").execute().actionGet();
client.admin().indices().prepareCreate("test2").execute().actionGet();
@ -109,5 +110,24 @@ public class SimpleIndexStatsTests extends AbstractNodesTests {
assertThat(stats.primaries().indexing().typeStats().get("type1").indexCount(), equalTo(1l));
assertThat(stats.primaries().indexing().typeStats().get("type").indexCount(), equalTo(1l));
assertThat(stats.primaries().indexing().typeStats().get("type2"), nullValue());
assertThat(stats.total().get().count(), equalTo(0l));
// check get
GetResponse getResponse = client.prepareGet("test1", "type1", "1").execute().actionGet();
assertThat(getResponse.exists(), equalTo(true));
stats = client.admin().indices().prepareStats().execute().actionGet();
assertThat(stats.total().get().count(), equalTo(1l));
assertThat(stats.total().get().existsCount(), equalTo(1l));
assertThat(stats.total().get().missingCount(), equalTo(0l));
// missing get
getResponse = client.prepareGet("test1", "type1", "2").execute().actionGet();
assertThat(getResponse.exists(), equalTo(false));
stats = client.admin().indices().prepareStats().execute().actionGet();
assertThat(stats.total().get().count(), equalTo(2l));
assertThat(stats.total().get().existsCount(), equalTo(1l));
assertThat(stats.total().get().missingCount(), equalTo(1l));
}
}