Stats: Expose IndexWriter and VersionMap RAM usage to ShardStats and _cat endpoint
This commit adds the RAM usage of IndexWriter and VersionMap Closes #6483
This commit is contained in:
parent
4fb79fe787
commit
d0d1b98d23
|
@ -179,4 +179,8 @@ operations |9
|
||||||
|`segments.count` |`sc`, `segmentsCount` |No |Number of segments |4
|
|`segments.count` |`sc`, `segmentsCount` |No |Number of segments |4
|
||||||
|`segments.memory` |`sm`, `segmentsMemory` |No |Memory used by
|
|`segments.memory` |`sm`, `segmentsMemory` |No |Memory used by
|
||||||
segments |1.4kb
|
segments |1.4kb
|
||||||
|
|`segments.index_writer_memory` |`siwm`, `segmentsIndexWriterMemory` |No
|
||||||
|
|Memory used by index writer |1.2kb
|
||||||
|
|`segments.version_map_memory` |`svmm`, `segmentsVersionMapMemory` |No
|
||||||
|
|Memory used by version map |1.0kb
|
||||||
|=======================================================================
|
|=======================================================================
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
|
|
||||||
package org.elasticsearch.index.engine;
|
package org.elasticsearch.index.engine;
|
||||||
|
|
||||||
|
import org.elasticsearch.Version;
|
||||||
import org.elasticsearch.common.io.stream.StreamInput;
|
import org.elasticsearch.common.io.stream.StreamInput;
|
||||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||||
import org.elasticsearch.common.io.stream.Streamable;
|
import org.elasticsearch.common.io.stream.Streamable;
|
||||||
|
@ -36,6 +37,8 @@ public class SegmentsStats implements Streamable, ToXContent {
|
||||||
|
|
||||||
private long count;
|
private long count;
|
||||||
private long memoryInBytes;
|
private long memoryInBytes;
|
||||||
|
private long indexWriterMemoryInBytes;
|
||||||
|
private long versionMapMemoryInBytes;
|
||||||
|
|
||||||
public SegmentsStats() {
|
public SegmentsStats() {
|
||||||
|
|
||||||
|
@ -46,11 +49,21 @@ public class SegmentsStats implements Streamable, ToXContent {
|
||||||
this.memoryInBytes += memoryInBytes;
|
this.memoryInBytes += memoryInBytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void addIndexWriterMemoryInBytes(long indexWriterMemoryInBytes) {
|
||||||
|
this.indexWriterMemoryInBytes += indexWriterMemoryInBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addVersionMapMemoryInBytes(long versionMapMemoryInBytes) {
|
||||||
|
this.versionMapMemoryInBytes += versionMapMemoryInBytes;
|
||||||
|
}
|
||||||
|
|
||||||
public void add(SegmentsStats mergeStats) {
|
public void add(SegmentsStats mergeStats) {
|
||||||
if (mergeStats == null) {
|
if (mergeStats == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
add(mergeStats.count, mergeStats.memoryInBytes);
|
add(mergeStats.count, mergeStats.memoryInBytes);
|
||||||
|
addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes);
|
||||||
|
addVersionMapMemoryInBytes(mergeStats.versionMapMemoryInBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -71,6 +84,28 @@ public class SegmentsStats implements Streamable, ToXContent {
|
||||||
return new ByteSizeValue(memoryInBytes);
|
return new ByteSizeValue(memoryInBytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Estimation of the memory usage by index writer
|
||||||
|
*/
|
||||||
|
public long getIndexWriterMemoryInBytes() {
|
||||||
|
return this.indexWriterMemoryInBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ByteSizeValue getIndexWriterMemory() {
|
||||||
|
return new ByteSizeValue(indexWriterMemoryInBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Estimation of the memory usage by version map
|
||||||
|
*/
|
||||||
|
public long getVersionMapMemoryInBytes() {
|
||||||
|
return this.versionMapMemoryInBytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ByteSizeValue getVersionMapMemory() {
|
||||||
|
return new ByteSizeValue(versionMapMemoryInBytes);
|
||||||
|
}
|
||||||
|
|
||||||
public static SegmentsStats readSegmentsStats(StreamInput in) throws IOException {
|
public static SegmentsStats readSegmentsStats(StreamInput in) throws IOException {
|
||||||
SegmentsStats stats = new SegmentsStats();
|
SegmentsStats stats = new SegmentsStats();
|
||||||
stats.readFrom(in);
|
stats.readFrom(in);
|
||||||
|
@ -82,6 +117,8 @@ public class SegmentsStats implements Streamable, ToXContent {
|
||||||
builder.startObject(Fields.SEGMENTS);
|
builder.startObject(Fields.SEGMENTS);
|
||||||
builder.field(Fields.COUNT, count);
|
builder.field(Fields.COUNT, count);
|
||||||
builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, memoryInBytes);
|
builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, memoryInBytes);
|
||||||
|
builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes);
|
||||||
|
builder.byteSizeField(Fields.VERSION_MAP_MEMORY_IN_BYTES, Fields.VERSION_MAP_MEMORY, versionMapMemoryInBytes);
|
||||||
builder.endObject();
|
builder.endObject();
|
||||||
return builder;
|
return builder;
|
||||||
}
|
}
|
||||||
|
@ -91,17 +128,29 @@ public class SegmentsStats implements Streamable, ToXContent {
|
||||||
static final XContentBuilderString COUNT = new XContentBuilderString("count");
|
static final XContentBuilderString COUNT = new XContentBuilderString("count");
|
||||||
static final XContentBuilderString MEMORY = new XContentBuilderString("memory");
|
static final XContentBuilderString MEMORY = new XContentBuilderString("memory");
|
||||||
static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes");
|
static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes");
|
||||||
|
static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory");
|
||||||
|
static final XContentBuilderString INDEX_WRITER_MEMORY_IN_BYTES = new XContentBuilderString("index_writer_memory_in_bytes");
|
||||||
|
static final XContentBuilderString VERSION_MAP_MEMORY = new XContentBuilderString("version_map_memory");
|
||||||
|
static final XContentBuilderString VERSION_MAP_MEMORY_IN_BYTES = new XContentBuilderString("version_map_memory_in_bytes");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void readFrom(StreamInput in) throws IOException {
|
public void readFrom(StreamInput in) throws IOException {
|
||||||
count = in.readVLong();
|
count = in.readVLong();
|
||||||
memoryInBytes = in.readLong();
|
memoryInBytes = in.readLong();
|
||||||
|
if (in.getVersion().onOrAfter(Version.V_1_3_0)) {
|
||||||
|
indexWriterMemoryInBytes = in.readLong();
|
||||||
|
versionMapMemoryInBytes = in.readLong();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void writeTo(StreamOutput out) throws IOException {
|
public void writeTo(StreamOutput out) throws IOException {
|
||||||
out.writeVLong(count);
|
out.writeVLong(count);
|
||||||
out.writeLong(memoryInBytes);
|
out.writeLong(memoryInBytes);
|
||||||
|
if (out.getVersion().onOrAfter(Version.V_1_3_0)) {
|
||||||
|
out.writeLong(indexWriterMemoryInBytes);
|
||||||
|
out.writeLong(versionMapMemoryInBytes);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1136,6 +1136,8 @@ public class InternalEngine extends AbstractIndexShardComponent implements Engin
|
||||||
for (AtomicReaderContext reader : searcher.reader().leaves()) {
|
for (AtomicReaderContext reader : searcher.reader().leaves()) {
|
||||||
stats.add(1, getReaderRamBytesUsed(reader));
|
stats.add(1, getReaderRamBytesUsed(reader));
|
||||||
}
|
}
|
||||||
|
stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed());
|
||||||
|
stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed());
|
||||||
return stats;
|
return stats;
|
||||||
} finally {
|
} finally {
|
||||||
searcher.close();
|
searcher.close();
|
||||||
|
|
|
@ -240,6 +240,12 @@ public class RestIndicesAction extends AbstractCatAction {
|
||||||
table.addCell("segments.memory", "sibling:pri;alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
|
table.addCell("segments.memory", "sibling:pri;alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
|
||||||
table.addCell("pri.segments.memory", "default:false;text-align:right;desc:memory used by segments");
|
table.addCell("pri.segments.memory", "default:false;text-align:right;desc:memory used by segments");
|
||||||
|
|
||||||
|
table.addCell("segments.index_writer_memory", "sibling:pri;alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
|
||||||
|
table.addCell("pri.segments.index_writer_memory", "default:false;text-align:right;desc:memory used by index writer");
|
||||||
|
|
||||||
|
table.addCell("segments.version_map_memory", "sibling:pri;alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
|
||||||
|
table.addCell("pri.segments.version_map_memory", "default:false;text-align:right;desc:memory used by version map");
|
||||||
|
|
||||||
table.addCell("warmer.current", "sibling:pri;alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops");
|
table.addCell("warmer.current", "sibling:pri;alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops");
|
||||||
table.addCell("pri.warmer.current", "default:false;text-align:right;desc:current warmer ops");
|
table.addCell("pri.warmer.current", "default:false;text-align:right;desc:current warmer ops");
|
||||||
|
|
||||||
|
@ -413,6 +419,12 @@ public class RestIndicesAction extends AbstractCatAction {
|
||||||
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getMemory());
|
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getMemory());
|
||||||
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getMemory());
|
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getMemory());
|
||||||
|
|
||||||
|
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getIndexWriterMemory());
|
||||||
|
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getIndexWriterMemory());
|
||||||
|
|
||||||
|
table.addCell(indexStats == null ? null : indexStats.getTotal().getSegments().getVersionMapMemory());
|
||||||
|
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getSegments().getVersionMapMemory());
|
||||||
|
|
||||||
table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().current());
|
table.addCell(indexStats == null ? null : indexStats.getTotal().getWarmer().current());
|
||||||
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().current());
|
table.addCell(indexStats == null ? null : indexStats.getPrimaries().getWarmer().current());
|
||||||
|
|
||||||
|
|
|
@ -169,6 +169,8 @@ public class RestNodesAction extends AbstractCatAction {
|
||||||
|
|
||||||
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
|
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
|
||||||
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
|
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
|
||||||
|
table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
|
||||||
|
table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
|
||||||
|
|
||||||
table.addCell("suggest.current", "alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops");
|
table.addCell("suggest.current", "alias:suc,suggestCurrent;default:false;text-align:right;desc:number of current suggest ops");
|
||||||
table.addCell("suggest.time", "alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest");
|
table.addCell("suggest.time", "alias:suti,suggestTime;default:false;text-align:right;desc:time spend in suggest");
|
||||||
|
@ -271,6 +273,8 @@ public class RestNodesAction extends AbstractCatAction {
|
||||||
|
|
||||||
table.addCell(stats == null ? null : stats.getIndices().getSegments().getCount());
|
table.addCell(stats == null ? null : stats.getIndices().getSegments().getCount());
|
||||||
table.addCell(stats == null ? null : stats.getIndices().getSegments().getMemory());
|
table.addCell(stats == null ? null : stats.getIndices().getSegments().getMemory());
|
||||||
|
table.addCell(stats == null ? null : stats.getIndices().getSegments().getIndexWriterMemory());
|
||||||
|
table.addCell(stats == null ? null : stats.getIndices().getSegments().getVersionMapMemory());
|
||||||
|
|
||||||
table.addCell(stats == null ? null : stats.getIndices().getSuggest().getCurrent());
|
table.addCell(stats == null ? null : stats.getIndices().getSuggest().getCurrent());
|
||||||
table.addCell(stats == null ? null : stats.getIndices().getSuggest().getTime());
|
table.addCell(stats == null ? null : stats.getIndices().getSuggest().getTime());
|
||||||
|
|
|
@ -145,6 +145,8 @@ public class RestShardsAction extends AbstractCatAction {
|
||||||
|
|
||||||
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
|
table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
|
||||||
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
|
table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
|
||||||
|
table.addCell("segments.index_writer_memory", "alias:siwm,segmentsIndexWriterMemory;default:false;text-align:right;desc:memory used by index writer");
|
||||||
|
table.addCell("segments.version_map_memory", "alias:svmm,segmentsVersionMapMemory;default:false;text-align:right;desc:memory used by version map");
|
||||||
|
|
||||||
table.addCell("warmer.current", "alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops");
|
table.addCell("warmer.current", "alias:wc,warmerCurrent;default:false;text-align:right;desc:current warmer ops");
|
||||||
table.addCell("warmer.total", "alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops");
|
table.addCell("warmer.total", "alias:wto,warmerTotal;default:false;text-align:right;desc:total warmer ops");
|
||||||
|
@ -242,6 +244,8 @@ public class RestShardsAction extends AbstractCatAction {
|
||||||
|
|
||||||
table.addCell(shardStats == null ? null : shardStats.getSegments().getCount());
|
table.addCell(shardStats == null ? null : shardStats.getSegments().getCount());
|
||||||
table.addCell(shardStats == null ? null : shardStats.getSegments().getMemory());
|
table.addCell(shardStats == null ? null : shardStats.getSegments().getMemory());
|
||||||
|
table.addCell(shardStats == null ? null : shardStats.getSegments().getIndexWriterMemory());
|
||||||
|
table.addCell(shardStats == null ? null : shardStats.getSegments().getVersionMapMemory());
|
||||||
|
|
||||||
table.addCell(shardStats == null ? null : shardStats.getWarmer().current());
|
table.addCell(shardStats == null ? null : shardStats.getWarmer().current());
|
||||||
table.addCell(shardStats == null ? null : shardStats.getWarmer().total());
|
table.addCell(shardStats == null ? null : shardStats.getWarmer().total());
|
||||||
|
|
|
@ -193,10 +193,15 @@ public class SimpleIndexStatsTests extends ElasticsearchIntegrationTest {
|
||||||
for (int i = 0; i < 20; i++) {
|
for (int i = 0; i < 20; i++) {
|
||||||
index("test1", "type1", Integer.toString(i), "field", "value");
|
index("test1", "type1", Integer.toString(i), "field", "value");
|
||||||
index("test1", "type2", Integer.toString(i), "field", "value");
|
index("test1", "type2", Integer.toString(i), "field", "value");
|
||||||
client().admin().indices().prepareFlush().get();
|
|
||||||
}
|
}
|
||||||
client().admin().indices().prepareOptimize().setWaitForMerge(true).setMaxNumSegments(1).execute().actionGet();
|
|
||||||
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
|
IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).get();
|
||||||
|
assertThat(stats.getTotal().getSegments().getIndexWriterMemoryInBytes(), greaterThan(0l));
|
||||||
|
assertThat(stats.getTotal().getSegments().getVersionMapMemoryInBytes(), greaterThan(0l));
|
||||||
|
|
||||||
|
client().admin().indices().prepareFlush().get();
|
||||||
|
client().admin().indices().prepareOptimize().setWaitForMerge(true).setMaxNumSegments(1).execute().actionGet();
|
||||||
|
stats = client().admin().indices().prepareStats().setSegments(true).get();
|
||||||
|
|
||||||
assertThat(stats.getTotal().getSegments(), notNullValue());
|
assertThat(stats.getTotal().getSegments(), notNullValue());
|
||||||
assertThat(stats.getTotal().getSegments().getCount(), equalTo((long)test1.totalNumShards));
|
assertThat(stats.getTotal().getSegments().getCount(), equalTo((long)test1.totalNumShards));
|
||||||
|
|
Loading…
Reference in New Issue