honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add the memory used on segment/segments stats 

The memory used for the Lucene index (term dict, bloom filter, ...) can now be reported per segment using the segments API, and on the segments flag on node/indices stats
closes #4512
This commit is contained in:
Shay Banon 2013-12-18 22:21:53 +01:00
parent 0a016716ed
commit 95ca06cf09
5 changed files with 45 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java
View File

@ -127,6 +127,7 @@ public class IndicesSegmentResponse extends BroadcastOperationResponse implement
builder.field(Fields.NUM_DOCS, segment.getNumDocs());
builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs());
builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, segment.getSizeInBytes());
builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, segment.getMemoryInBytes());
builder.field(Fields.COMMITTED, segment.isCommitted());
builder.field(Fields.SEARCH, segment.isSearch());
if (segment.getVersion() != null) {
@ -177,5 +178,7 @@ public class IndicesSegmentResponse extends BroadcastOperationResponse implement
static final XContentBuilderString VERSION = new XContentBuilderString("version");
static final XContentBuilderString COMPOUND = new XContentBuilderString("compound");
static final XContentBuilderString MERGE_ID = new XContentBuilderString("merge_id");
static final XContentBuilderString MEMORY = new XContentBuilderString("memory");
static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes");
}
}

14
src/main/java/org/elasticsearch/index/engine/Segment.java
View File

@ -40,6 +40,7 @@ public class Segment implements Streamable {
public String version = null;
public Boolean compound = null;
public String mergeId;
public long memoryInBytes;
Segment() {
}
@ -99,6 +100,13 @@ public class Segment implements Streamable {
return this.mergeId;
}
/**
* Estimation of the memory usage used by a segment.
*/
public long getMemoryInBytes() {
return this.memoryInBytes;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
@ -136,6 +144,9 @@ public class Segment implements Streamable {
if (in.getVersion().onOrAfter(Version.V_0_90_6)) {
mergeId = in.readOptionalString();
}
if (in.getVersion().after(Version.V_0_90_8)) {
memoryInBytes = in.readLong();
}
}
@Override
@ -151,5 +162,8 @@ public class Segment implements Streamable {
if (out.getVersion().onOrAfter(Version.V_0_90_6)) {
out.writeOptionalString(mergeId);
}
if (out.getVersion().after(Version.V_0_90_8)) {
out.writeLong(memoryInBytes);
}
}
}

22
src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.engine;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Streamable;
@ -34,13 +35,15 @@ import java.io.IOException;
public class SegmentsStats implements Streamable, ToXContent {
private long count;
private long memoryInBytes;
public SegmentsStats() {
}
public void add(long count) {
public void add(long count, long memoryInBytes) {
this.count += count;
this.memoryInBytes += memoryInBytes;
}
public void add(SegmentsStats mergeStats) {
@ -48,6 +51,7 @@ public class SegmentsStats implements Streamable, ToXContent {
return;
}
this.count += mergeStats.count;
this.memoryInBytes += mergeStats.memoryInBytes;
}
/**
@ -57,6 +61,13 @@ public class SegmentsStats implements Streamable, ToXContent {
return this.count;
}
/**
* Estimation of the memory usage used by a segment.
*/
public long getMemoryInBytes() {
return this.memoryInBytes;
}
public static SegmentsStats readSegmentsStats(StreamInput in) throws IOException {
SegmentsStats stats = new SegmentsStats();
stats.readFrom(in);
@ -67,6 +78,7 @@ public class SegmentsStats implements Streamable, ToXContent {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(Fields.SEGMENTS);
builder.field(Fields.COUNT, count);
builder.byteSizeField(Fields.MEMORY_IN_BYTES, Fields.MEMORY, memoryInBytes);
builder.endObject();
return builder;
}
@ -74,15 +86,23 @@ public class SegmentsStats implements Streamable, ToXContent {
static final class Fields {
static final XContentBuilderString SEGMENTS = new XContentBuilderString("segments");
static final XContentBuilderString COUNT = new XContentBuilderString("count");
static final XContentBuilderString MEMORY = new XContentBuilderString("memory");
static final XContentBuilderString MEMORY_IN_BYTES = new XContentBuilderString("memory_in_bytes");
}
@Override
public void readFrom(StreamInput in) throws IOException {
count = in.readVLong();
if (in.getVersion().after(Version.V_0_90_8)) {
memoryInBytes = in.readLong();
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(count);
if (out.getVersion().after(Version.V_0_90_8)) {
out.writeLong(memoryInBytes);
}
}
}

6
src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java
View File

@ -1125,7 +1125,10 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
Searcher searcher = acquireSearcher("segments_stats");
try {
SegmentsStats stats = new SegmentsStats();
stats.add(searcher.reader().leaves().size());
for (AtomicReaderContext reader : searcher.reader().leaves()) {
assert reader.reader() instanceof SegmentReader;
stats.add(1, ((SegmentReader) reader.reader()).ramBytesUsed());
}
return stats;
} finally {
searcher.release();
@ -1160,6 +1163,7 @@ public class RobinEngine extends AbstractIndexShardComponent implements Engine {
} catch (IOException e) {
logger.trace("failed to get size for [{}]", e, info.info.name);
}
segment.memoryInBytes = ((SegmentReader) reader.reader()).ramBytesUsed();
segments.put(info.info.name, segment);
}
} finally {

3
src/test/java/org/elasticsearch/indices/stats/SimpleIndexStatsTests.java
View File

@ -191,7 +191,7 @@ public class SimpleIndexStatsTests extends ElasticsearchIntegrationTest {
}
@Test
public void testSegmentsStasts() {
public void testSegmentsStats() {
prepareCreate("test1", 2).setSettings("index.number_of_shards", 5, "index.number_of_replicas", 1).get();
ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().get();
@ -207,6 +207,7 @@ public class SimpleIndexStatsTests extends ElasticsearchIntegrationTest {
assertThat(stats.getTotal().getSegments(), notNullValue());
assertThat(stats.getTotal().getSegments().getCount(), equalTo(10l));
assertThat(stats.getTotal().getSegments().getMemoryInBytes(), greaterThan(0l));
}
@Test