honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add points to SegmentStats. #17775 

This way points memory and disk usage will be reported in the stats API.

Closes #16974
This commit is contained in:
Adrien Grand 2016-04-15 10:21:36 +02:00
parent 7f6a765a1e
commit eb4ba38032
4 changed files with 51 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -426,6 +426,7 @@ public abstract class Engine implements Closeable {
stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader()));
stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader()));
stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader()));
stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader()));
stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader()));
if (includeSegmentFileSizes) {

55
core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java
View File

@ -21,7 +21,6 @@ package org.elasticsearch.index.engine;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import org.elasticsearch.Version;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
@ -42,6 +41,7 @@ public class SegmentsStats implements Streamable, ToXContent {
private long storedFieldsMemoryInBytes;
private long termVectorsMemoryInBytes;
private long normsMemoryInBytes;
private long pointsMemoryInBytes;
private long docValuesMemoryInBytes;
private long indexWriterMemoryInBytes;
private long indexWriterMaxMemoryInBytes;
@ -67,6 +67,8 @@ public class SegmentsStats implements Streamable, ToXContent {
.fPut("pay", "Payloads")
.fPut("nvd", "Norms")
.fPut("nvm", "Norms")
.fPut("dii", "Points")
.fPut("dim", "Points")
.fPut("dvd", "DocValues")
.fPut("dvm", "DocValues")
.fPut("tvx", "Term Vector Index")
@ -98,6 +100,10 @@ public class SegmentsStats implements Streamable, ToXContent {
this.normsMemoryInBytes += normsMemoryInBytes;
}
public void addPointsMemoryInBytes(long pointsMemoryInBytes) {
this.pointsMemoryInBytes += pointsMemoryInBytes;
}
public void addDocValuesMemoryInBytes(long docValuesMemoryInBytes) {
this.docValuesMemoryInBytes += docValuesMemoryInBytes;
}
@ -143,6 +149,7 @@ public class SegmentsStats implements Streamable, ToXContent {
addStoredFieldsMemoryInBytes(mergeStats.storedFieldsMemoryInBytes);
addTermVectorsMemoryInBytes(mergeStats.termVectorsMemoryInBytes);
addNormsMemoryInBytes(mergeStats.normsMemoryInBytes);
addPointsMemoryInBytes(mergeStats.pointsMemoryInBytes);
addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes);
addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes);
addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes);
@ -213,6 +220,17 @@ public class SegmentsStats implements Streamable, ToXContent {
return new ByteSizeValue(normsMemoryInBytes);
}
/**
* Estimation of the points memory usage by a segment.
*/
public long getPointsMemoryInBytes() {
return this.pointsMemoryInBytes;
}
public ByteSizeValue getPointsMemory() {
return new ByteSizeValue(pointsMemoryInBytes);
}
/**
* Estimation of the doc values memory usage by a segment.
*/
@ -287,6 +305,7 @@ public class SegmentsStats implements Streamable, ToXContent {
builder.byteSizeField(Fields.STORED_FIELDS_MEMORY_IN_BYTES, Fields.STORED_FIELDS_MEMORY, storedFieldsMemoryInBytes);
builder.byteSizeField(Fields.TERM_VECTORS_MEMORY_IN_BYTES, Fields.TERM_VECTORS_MEMORY, termVectorsMemoryInBytes);
builder.byteSizeField(Fields.NORMS_MEMORY_IN_BYTES, Fields.NORMS_MEMORY, normsMemoryInBytes);
builder.byteSizeField(Fields.POINTS_MEMORY_IN_BYTES, Fields.POINTS_MEMORY, pointsMemoryInBytes);
builder.byteSizeField(Fields.DOC_VALUES_MEMORY_IN_BYTES, Fields.DOC_VALUES_MEMORY, docValuesMemoryInBytes);
builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes);
builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes);
@ -318,6 +337,8 @@ public class SegmentsStats implements Streamable, ToXContent {
static final XContentBuilderString TERM_VECTORS_MEMORY_IN_BYTES = new XContentBuilderString("term_vectors_memory_in_bytes");
static final XContentBuilderString NORMS_MEMORY = new XContentBuilderString("norms_memory");
static final XContentBuilderString NORMS_MEMORY_IN_BYTES = new XContentBuilderString("norms_memory_in_bytes");
static final XContentBuilderString POINTS_MEMORY = new XContentBuilderString("points_memory");
static final XContentBuilderString POINTS_MEMORY_IN_BYTES = new XContentBuilderString("points_memory_in_bytes");
static final XContentBuilderString DOC_VALUES_MEMORY = new XContentBuilderString("doc_values_memory");
static final XContentBuilderString DOC_VALUES_MEMORY_IN_BYTES = new XContentBuilderString("doc_values_memory_in_bytes");
static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory");
@ -342,24 +363,21 @@ public class SegmentsStats implements Streamable, ToXContent {
storedFieldsMemoryInBytes = in.readLong();
termVectorsMemoryInBytes = in.readLong();
normsMemoryInBytes = in.readLong();
pointsMemoryInBytes = in.readLong();
docValuesMemoryInBytes = in.readLong();
indexWriterMemoryInBytes = in.readLong();
versionMapMemoryInBytes = in.readLong();
indexWriterMaxMemoryInBytes = in.readLong();
bitsetMemoryInBytes = in.readLong();
if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
int size = in.readVInt();
ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(size);
for (int i = 0; i < size; i++) {
String key = in.readString();
Long value = in.readLong();
map.put(key, value);
}
fileSizes = map.build();
} else {
fileSizes = ImmutableOpenMap.of();
int size = in.readVInt();
ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(size);
for (int i = 0; i < size; i++) {
String key = in.readString();
Long value = in.readLong();
map.put(key, value);
}
fileSizes = map.build();
}
@Override
@ -370,19 +388,18 @@ public class SegmentsStats implements Streamable, ToXContent {
out.writeLong(storedFieldsMemoryInBytes);
out.writeLong(termVectorsMemoryInBytes);
out.writeLong(normsMemoryInBytes);
out.writeLong(pointsMemoryInBytes);
out.writeLong(docValuesMemoryInBytes);
out.writeLong(indexWriterMemoryInBytes);
out.writeLong(versionMapMemoryInBytes);
out.writeLong(indexWriterMaxMemoryInBytes);
out.writeLong(bitsetMemoryInBytes);
if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
out.writeVInt(fileSizes.size());
for (Iterator<ObjectObjectCursor<String, Long>> it = fileSizes.iterator(); it.hasNext();) {
ObjectObjectCursor<String, Long> entry = it.next();
out.writeString(entry.key);
out.writeLong(entry.value);
}
out.writeVInt(fileSizes.size());
for (Iterator<ObjectObjectCursor<String, Long>> it = fileSizes.iterator(); it.hasNext();) {
ObjectObjectCursor<String, Long> entry = it.next();
out.writeString(entry.key);
out.writeLong(entry.value);
}
}
}

10
core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java
View File

@ -41,6 +41,7 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
assertEquals(0, stats.getStoredFieldsMemoryInBytes());
assertEquals(0, stats.getTermVectorsMemoryInBytes());
assertEquals(0, stats.getNormsMemoryInBytes());
assertEquals(0, stats.getPointsMemoryInBytes());
assertEquals(0, stats.getDocValuesMemoryInBytes());
}
@ -58,12 +59,15 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
.field("type", "text")
.field("term_vector", "with_positions_offsets_payloads")
.endObject()
.startObject("baz")
.field("type", "long")
.endObject()
.endObject()
.endObject()
.endObject();
assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping));
ensureGreen("test");
client().prepareIndex("test", "doc", "1").setSource("foo", "bar", "bar", "baz").get();
client().prepareIndex("test", "doc", "1").setSource("foo", "bar", "bar", "baz", "baz", 42).get();
client().admin().indices().prepareRefresh("test").get();
IndicesStatsResponse rsp = client().admin().indices().prepareStats("test").get();
@ -72,10 +76,11 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getTermVectorsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getPointsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L));
// now check multiple segments stats are merged together
client().prepareIndex("test", "doc", "2").setSource("foo", "bar", "bar", "baz").get();
client().prepareIndex("test", "doc", "2").setSource("foo", "bar", "bar", "baz", "baz", 43).get();
client().admin().indices().prepareRefresh("test").get();
rsp = client().admin().indices().prepareStats("test").get();
@ -84,6 +89,7 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
assertThat(stats2.getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes()));
assertThat(stats2.getTermVectorsMemoryInBytes(), greaterThan(stats.getTermVectorsMemoryInBytes()));
assertThat(stats2.getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
assertThat(stats2.getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes()));
assertThat(stats2.getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
}

6
core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.engine;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig;
@ -170,6 +171,7 @@ public class ShadowEngineTests extends ESTestCase {
Field versionField = new NumericDocValuesField("_version", 0);
document.add(uidField);
document.add(versionField);
document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage
return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingsUpdate);
}
@ -300,6 +302,7 @@ public class ShadowEngineTests extends ESTestCase {
assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L));
assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getPointsMemoryInBytes(), greaterThan(0L));
assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L));
assertThat(segments.get(0).isCommitted(), equalTo(false));
assertThat(segments.get(0).isSearch(), equalTo(true));
@ -317,6 +320,7 @@ public class ShadowEngineTests extends ESTestCase {
assertThat(stats.getStoredFieldsMemoryInBytes(), equalTo(0L));
assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L));
assertThat(stats.getNormsMemoryInBytes(), equalTo(0L));
assertThat(stats.getPointsMemoryInBytes(), equalTo(0L));
assertThat(stats.getDocValuesMemoryInBytes(), equalTo(0L));
assertThat(segments.size(), equalTo(0));
@ -356,6 +360,7 @@ public class ShadowEngineTests extends ESTestCase {
assertThat(primaryEngine.segmentsStats(false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes()));
assertThat(primaryEngine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L));
assertThat(primaryEngine.segmentsStats(false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
assertThat(primaryEngine.segmentsStats(false).getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes()));
assertThat(primaryEngine.segmentsStats(false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));
@ -380,6 +385,7 @@ public class ShadowEngineTests extends ESTestCase {
assertThat(replicaEngine.segmentsStats(false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes()));
assertThat(replicaEngine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L));
assertThat(replicaEngine.segmentsStats(false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
assertThat(replicaEngine.segmentsStats(false).getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes()));
assertThat(replicaEngine.segmentsStats(false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
assertThat(segments.get(0).isCommitted(), equalTo(true));