From eb4ba3803226fa9fa2bcbfaf6b04707775375efb Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 15 Apr 2016 10:21:36 +0200 Subject: [PATCH] Add points to SegmentStats. #17775 This way points memory and disk usage will be reported in the stats API. Closes #16974 --- .../elasticsearch/index/engine/Engine.java | 1 + .../index/engine/SegmentsStats.java | 55 ++++++++++++------- .../indices/stats/IndicesStatsTests.java | 10 +++- .../index/engine/ShadowEngineTests.java | 6 ++ 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/engine/Engine.java b/core/src/main/java/org/elasticsearch/index/engine/Engine.java index 7ed930ab956..5d702f44918 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/core/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -426,6 +426,7 @@ public abstract class Engine implements Closeable { stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader())); stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader())); + stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader())); stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader())); if (includeSegmentFileSizes) { diff --git a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java index 10948f35d54..b4b80e32680 100644 --- a/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java +++ b/core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java @@ -21,7 +21,6 @@ package org.elasticsearch.index.engine; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; -import org.elasticsearch.Version; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -42,6 +41,7 @@ public class SegmentsStats implements Streamable, ToXContent { private long storedFieldsMemoryInBytes; private long termVectorsMemoryInBytes; private long normsMemoryInBytes; + private long pointsMemoryInBytes; private long docValuesMemoryInBytes; private long indexWriterMemoryInBytes; private long indexWriterMaxMemoryInBytes; @@ -67,6 +67,8 @@ public class SegmentsStats implements Streamable, ToXContent { .fPut("pay", "Payloads") .fPut("nvd", "Norms") .fPut("nvm", "Norms") + .fPut("dii", "Points") + .fPut("dim", "Points") .fPut("dvd", "DocValues") .fPut("dvm", "DocValues") .fPut("tvx", "Term Vector Index") @@ -98,6 +100,10 @@ public class SegmentsStats implements Streamable, ToXContent { this.normsMemoryInBytes += normsMemoryInBytes; } + public void addPointsMemoryInBytes(long pointsMemoryInBytes) { + this.pointsMemoryInBytes += pointsMemoryInBytes; + } + public void addDocValuesMemoryInBytes(long docValuesMemoryInBytes) { this.docValuesMemoryInBytes += docValuesMemoryInBytes; } @@ -143,6 +149,7 @@ public class SegmentsStats implements Streamable, ToXContent { addStoredFieldsMemoryInBytes(mergeStats.storedFieldsMemoryInBytes); addTermVectorsMemoryInBytes(mergeStats.termVectorsMemoryInBytes); addNormsMemoryInBytes(mergeStats.normsMemoryInBytes); + addPointsMemoryInBytes(mergeStats.pointsMemoryInBytes); addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes); addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes); addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes); @@ -213,6 +220,17 @@ public class SegmentsStats implements Streamable, ToXContent { return new ByteSizeValue(normsMemoryInBytes); } + /** + * Estimation of the points memory usage by a segment. + */ + public long getPointsMemoryInBytes() { + return this.pointsMemoryInBytes; + } + + public ByteSizeValue getPointsMemory() { + return new ByteSizeValue(pointsMemoryInBytes); + } + /** * Estimation of the doc values memory usage by a segment. */ @@ -287,6 +305,7 @@ public class SegmentsStats implements Streamable, ToXContent { builder.byteSizeField(Fields.STORED_FIELDS_MEMORY_IN_BYTES, Fields.STORED_FIELDS_MEMORY, storedFieldsMemoryInBytes); builder.byteSizeField(Fields.TERM_VECTORS_MEMORY_IN_BYTES, Fields.TERM_VECTORS_MEMORY, termVectorsMemoryInBytes); builder.byteSizeField(Fields.NORMS_MEMORY_IN_BYTES, Fields.NORMS_MEMORY, normsMemoryInBytes); + builder.byteSizeField(Fields.POINTS_MEMORY_IN_BYTES, Fields.POINTS_MEMORY, pointsMemoryInBytes); builder.byteSizeField(Fields.DOC_VALUES_MEMORY_IN_BYTES, Fields.DOC_VALUES_MEMORY, docValuesMemoryInBytes); builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes); builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes); @@ -318,6 +337,8 @@ public class SegmentsStats implements Streamable, ToXContent { static final XContentBuilderString TERM_VECTORS_MEMORY_IN_BYTES = new XContentBuilderString("term_vectors_memory_in_bytes"); static final XContentBuilderString NORMS_MEMORY = new XContentBuilderString("norms_memory"); static final XContentBuilderString NORMS_MEMORY_IN_BYTES = new XContentBuilderString("norms_memory_in_bytes"); + static final XContentBuilderString POINTS_MEMORY = new XContentBuilderString("points_memory"); + static final XContentBuilderString POINTS_MEMORY_IN_BYTES = new XContentBuilderString("points_memory_in_bytes"); static final XContentBuilderString DOC_VALUES_MEMORY = new XContentBuilderString("doc_values_memory"); static final XContentBuilderString DOC_VALUES_MEMORY_IN_BYTES = new XContentBuilderString("doc_values_memory_in_bytes"); static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory"); @@ -342,24 +363,21 @@ public class SegmentsStats implements Streamable, ToXContent { storedFieldsMemoryInBytes = in.readLong(); termVectorsMemoryInBytes = in.readLong(); normsMemoryInBytes = in.readLong(); + pointsMemoryInBytes = in.readLong(); docValuesMemoryInBytes = in.readLong(); indexWriterMemoryInBytes = in.readLong(); versionMapMemoryInBytes = in.readLong(); indexWriterMaxMemoryInBytes = in.readLong(); bitsetMemoryInBytes = in.readLong(); - if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { - int size = in.readVInt(); - ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(size); - for (int i = 0; i < size; i++) { - String key = in.readString(); - Long value = in.readLong(); - map.put(key, value); - } - fileSizes = map.build(); - } else { - fileSizes = ImmutableOpenMap.of(); + int size = in.readVInt(); + ImmutableOpenMap.Builder map = ImmutableOpenMap.builder(size); + for (int i = 0; i < size; i++) { + String key = in.readString(); + Long value = in.readLong(); + map.put(key, value); } + fileSizes = map.build(); } @Override @@ -370,19 +388,18 @@ public class SegmentsStats implements Streamable, ToXContent { out.writeLong(storedFieldsMemoryInBytes); out.writeLong(termVectorsMemoryInBytes); out.writeLong(normsMemoryInBytes); + out.writeLong(pointsMemoryInBytes); out.writeLong(docValuesMemoryInBytes); out.writeLong(indexWriterMemoryInBytes); out.writeLong(versionMapMemoryInBytes); out.writeLong(indexWriterMaxMemoryInBytes); out.writeLong(bitsetMemoryInBytes); - if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) { - out.writeVInt(fileSizes.size()); - for (Iterator> it = fileSizes.iterator(); it.hasNext();) { - ObjectObjectCursor entry = it.next(); - out.writeString(entry.key); - out.writeLong(entry.value); - } + out.writeVInt(fileSizes.size()); + for (Iterator> it = fileSizes.iterator(); it.hasNext();) { + ObjectObjectCursor entry = it.next(); + out.writeString(entry.key); + out.writeLong(entry.value); } } } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java index 77bd7c89927..726dccee597 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java @@ -41,6 +41,7 @@ public class IndicesStatsTests extends ESSingleNodeTestCase { assertEquals(0, stats.getStoredFieldsMemoryInBytes()); assertEquals(0, stats.getTermVectorsMemoryInBytes()); assertEquals(0, stats.getNormsMemoryInBytes()); + assertEquals(0, stats.getPointsMemoryInBytes()); assertEquals(0, stats.getDocValuesMemoryInBytes()); } @@ -58,12 +59,15 @@ public class IndicesStatsTests extends ESSingleNodeTestCase { .field("type", "text") .field("term_vector", "with_positions_offsets_payloads") .endObject() + .startObject("baz") + .field("type", "long") + .endObject() .endObject() .endObject() .endObject(); assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping)); ensureGreen("test"); - client().prepareIndex("test", "doc", "1").setSource("foo", "bar", "bar", "baz").get(); + client().prepareIndex("test", "doc", "1").setSource("foo", "bar", "bar", "baz", "baz", 42).get(); client().admin().indices().prepareRefresh("test").get(); IndicesStatsResponse rsp = client().admin().indices().prepareStats("test").get(); @@ -72,10 +76,11 @@ public class IndicesStatsTests extends ESSingleNodeTestCase { assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L)); assertThat(stats.getTermVectorsMemoryInBytes(), greaterThan(0L)); assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L)); + assertThat(stats.getPointsMemoryInBytes(), greaterThan(0L)); assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L)); // now check multiple segments stats are merged together - client().prepareIndex("test", "doc", "2").setSource("foo", "bar", "bar", "baz").get(); + client().prepareIndex("test", "doc", "2").setSource("foo", "bar", "bar", "baz", "baz", 43).get(); client().admin().indices().prepareRefresh("test").get(); rsp = client().admin().indices().prepareStats("test").get(); @@ -84,6 +89,7 @@ public class IndicesStatsTests extends ESSingleNodeTestCase { assertThat(stats2.getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes())); assertThat(stats2.getTermVectorsMemoryInBytes(), greaterThan(stats.getTermVectorsMemoryInBytes())); assertThat(stats2.getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); + assertThat(stats2.getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes())); assertThat(stats2.getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); } diff --git a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java index f33d26a80d6..6695fb471f6 100644 --- a/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java +++ b/core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.engine; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig; @@ -170,6 +171,7 @@ public class ShadowEngineTests extends ESTestCase { Field versionField = new NumericDocValuesField("_version", 0); document.add(uidField); document.add(versionField); + document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingsUpdate); } @@ -300,6 +302,7 @@ public class ShadowEngineTests extends ESTestCase { assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L)); assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L)); assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L)); + assertThat(stats.getPointsMemoryInBytes(), greaterThan(0L)); assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L)); assertThat(segments.get(0).isCommitted(), equalTo(false)); assertThat(segments.get(0).isSearch(), equalTo(true)); @@ -317,6 +320,7 @@ public class ShadowEngineTests extends ESTestCase { assertThat(stats.getStoredFieldsMemoryInBytes(), equalTo(0L)); assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L)); assertThat(stats.getNormsMemoryInBytes(), equalTo(0L)); + assertThat(stats.getPointsMemoryInBytes(), equalTo(0L)); assertThat(stats.getDocValuesMemoryInBytes(), equalTo(0L)); assertThat(segments.size(), equalTo(0)); @@ -356,6 +360,7 @@ public class ShadowEngineTests extends ESTestCase { assertThat(primaryEngine.segmentsStats(false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes())); assertThat(primaryEngine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L)); assertThat(primaryEngine.segmentsStats(false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); + assertThat(primaryEngine.segmentsStats(false).getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes())); assertThat(primaryEngine.segmentsStats(false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); @@ -380,6 +385,7 @@ public class ShadowEngineTests extends ESTestCase { assertThat(replicaEngine.segmentsStats(false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes())); assertThat(replicaEngine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L)); assertThat(replicaEngine.segmentsStats(false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); + assertThat(replicaEngine.segmentsStats(false).getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes())); assertThat(replicaEngine.segmentsStats(false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true));