Add points to SegmentStats. #17775

This way points memory and disk usage will be reported in the stats API.

Closes #16974

core/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -426,6 +426,7 @@ public abstract class Engine implements Closeable {
                 stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader()));
                 stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader()));
                 stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader()));
+                stats.addPointsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPointsReader()));
                 stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader()));
 
                 if (includeSegmentFileSizes) {

core/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java

@@ -21,7 +21,6 @@ package org.elasticsearch.index.engine;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
@@ -42,6 +41,7 @@ public class SegmentsStats implements Streamable, ToXContent {
     private long storedFieldsMemoryInBytes;
     private long termVectorsMemoryInBytes;
     private long normsMemoryInBytes;
+    private long pointsMemoryInBytes;
     private long docValuesMemoryInBytes;
     private long indexWriterMemoryInBytes;
     private long indexWriterMaxMemoryInBytes;
@@ -67,6 +67,8 @@ public class SegmentsStats implements Streamable, ToXContent {
             .fPut("pay", "Payloads")
             .fPut("nvd", "Norms")
             .fPut("nvm", "Norms")
+            .fPut("dii", "Points")
+            .fPut("dim", "Points")
             .fPut("dvd", "DocValues")
             .fPut("dvm", "DocValues")
             .fPut("tvx", "Term Vector Index")
@@ -98,6 +100,10 @@ public class SegmentsStats implements Streamable, ToXContent {
         this.normsMemoryInBytes += normsMemoryInBytes;
     }
 
+    public void addPointsMemoryInBytes(long pointsMemoryInBytes) {
+        this.pointsMemoryInBytes += pointsMemoryInBytes;
+    }
+
     public void addDocValuesMemoryInBytes(long docValuesMemoryInBytes) {
         this.docValuesMemoryInBytes += docValuesMemoryInBytes;
     }
@@ -143,6 +149,7 @@ public class SegmentsStats implements Streamable, ToXContent {
         addStoredFieldsMemoryInBytes(mergeStats.storedFieldsMemoryInBytes);
         addTermVectorsMemoryInBytes(mergeStats.termVectorsMemoryInBytes);
         addNormsMemoryInBytes(mergeStats.normsMemoryInBytes);
+        addPointsMemoryInBytes(mergeStats.pointsMemoryInBytes);
         addDocValuesMemoryInBytes(mergeStats.docValuesMemoryInBytes);
         addIndexWriterMemoryInBytes(mergeStats.indexWriterMemoryInBytes);
         addIndexWriterMaxMemoryInBytes(mergeStats.indexWriterMaxMemoryInBytes);
@@ -213,6 +220,17 @@ public class SegmentsStats implements Streamable, ToXContent {
         return new ByteSizeValue(normsMemoryInBytes);
     }
 
+    /**
+     * Estimation of the points memory usage by a segment.
+     */
+    public long getPointsMemoryInBytes() {
+        return this.pointsMemoryInBytes;
+    }
+
+    public ByteSizeValue getPointsMemory() {
+        return new ByteSizeValue(pointsMemoryInBytes);
+    }
+
     /**
      * Estimation of the doc values memory usage by a segment.
      */
@@ -287,6 +305,7 @@ public class SegmentsStats implements Streamable, ToXContent {
         builder.byteSizeField(Fields.STORED_FIELDS_MEMORY_IN_BYTES, Fields.STORED_FIELDS_MEMORY, storedFieldsMemoryInBytes);
         builder.byteSizeField(Fields.TERM_VECTORS_MEMORY_IN_BYTES, Fields.TERM_VECTORS_MEMORY, termVectorsMemoryInBytes);
         builder.byteSizeField(Fields.NORMS_MEMORY_IN_BYTES, Fields.NORMS_MEMORY, normsMemoryInBytes);
+        builder.byteSizeField(Fields.POINTS_MEMORY_IN_BYTES, Fields.POINTS_MEMORY, pointsMemoryInBytes);
         builder.byteSizeField(Fields.DOC_VALUES_MEMORY_IN_BYTES, Fields.DOC_VALUES_MEMORY, docValuesMemoryInBytes);
         builder.byteSizeField(Fields.INDEX_WRITER_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MEMORY, indexWriterMemoryInBytes);
         builder.byteSizeField(Fields.INDEX_WRITER_MAX_MEMORY_IN_BYTES, Fields.INDEX_WRITER_MAX_MEMORY, indexWriterMaxMemoryInBytes);
@@ -318,6 +337,8 @@ public class SegmentsStats implements Streamable, ToXContent {
         static final XContentBuilderString TERM_VECTORS_MEMORY_IN_BYTES = new XContentBuilderString("term_vectors_memory_in_bytes");
         static final XContentBuilderString NORMS_MEMORY = new XContentBuilderString("norms_memory");
         static final XContentBuilderString NORMS_MEMORY_IN_BYTES = new XContentBuilderString("norms_memory_in_bytes");
+        static final XContentBuilderString POINTS_MEMORY = new XContentBuilderString("points_memory");
+        static final XContentBuilderString POINTS_MEMORY_IN_BYTES = new XContentBuilderString("points_memory_in_bytes");
         static final XContentBuilderString DOC_VALUES_MEMORY = new XContentBuilderString("doc_values_memory");
         static final XContentBuilderString DOC_VALUES_MEMORY_IN_BYTES = new XContentBuilderString("doc_values_memory_in_bytes");
         static final XContentBuilderString INDEX_WRITER_MEMORY = new XContentBuilderString("index_writer_memory");
@@ -342,24 +363,21 @@ public class SegmentsStats implements Streamable, ToXContent {
         storedFieldsMemoryInBytes = in.readLong();
         termVectorsMemoryInBytes = in.readLong();
         normsMemoryInBytes = in.readLong();
+        pointsMemoryInBytes = in.readLong();
         docValuesMemoryInBytes = in.readLong();
         indexWriterMemoryInBytes = in.readLong();
         versionMapMemoryInBytes = in.readLong();
         indexWriterMaxMemoryInBytes = in.readLong();
         bitsetMemoryInBytes = in.readLong();
 
-        if (in.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
+        int size = in.readVInt();
-            int size = in.readVInt();
+        ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(size);
-            ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(size);
+        for (int i = 0; i < size; i++) {
-            for (int i = 0; i < size; i++) {
+            String key = in.readString();
-                String key = in.readString();
+            Long value = in.readLong();
-                Long value = in.readLong();
+            map.put(key, value);
-                map.put(key, value);
-            }
-            fileSizes = map.build();
-        } else {
-            fileSizes = ImmutableOpenMap.of();
         }
+        fileSizes = map.build();
     }
 
     @Override
@@ -370,19 +388,18 @@ public class SegmentsStats implements Streamable, ToXContent {
         out.writeLong(storedFieldsMemoryInBytes);
         out.writeLong(termVectorsMemoryInBytes);
         out.writeLong(normsMemoryInBytes);
+        out.writeLong(pointsMemoryInBytes);
         out.writeLong(docValuesMemoryInBytes);
         out.writeLong(indexWriterMemoryInBytes);
         out.writeLong(versionMapMemoryInBytes);
         out.writeLong(indexWriterMaxMemoryInBytes);
         out.writeLong(bitsetMemoryInBytes);
 
-        if (out.getVersion().onOrAfter(Version.V_5_0_0_alpha1)) {
+        out.writeVInt(fileSizes.size());
-            out.writeVInt(fileSizes.size());
+        for (Iterator<ObjectObjectCursor<String, Long>> it = fileSizes.iterator(); it.hasNext();) {
-            for (Iterator<ObjectObjectCursor<String, Long>> it = fileSizes.iterator(); it.hasNext();) {
+            ObjectObjectCursor<String, Long> entry = it.next();
-                ObjectObjectCursor<String, Long> entry = it.next();
+            out.writeString(entry.key);
-                out.writeString(entry.key);
+            out.writeLong(entry.value);
-                out.writeLong(entry.value);
-            }
         }
     }
 }

core/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsTests.java

@@ -41,6 +41,7 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         assertEquals(0, stats.getStoredFieldsMemoryInBytes());
         assertEquals(0, stats.getTermVectorsMemoryInBytes());
         assertEquals(0, stats.getNormsMemoryInBytes());
+        assertEquals(0, stats.getPointsMemoryInBytes());
         assertEquals(0, stats.getDocValuesMemoryInBytes());
     }
 
@@ -58,12 +59,15 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
                             .field("type", "text")
                             .field("term_vector", "with_positions_offsets_payloads")
                         .endObject()
+                        .startObject("baz")
+                            .field("type", "long")
+                        .endObject()
                     .endObject()
                 .endObject()
             .endObject();
         assertAcked(client().admin().indices().prepareCreate("test").addMapping("doc", mapping));
         ensureGreen("test");
-        client().prepareIndex("test", "doc", "1").setSource("foo", "bar", "bar", "baz").get();
+        client().prepareIndex("test", "doc", "1").setSource("foo", "bar", "bar", "baz", "baz", 42).get();
         client().admin().indices().prepareRefresh("test").get();
 
         IndicesStatsResponse rsp = client().admin().indices().prepareStats("test").get();
@@ -72,10 +76,11 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L));
         assertThat(stats.getTermVectorsMemoryInBytes(), greaterThan(0L));
         assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L));
+        assertThat(stats.getPointsMemoryInBytes(), greaterThan(0L));
         assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L));
 
         // now check multiple segments stats are merged together
-        client().prepareIndex("test", "doc", "2").setSource("foo", "bar", "bar", "baz").get();
+        client().prepareIndex("test", "doc", "2").setSource("foo", "bar", "bar", "baz", "baz", 43).get();
         client().admin().indices().prepareRefresh("test").get();
 
         rsp = client().admin().indices().prepareStats("test").get();
@@ -84,6 +89,7 @@ public class IndicesStatsTests extends ESSingleNodeTestCase {
         assertThat(stats2.getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes()));
         assertThat(stats2.getTermVectorsMemoryInBytes(), greaterThan(stats.getTermVectorsMemoryInBytes()));
         assertThat(stats2.getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
+        assertThat(stats2.getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes()));
         assertThat(stats2.getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
     }
 

core/src/test/java/org/elasticsearch/index/engine/ShadowEngineTests.java

@@ -21,6 +21,7 @@ package org.elasticsearch.index.engine;
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -170,6 +171,7 @@ public class ShadowEngineTests extends ESTestCase {
         Field versionField = new NumericDocValuesField("_version", 0);
         document.add(uidField);
         document.add(versionField);
+        document.add(new LongPoint("point_field", 42)); // so that points report memory/disk usage
         return new ParsedDocument(uidField, versionField, id, type, routing, timestamp, ttl, Arrays.asList(document), source, mappingsUpdate);
     }
 
@@ -300,6 +302,7 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L));
         assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L));
         assertThat(stats.getNormsMemoryInBytes(), greaterThan(0L));
+        assertThat(stats.getPointsMemoryInBytes(), greaterThan(0L));
         assertThat(stats.getDocValuesMemoryInBytes(), greaterThan(0L));
         assertThat(segments.get(0).isCommitted(), equalTo(false));
         assertThat(segments.get(0).isSearch(), equalTo(true));
@@ -317,6 +320,7 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(stats.getStoredFieldsMemoryInBytes(), equalTo(0L));
         assertThat(stats.getTermVectorsMemoryInBytes(), equalTo(0L));
         assertThat(stats.getNormsMemoryInBytes(), equalTo(0L));
+        assertThat(stats.getPointsMemoryInBytes(), equalTo(0L));
         assertThat(stats.getDocValuesMemoryInBytes(), equalTo(0L));
         assertThat(segments.size(), equalTo(0));
 
@@ -356,6 +360,7 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(primaryEngine.segmentsStats(false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes()));
         assertThat(primaryEngine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L));
         assertThat(primaryEngine.segmentsStats(false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
+        assertThat(primaryEngine.segmentsStats(false).getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes()));
         assertThat(primaryEngine.segmentsStats(false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
         assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
         assertThat(segments.get(0).isCommitted(), equalTo(true));
@@ -380,6 +385,7 @@ public class ShadowEngineTests extends ESTestCase {
         assertThat(replicaEngine.segmentsStats(false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes()));
         assertThat(replicaEngine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L));
         assertThat(replicaEngine.segmentsStats(false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes()));
+        assertThat(replicaEngine.segmentsStats(false).getPointsMemoryInBytes(), greaterThan(stats.getPointsMemoryInBytes()));
         assertThat(replicaEngine.segmentsStats(false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes()));
         assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
         assertThat(segments.get(0).isCommitted(), equalTo(true));