Allow inclusion of unloaded segments in stats (#39512)

Today we have no chance to fetch actual segment stats for segments that
are currently unloaded. This is relevant in the case of frozen indices.
This allows to monitor how much memory a frozen index would use if it was
unfrozen.

rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json

@@ -57,6 +57,11 @@
           "type": "boolean",
           "description": "Verbose mode. Display column headers",
           "default": false
+        },
+        "include_unloaded_segments": {
+          "type": "boolean",
+          "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory",
+          "default": false
         }
       }
     },

rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json

@@ -52,6 +52,11 @@
           "type": "boolean",
           "description": "Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)",
           "default": false
+        },
+        "include_unloaded_segments": {
+          "type": "boolean",
+          "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory",
+          "default": false
         }
       }
     },

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java

@@ -207,7 +207,7 @@ public class CommonStats implements Writeable, ToXContentFragment {
                         completion = indexShard.completionStats(flags.completionDataFields());
                         break;
                     case Segments:
-                        segments = indexShard.segmentStats(flags.includeSegmentFileSizes());
+                        segments = indexShard.segmentStats(flags.includeSegmentFileSizes(), flags.includeUnloadedSegments());
                         break;
                     case Translog:
                         translog = indexShard.translogStats();

server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.action.admin.indices.stats;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
@@ -38,6 +39,7 @@ public class CommonStatsFlags implements Writeable, Cloneable {
     private String[] fieldDataFields = null;
     private String[] completionDataFields = null;
     private boolean includeSegmentFileSizes = false;
+    private boolean includeUnloadedSegments = false;
 
     /**
      * @param flags flags to set. If no flags are supplied, default flags will be set.
@@ -62,6 +64,9 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         fieldDataFields = in.readStringArray();
         completionDataFields = in.readStringArray();
         includeSegmentFileSizes = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            includeUnloadedSegments = in.readBoolean();
+        }
     }
 
     @Override
@@ -77,6 +82,9 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         out.writeStringArrayNullable(fieldDataFields);
         out.writeStringArrayNullable(completionDataFields);
         out.writeBoolean(includeSegmentFileSizes);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeBoolean(includeUnloadedSegments);
+        }
     }
 
     /**
@@ -89,6 +97,7 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         fieldDataFields = null;
         completionDataFields = null;
         includeSegmentFileSizes = false;
+        includeUnloadedSegments = false;
         return this;
     }
 
@@ -102,6 +111,7 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         fieldDataFields = null;
         completionDataFields = null;
         includeSegmentFileSizes = false;
+        includeUnloadedSegments = false;
         return this;
     }
 
@@ -170,6 +180,15 @@ public class CommonStatsFlags implements Writeable, Cloneable {
         return this;
     }
 
+    public CommonStatsFlags includeUnloadedSegments(boolean includeUnloadedSegments) {
+        this.includeUnloadedSegments = includeUnloadedSegments;
+        return this;
+    }
+
+    public boolean includeUnloadedSegments() {
+        return this.includeUnloadedSegments;
+    }
+
     public boolean includeSegmentFileSizes() {
         return this.includeSegmentFileSizes;
     }

server/src/main/java/org/elasticsearch/index/engine/Engine.java

@@ -821,7 +821,7 @@ public abstract class Engine implements Closeable {
     /**
      * Global stats on segments.
      */
-    public SegmentsStats segmentsStats(boolean includeSegmentFileSizes) {
+    public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
         ensureOpen();
         Set<String> segmentName = new HashSet<>();
         SegmentsStats stats = new SegmentsStats();
@@ -845,7 +845,7 @@ public abstract class Engine implements Closeable {
         return stats;
     }
 
-    private void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) {
+    protected void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) {
         stats.add(1, segmentReader.ramBytesUsed());
         stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader()));
         stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader()));

server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java

@@ -391,4 +391,8 @@ public class SegmentsStats implements Streamable, ToXContentFragment {
             out.writeLong(entry.value.longValue());
         }
     }
+
+    public void clearFileSizes() {
+        fileSizes = ImmutableOpenMap.of();
+    }
 }

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -999,8 +999,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl
         return engine.getMergeStats();
     }
 
-    public SegmentsStats segmentStats(boolean includeSegmentFileSizes) {
+    public SegmentsStats segmentStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
-        SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes);
+        SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes, includeUnloadedSegments);
         segmentsStats.addBitsetMemoryInBytes(shardBitsetFilterCache.getMemorySizeInBytes());
         return segmentsStats;
     }

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java

@@ -121,6 +121,7 @@ public class RestIndicesStatsAction extends BaseRestHandler {
 
         if (indicesStatsRequest.segments()) {
             indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false));
+            indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false));
         }
 
         return channel -> client.admin().indices().stats(indicesStatsRequest, new RestToXContentListener<>(channel));

server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -116,6 +116,7 @@ public class RestIndicesAction extends AbstractCatAction {
                         indicesStatsRequest.indices(indices);
                         indicesStatsRequest.indicesOptions(strictExpandIndicesOptions);
                         indicesStatsRequest.all();
+                        indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false));
 
                         client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
                             @Override

server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java

@@ -272,8 +272,8 @@ public class InternalEngineTests extends EngineTestCase {
              InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) {
             List<Segment> segments = engine.segments(false);
             assertThat(segments.isEmpty(), equalTo(true));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(0L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(0L));
-            assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L));
+            assertThat(engine.segmentsStats(false, false).getMemoryInBytes(), equalTo(0L));
 
             // create two docs and refresh
             ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
@@ -287,7 +287,7 @@ public class InternalEngineTests extends EngineTestCase {
 
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(1));
-            SegmentsStats stats = engine.segmentsStats(false);
+            SegmentsStats stats = engine.segmentsStats(false, false);
             assertThat(stats.getCount(), equalTo(1L));
             assertThat(stats.getTermsMemoryInBytes(), greaterThan(0L));
             assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L));
@@ -306,7 +306,7 @@ public class InternalEngineTests extends EngineTestCase {
 
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(1));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(1L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(1L));
             assertThat(segments.get(0).isCommitted(), equalTo(true));
             assertThat(segments.get(0).isSearch(), equalTo(true));
             assertThat(segments.get(0).getNumDocs(), equalTo(2));
@@ -319,15 +319,15 @@ public class InternalEngineTests extends EngineTestCase {
 
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(2));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(2L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L));
-            assertThat(engine.segmentsStats(false).getTermsMemoryInBytes(),
+            assertThat(engine.segmentsStats(false, false).getTermsMemoryInBytes(),
                 greaterThan(stats.getTermsMemoryInBytes()));
-            assertThat(engine.segmentsStats(false).getStoredFieldsMemoryInBytes(),
+            assertThat(engine.segmentsStats(false, false).getStoredFieldsMemoryInBytes(),
                 greaterThan(stats.getStoredFieldsMemoryInBytes()));
-            assertThat(engine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L));
+            assertThat(engine.segmentsStats(false, false).getTermVectorsMemoryInBytes(), equalTo(0L));
-            assertThat(engine.segmentsStats(false).getNormsMemoryInBytes(),
+            assertThat(engine.segmentsStats(false, false).getNormsMemoryInBytes(),
                 greaterThan(stats.getNormsMemoryInBytes()));
-            assertThat(engine.segmentsStats(false).getDocValuesMemoryInBytes(),
+            assertThat(engine.segmentsStats(false, false).getDocValuesMemoryInBytes(),
                 greaterThan(stats.getDocValuesMemoryInBytes()));
             assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
             assertThat(segments.get(0).isCommitted(), equalTo(true));
@@ -349,7 +349,7 @@ public class InternalEngineTests extends EngineTestCase {
 
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(2));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(2L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L));
             assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
             assertThat(segments.get(0).isCommitted(), equalTo(true));
             assertThat(segments.get(0).isSearch(), equalTo(true));
@@ -370,7 +370,7 @@ public class InternalEngineTests extends EngineTestCase {
 
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(3));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(3L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(3L));
             assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
             assertThat(segments.get(0).isCommitted(), equalTo(true));
             assertThat(segments.get(0).isSearch(), equalTo(true));
@@ -397,7 +397,7 @@ public class InternalEngineTests extends EngineTestCase {
 
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(4));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(4L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L));
             assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
             assertThat(segments.get(0).isCommitted(), equalTo(true));
             assertThat(segments.get(0).isSearch(), equalTo(true));
@@ -427,7 +427,7 @@ public class InternalEngineTests extends EngineTestCase {
             engine.refresh("test");
             segments = engine.segments(false);
             assertThat(segments.size(), equalTo(4));
-            assertThat(engine.segmentsStats(false).getCount(), equalTo(4L));
+            assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L));
             assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true));
             assertThat(segments.get(0).isCommitted(), equalTo(true));
             assertThat(segments.get(0).isSearch(), equalTo(true));
@@ -572,13 +572,13 @@ public class InternalEngineTests extends EngineTestCase {
     public void testSegmentsStatsIncludingFileSizes() throws Exception {
         try (Store store = createStore();
              Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) {
-            assertThat(engine.segmentsStats(true).getFileSizes().size(), equalTo(0));
+            assertThat(engine.segmentsStats(true, false).getFileSizes().size(), equalTo(0));
 
             ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
             engine.index(indexForDoc(doc));
             engine.refresh("test");
 
-            SegmentsStats stats = engine.segmentsStats(true);
+            SegmentsStats stats = engine.segmentsStats(true, false);
             assertThat(stats.getFileSizes().size(), greaterThan(0));
             assertThat(() -> stats.getFileSizes().valuesIt(), everyItem(greaterThan(0L)));
 
@@ -588,7 +588,7 @@ public class InternalEngineTests extends EngineTestCase {
             engine.index(indexForDoc(doc2));
             engine.refresh("test");
 
-            assertThat(engine.segmentsStats(true).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value));
+            assertThat(engine.segmentsStats(true, false).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value));
         }
     }
 
@@ -3698,23 +3698,23 @@ public class InternalEngineTests extends EngineTestCase {
                 NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
         try (Store store = createStore(newFSDirectory(storeDir)); Engine engine = createEngine(configSupplier.apply(store))) {
             assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP,
-                engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+                engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp());
             final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(),
                 new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
             engine.index(appendOnlyPrimary(doc, true, timestamp1));
-            assertEquals(timestamp1, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+            assertEquals(timestamp1, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp());
         }
         try (Store store = createStore(newFSDirectory(storeDir));
              InternalEngine engine = new InternalEngine(configSupplier.apply(store))) {
             assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP,
-                engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+                engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp());
             engine.initializeMaxSeqNoOfUpdatesOrDeletes();
             engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
-            assertEquals(timestamp1, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+            assertEquals(timestamp1, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp());
             final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(),
                 new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
             engine.index(appendOnlyPrimary(doc, true, timestamp2));
-            assertEquals(maxTimestamp12, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+            assertEquals(maxTimestamp12, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp());
             globalCheckpoint.set(1); // make sure flush cleans up commits for later.
             engine.flush();
         }
@@ -3725,7 +3725,7 @@ public class InternalEngineTests extends EngineTestCase {
                 store.associateIndexWithNewTranslog(translogUUID);
             }
             try (Engine engine = new InternalEngine(configSupplier.apply(store))) {
-                assertEquals(maxTimestamp12, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp());
+                assertEquals(maxTimestamp12, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp());
             }
         }
     }

server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java

@@ -229,8 +229,8 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase
             IndexShard replica = shards.addReplica();
             shards.recoverReplica(replica);
 
-            SegmentsStats segmentsStats = replica.segmentStats(false);
+            SegmentsStats segmentsStats = replica.segmentStats(false, false);
-            SegmentsStats primarySegmentStats = shards.getPrimary().segmentStats(false);
+            SegmentsStats primarySegmentStats = shards.getPrimary().segmentStats(false, false);
             assertNotEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, primarySegmentStats.getMaxUnsafeAutoIdTimestamp());
             assertEquals(primarySegmentStats.getMaxUnsafeAutoIdTimestamp(), segmentsStats.getMaxUnsafeAutoIdTimestamp());
             assertNotEquals(Long.MAX_VALUE, segmentsStats.getMaxUnsafeAutoIdTimestamp());

server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java

@@ -3314,7 +3314,7 @@ public class IndexShardTests extends IndexShardTestCase {
         indexDoc(primary, "_doc", "0", "{\"foo\" : \"foo\"}");
         primary.refresh("forced refresh");
 
-        SegmentsStats ss = primary.segmentStats(randomBoolean());
+        SegmentsStats ss = primary.segmentStats(randomBoolean(), randomBoolean());
         CircuitBreaker breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
         assertThat(ss.getMemoryInBytes(), equalTo(breaker.getUsed()));
         final long preRefreshBytes = ss.getMemoryInBytes();
@@ -3323,13 +3323,13 @@ public class IndexShardTests extends IndexShardTestCase {
         indexDoc(primary, "_doc", "2", "{\"foo\" : \"baz\"}");
         indexDoc(primary, "_doc", "3", "{\"foo\" : \"eggplant\"}");
 
-        ss = primary.segmentStats(randomBoolean());
+        ss = primary.segmentStats(randomBoolean(), randomBoolean());
         breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
         assertThat(preRefreshBytes, equalTo(breaker.getUsed()));
 
         primary.refresh("refresh");
 
-        ss = primary.segmentStats(randomBoolean());
+        ss = primary.segmentStats(randomBoolean(), randomBoolean());
         breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
         assertThat(breaker.getUsed(), equalTo(ss.getMemoryInBytes()));
         assertThat(breaker.getUsed(), greaterThan(preRefreshBytes));
@@ -3339,7 +3339,7 @@ public class IndexShardTests extends IndexShardTestCase {
         // Forces a refresh with the INTERNAL scope
         ((InternalEngine) primary.getEngine()).writeIndexingBuffer();
 
-        ss = primary.segmentStats(randomBoolean());
+        ss = primary.segmentStats(randomBoolean(), randomBoolean());
         breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
         assertThat(breaker.getUsed(), equalTo(ss.getMemoryInBytes()));
         assertThat(breaker.getUsed(), greaterThan(preRefreshBytes));
@@ -3356,7 +3356,7 @@ public class IndexShardTests extends IndexShardTestCase {
         }
         primary.refresh("force refresh");
 
-        ss = primary.segmentStats(randomBoolean());
+        ss = primary.segmentStats(randomBoolean(), randomBoolean());
         breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
         assertThat(breaker.getUsed(), lessThan(postRefreshBytes));
 
@@ -3447,7 +3447,7 @@ public class IndexShardTests extends IndexShardTestCase {
         IOUtils.close(searchers);
         primary.refresh("test");
 
-        SegmentsStats ss = primary.segmentStats(randomBoolean());
+        SegmentsStats ss = primary.segmentStats(randomBoolean(), randomBoolean());
         CircuitBreaker breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING);
         long segmentMem = ss.getMemoryInBytes();
         long breakerMem = breaker.getUsed();

server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java

@@ -454,7 +454,7 @@ public class RecoverySourceHandlerTests extends ESTestCase {
         final StartRecoveryRequest request = getStartRecoveryRequest();
         final IndexShard shard = mock(IndexShard.class);
         when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class));
-        when(shard.segmentStats(anyBoolean())).thenReturn(mock(SegmentsStats.class));
+        when(shard.segmentStats(anyBoolean(), anyBoolean())).thenReturn(mock(SegmentsStats.class));
         when(shard.isRelocatedPrimary()).thenReturn(true);
         when(shard.acquireSafeIndexCommit()).thenReturn(mock(Engine.IndexCommitRef.class));
         doAnswer(invocation -> {

x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java

@@ -20,6 +20,7 @@ import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.PointValues;
 import org.apache.lucene.index.SegmentCommitInfo;
+import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.index.SortedDocValues;
 import org.apache.lucene.index.SortedNumericDocValues;
 import org.apache.lucene.index.SortedSetDocValues;
@@ -68,6 +69,7 @@ import java.util.function.Function;
 public final class FrozenEngine extends ReadOnlyEngine {
     public static final Setting<Boolean> INDEX_FROZEN = Setting.boolSetting("index.frozen", false, Setting.Property.IndexScope,
         Setting.Property.PrivateIndex);
+    private final SegmentsStats stats;
     private volatile DirectoryReader lastOpenedReader;
     private final DirectoryReader canMatchReader;
 
@@ -79,6 +81,13 @@ public final class FrozenEngine extends ReadOnlyEngine {
         try (DirectoryReader reader = DirectoryReader.open(directory)) {
             canMatchReader = ElasticsearchDirectoryReader.wrap(new RewriteCachingDirectoryReader(directory, reader.leaves()),
                 config.getShardId());
+            // we record the segment stats here - that's what the reader needs when it's open and it give the user
+            // an idea of what it can save when it's closed
+            this.stats = new SegmentsStats();
+            for (LeafReaderContext ctx : reader.getContext().leaves()) {
+                SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
+                fillSegmentStats(segmentReader, true, stats);
+            }
             success = true;
         } catch (IOException e) {
             throw new UncheckedIOException(e);
@@ -585,6 +594,21 @@ public final class FrozenEngine extends ReadOnlyEngine {
         }
     }
 
+    @Override
+    public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
+        if (includeUnloadedSegments) {
+            final SegmentsStats stats = new SegmentsStats();
+            stats.add(this.stats);
+            if (includeSegmentFileSizes == false) {
+                stats.clearFileSizes();
+            }
+            return stats;
+        } else {
+            return super.segmentsStats(includeSegmentFileSizes, includeUnloadedSegments);
+        }
+
+    }
+
     synchronized boolean isReaderOpen() {
         return lastOpenedReader != null;
     } // this is mainly for tests

x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java

@@ -117,16 +117,18 @@ public class FrozenEngineTests extends EngineTestCase {
                 listener.reset();
                 try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig)) {
                     Engine.Searcher searcher = frozenEngine.acquireSearcher("test");
-                    SegmentsStats segmentsStats = frozenEngine.segmentsStats(randomBoolean());
+                    SegmentsStats segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
                     assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
                     FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release();
                     assertEquals(1, listener.afterRefresh.get());
-                    segmentsStats = frozenEngine.segmentsStats(randomBoolean());
+                    segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
                     assertEquals(0, segmentsStats.getCount());
+                    segmentsStats = frozenEngine.segmentsStats(randomBoolean(), true);
+                    assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
                     assertEquals(1, listener.afterRefresh.get());
                     assertFalse(frozenEngine.isReaderOpen());
                     FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset();
-                    segmentsStats = frozenEngine.segmentsStats(randomBoolean());
+                    segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false);
                     assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount());
                     searcher.close();
                 }