Return cached segments stats if `include_unloaded_segments` is true (#39698)

Today we don't return segments stats for closed indices which makes it
hard to tell how much memory such an index would require. With this change
we return the statistics if requested by setting `include_unloaded_segments` to
true on the rest request.

Relates to #39512

rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json

@@ -57,6 +57,17 @@
           "type": "boolean",
           "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory",
           "default": false
+        },
+        "expand_wildcards": {
+          "type" : "enum",
+          "options" : ["open","closed","none","all"],
+          "default" : "open",
+          "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both."
+        },
+        "forbid_closed_indices": {
+          "type": "boolean",
+          "description": "If set to false stats will also collected from closed indices if explicitly specified or if expand_wildcards expands to closed indices",
+          "default": true
         }
       }
     },

rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml

@@ -0,0 +1,63 @@
+---
+setup:
+  - do:
+      indices.create:
+          index: test
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 0
+
+  - do:
+      cluster.health:
+        wait_for_no_initializing_shards: true
+
+---
+"Segment Stats":
+
+  - skip:
+      version: " - 7.0.99"
+      reason: forbid_closed_indices is not supported in ealier version
+
+  - do:
+      indices.stats:
+        metric: [ segments ]
+  - set: { indices.test.primaries.segments.count: num_segments }
+
+  - do:
+      index:
+        index: test
+        id:    1
+        body:  { "foo": "bar" }
+
+  - do:
+      indices.flush:
+        index: test
+
+  - do:
+      indices.stats:
+        metric: [ segments ]
+  - gt: { indices.test.primaries.segments.count: $num_segments }
+  - set: { indices.test.primaries.segments.count: num_segments_after_flush }
+
+  - do:
+      indices.close:
+        index: test
+        wait_for_active_shards: "all"
+
+  - do:
+      indices.stats:
+        metric: segments
+        expand_wildcards: closed
+        forbid_closed_indices: false
+
+  - match: { indices.test.primaries.segments.count: 0 }
+
+  - do:
+      indices.stats:
+        metric: segments
+        include_unloaded_segments: true
+        expand_wildcards: closed
+        forbid_closed_indices: false
+
+  - match: { indices.test.primaries.segments.count: $num_segments_after_flush }

server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java

@@ -271,6 +271,11 @@ public class IndicesStatsRequest extends BroadcastRequest<IndicesStatsRequest> {
         return this;
     }
 
+    public IndicesStatsRequest includeUnloadedSegments(boolean includeUnloadedSegments) {
+        flags.includeUnloadedSegments(includeUnloadedSegments);
+        return this;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);

server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java

@@ -23,9 +23,13 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.SegmentReader;
 import org.apache.lucene.store.Directory;
+import org.elasticsearch.common.lucene.Lucene;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.List;
 import java.util.function.Function;
 
@@ -36,8 +40,20 @@ import java.util.function.Function;
  */
 public final class NoOpEngine extends ReadOnlyEngine {
 
+    private final SegmentsStats stats;
+
     public NoOpEngine(EngineConfig config) {
         super(config, null, null, true, Function.identity());
+        this.stats = new SegmentsStats();
+        Directory directory = store.directory();
+        try (DirectoryReader reader = DirectoryReader.open(directory)) {
+            for (LeafReaderContext ctx : reader.getContext().leaves()) {
+                SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
+                fillSegmentStats(segmentReader, true, stats);
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 
     @Override
@@ -47,17 +63,17 @@ public final class NoOpEngine extends ReadOnlyEngine {
         final IndexCommit indexCommit = indexCommits.get(indexCommits.size() - 1);
         return new DirectoryReader(directory, new LeafReader[0]) {
             @Override
-            protected DirectoryReader doOpenIfChanged() throws IOException {
+            protected DirectoryReader doOpenIfChanged() {
                 return null;
             }
 
             @Override
-            protected DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException {
+            protected DirectoryReader doOpenIfChanged(IndexCommit commit) {
                 return null;
             }
 
             @Override
-            protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException {
+            protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) {
                 return null;
             }
 
@@ -67,17 +83,17 @@ public final class NoOpEngine extends ReadOnlyEngine {
             }
 
             @Override
-            public boolean isCurrent() throws IOException {
+            public boolean isCurrent() {
                 return true;
             }
 
             @Override
-            public IndexCommit getIndexCommit() throws IOException {
+            public IndexCommit getIndexCommit()  {
                 return indexCommit;
             }
 
             @Override
-            protected void doClose() throws IOException {
+            protected void doClose() {
             }
 
             @Override
@@ -86,4 +102,18 @@ public final class NoOpEngine extends ReadOnlyEngine {
             }
         };
     }
+
+    @Override
+    public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
+        if (includeUnloadedSegments) {
+            final SegmentsStats stats = new SegmentsStats();
+            stats.add(this.stats);
+            if (includeSegmentFileSizes == false) {
+                stats.clearFileSizes();
+            }
+            return stats;
+        } else {
+            return super.segmentsStats(includeSegmentFileSizes, includeUnloadedSegments);
+        }
+    }
 }

server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java

@@ -69,7 +69,12 @@ public class RestIndicesStatsAction extends BaseRestHandler {
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest();
-        indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesStatsRequest.indicesOptions()));
+        boolean forbidClosedIndices = request.paramAsBoolean("forbid_closed_indices", true);
+        IndicesOptions defaultIndicesOption = forbidClosedIndices ? indicesStatsRequest.indicesOptions()
+            : IndicesOptions.strictExpandOpen();
+        assert indicesStatsRequest.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() : "IndicesStats default indices " +
+            "options changed";
+        indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, defaultIndicesOption));
         indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index")));
         indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types")));
 
@@ -121,7 +126,7 @@ public class RestIndicesStatsAction extends BaseRestHandler {
 
         if (indicesStatsRequest.segments()) {
             indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false));
-            indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false));
+            indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false));
         }
 
         return channel -> client.admin().indices().stats(indicesStatsRequest, new RestToXContentListener<>(channel));

server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -116,7 +116,7 @@ public class RestIndicesAction extends AbstractCatAction {
                         indicesStatsRequest.indices(indices);
                         indicesStatsRequest.indicesOptions(strictExpandIndicesOptions);
                         indicesStatsRequest.all();
-                        indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false));
+                        indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false));
 
                         client.admin().indices().stats(indicesStatsRequest, new RestResponseListener<IndicesStatsResponse>(channel) {
                             @Override

server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java

@@ -100,7 +100,7 @@ public class NoOpEngineTests extends EngineTestCase {
         noOpEngine.close();
     }
 
-    public void testNoOpEngineDocStats() throws Exception {
+    public void testNoOpEngineStats() throws Exception {
         IOUtils.close(engine, store);
         final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
         try (Store store = createStore()) {
@@ -131,8 +131,11 @@ public class NoOpEngineTests extends EngineTestCase {
             }
 
             final DocsStats expectedDocStats;
+            boolean includeFileSize = randomBoolean();
+            final SegmentsStats expectedSegmentStats;
             try (InternalEngine engine = createEngine(config)) {
                 expectedDocStats = engine.docStats();
+                expectedSegmentStats = engine.segmentsStats(includeFileSize, true);
             }
 
             try (NoOpEngine noOpEngine = new NoOpEngine(config)) {
@@ -140,6 +143,13 @@ public class NoOpEngineTests extends EngineTestCase {
                 assertEquals(expectedDocStats.getDeleted(), noOpEngine.docStats().getDeleted());
                 assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes());
                 assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes());
+                assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount());
+                assertEquals(expectedSegmentStats.getMemoryInBytes(), noOpEngine.segmentsStats(includeFileSize, true).getMemoryInBytes());
+                assertEquals(expectedSegmentStats.getFileSizes().size(),
+                    noOpEngine.segmentsStats(includeFileSize, true).getFileSizes().size());
+
+                assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getFileSizes().size());
+                assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getMemoryInBytes());
             } catch (AssertionError e) {
                 logger.error(config.getMergePolicy());
                 throw e;