From 235f57989fbb243a04d0a26914ce85bc27905404 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 20 Mar 2019 11:24:29 +0100 Subject: [PATCH] Return cached segments stats if `include_unloaded_segments` is true (#39698) Today we don't return segments stats for closed indices which makes it hard to tell how much memory such an index would require. With this change we return the statistics if requested by setting `include_unloaded_segments` to true on the rest request. Relates to #39512 --- .../rest-api-spec/api/indices.stats.json | 11 ++++ .../test/indices.stats/30_segments.yml | 63 +++++++++++++++++++ .../indices/stats/IndicesStatsRequest.java | 5 ++ .../index/engine/NoOpEngine.java | 42 +++++++++++-- .../admin/indices/RestIndicesStatsAction.java | 9 ++- .../rest/action/cat/RestIndicesAction.java | 2 +- .../index/engine/NoOpEngineTests.java | 12 +++- 7 files changed, 134 insertions(+), 10 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json index 270c379baa7..c86a2c1147a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json @@ -57,6 +57,17 @@ "type": "boolean", "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory", "default": false + }, + "expand_wildcards": { + "type" : "enum", + "options" : ["open","closed","none","all"], + "default" : "open", + "description" : "Whether to expand wildcard expression to concrete indices that are open, closed or both." + }, + "forbid_closed_indices": { + "type": "boolean", + "description": "If set to false stats will also collected from closed indices if explicitly specified or if expand_wildcards expands to closed indices", + "default": true } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml new file mode 100644 index 00000000000..9333006041a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.stats/30_segments.yml @@ -0,0 +1,63 @@ +--- +setup: + - do: + indices.create: + index: test + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + + - do: + cluster.health: + wait_for_no_initializing_shards: true + +--- +"Segment Stats": + + - skip: + version: " - 7.0.99" + reason: forbid_closed_indices is not supported in ealier version + + - do: + indices.stats: + metric: [ segments ] + - set: { indices.test.primaries.segments.count: num_segments } + + - do: + index: + index: test + id: 1 + body: { "foo": "bar" } + + - do: + indices.flush: + index: test + + - do: + indices.stats: + metric: [ segments ] + - gt: { indices.test.primaries.segments.count: $num_segments } + - set: { indices.test.primaries.segments.count: num_segments_after_flush } + + - do: + indices.close: + index: test + wait_for_active_shards: "all" + + - do: + indices.stats: + metric: segments + expand_wildcards: closed + forbid_closed_indices: false + + - match: { indices.test.primaries.segments.count: 0 } + + - do: + indices.stats: + metric: segments + include_unloaded_segments: true + expand_wildcards: closed + forbid_closed_indices: false + + - match: { indices.test.primaries.segments.count: $num_segments_after_flush } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java index ebc085ef47e..b162a232585 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java @@ -271,6 +271,11 @@ public class IndicesStatsRequest extends BroadcastRequest { return this; } + public IndicesStatsRequest includeUnloadedSegments(boolean includeUnloadedSegments) { + flags.includeUnloadedSegments(includeUnloadedSegments); + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java index fe1ad7a1a14..a41f07e994b 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java @@ -23,9 +23,13 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexCommit; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; +import org.elasticsearch.common.lucene.Lucene; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.List; import java.util.function.Function; @@ -36,8 +40,20 @@ import java.util.function.Function; */ public final class NoOpEngine extends ReadOnlyEngine { + private final SegmentsStats stats; + public NoOpEngine(EngineConfig config) { super(config, null, null, true, Function.identity()); + this.stats = new SegmentsStats(); + Directory directory = store.directory(); + try (DirectoryReader reader = DirectoryReader.open(directory)) { + for (LeafReaderContext ctx : reader.getContext().leaves()) { + SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); + fillSegmentStats(segmentReader, true, stats); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } } @Override @@ -47,17 +63,17 @@ public final class NoOpEngine extends ReadOnlyEngine { final IndexCommit indexCommit = indexCommits.get(indexCommits.size() - 1); return new DirectoryReader(directory, new LeafReader[0]) { @Override - protected DirectoryReader doOpenIfChanged() throws IOException { + protected DirectoryReader doOpenIfChanged() { return null; } @Override - protected DirectoryReader doOpenIfChanged(IndexCommit commit) throws IOException { + protected DirectoryReader doOpenIfChanged(IndexCommit commit) { return null; } @Override - protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) throws IOException { + protected DirectoryReader doOpenIfChanged(IndexWriter writer, boolean applyAllDeletes) { return null; } @@ -67,17 +83,17 @@ public final class NoOpEngine extends ReadOnlyEngine { } @Override - public boolean isCurrent() throws IOException { + public boolean isCurrent() { return true; } @Override - public IndexCommit getIndexCommit() throws IOException { + public IndexCommit getIndexCommit() { return indexCommit; } @Override - protected void doClose() throws IOException { + protected void doClose() { } @Override @@ -86,4 +102,18 @@ public final class NoOpEngine extends ReadOnlyEngine { } }; } + + @Override + public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { + if (includeUnloadedSegments) { + final SegmentsStats stats = new SegmentsStats(); + stats.add(this.stats); + if (includeSegmentFileSizes == false) { + stats.clearFileSizes(); + } + return stats; + } else { + return super.segmentsStats(includeSegmentFileSizes, includeUnloadedSegments); + } + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java index 85d9dc29ea8..c2d16ce5ac6 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java @@ -69,7 +69,12 @@ public class RestIndicesStatsAction extends BaseRestHandler { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { IndicesStatsRequest indicesStatsRequest = new IndicesStatsRequest(); - indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, indicesStatsRequest.indicesOptions())); + boolean forbidClosedIndices = request.paramAsBoolean("forbid_closed_indices", true); + IndicesOptions defaultIndicesOption = forbidClosedIndices ? indicesStatsRequest.indicesOptions() + : IndicesOptions.strictExpandOpen(); + assert indicesStatsRequest.indicesOptions() == IndicesOptions.strictExpandOpenAndForbidClosed() : "IndicesStats default indices " + + "options changed"; + indicesStatsRequest.indicesOptions(IndicesOptions.fromRequest(request, defaultIndicesOption)); indicesStatsRequest.indices(Strings.splitStringByCommaToArray(request.param("index"))); indicesStatsRequest.types(Strings.splitStringByCommaToArray(request.param("types"))); @@ -121,7 +126,7 @@ public class RestIndicesStatsAction extends BaseRestHandler { if (indicesStatsRequest.segments()) { indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false)); - indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false)); + indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false)); } return channel -> client.admin().indices().stats(indicesStatsRequest, new RestToXContentListener<>(channel)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index e28b85c6299..fb560cf7f77 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -116,7 +116,7 @@ public class RestIndicesAction extends AbstractCatAction { indicesStatsRequest.indices(indices); indicesStatsRequest.indicesOptions(strictExpandIndicesOptions); indicesStatsRequest.all(); - indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false)); + indicesStatsRequest.includeUnloadedSegments(request.paramAsBoolean("include_unloaded_segments", false)); client.admin().indices().stats(indicesStatsRequest, new RestResponseListener(channel) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java index b70ccf03aac..3a857a20468 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java @@ -100,7 +100,7 @@ public class NoOpEngineTests extends EngineTestCase { noOpEngine.close(); } - public void testNoOpEngineDocStats() throws Exception { + public void testNoOpEngineStats() throws Exception { IOUtils.close(engine, store); final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); try (Store store = createStore()) { @@ -131,8 +131,11 @@ public class NoOpEngineTests extends EngineTestCase { } final DocsStats expectedDocStats; + boolean includeFileSize = randomBoolean(); + final SegmentsStats expectedSegmentStats; try (InternalEngine engine = createEngine(config)) { expectedDocStats = engine.docStats(); + expectedSegmentStats = engine.segmentsStats(includeFileSize, true); } try (NoOpEngine noOpEngine = new NoOpEngine(config)) { @@ -140,6 +143,13 @@ public class NoOpEngineTests extends EngineTestCase { assertEquals(expectedDocStats.getDeleted(), noOpEngine.docStats().getDeleted()); assertEquals(expectedDocStats.getTotalSizeInBytes(), noOpEngine.docStats().getTotalSizeInBytes()); assertEquals(expectedDocStats.getAverageSizeInBytes(), noOpEngine.docStats().getAverageSizeInBytes()); + assertEquals(expectedSegmentStats.getCount(), noOpEngine.segmentsStats(includeFileSize, true).getCount()); + assertEquals(expectedSegmentStats.getMemoryInBytes(), noOpEngine.segmentsStats(includeFileSize, true).getMemoryInBytes()); + assertEquals(expectedSegmentStats.getFileSizes().size(), + noOpEngine.segmentsStats(includeFileSize, true).getFileSizes().size()); + + assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getFileSizes().size()); + assertEquals(0, noOpEngine.segmentsStats(includeFileSize, false).getMemoryInBytes()); } catch (AssertionError e) { logger.error(config.getMergePolicy()); throw e;