From d112c890417a2104e9b1e2d0d03364e1f8154787 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 5 Mar 2019 14:01:41 +0100 Subject: [PATCH] Allow inclusion of unloaded segments in stats (#39512) Today we have no chance to fetch actual segment stats for segments that are currently unloaded. This is relevant in the case of frozen indices. This allows to monitor how much memory a frozen index would use if it was unfrozen. --- .../rest-api-spec/api/cat.indices.json | 5 ++ .../rest-api-spec/api/indices.stats.json | 5 ++ .../admin/indices/stats/CommonStats.java | 2 +- .../admin/indices/stats/CommonStatsFlags.java | 19 ++++++++ .../elasticsearch/index/engine/Engine.java | 4 +- .../index/engine/SegmentsStats.java | 4 ++ .../elasticsearch/index/shard/IndexShard.java | 4 +- .../admin/indices/RestIndicesStatsAction.java | 1 + .../rest/action/cat/RestIndicesAction.java | 1 + .../index/engine/InternalEngineTests.java | 46 +++++++++---------- .../IndexLevelReplicationTests.java | 4 +- .../index/shard/IndexShardTests.java | 12 ++--- .../recovery/RecoverySourceHandlerTests.java | 2 +- .../index/engine/FrozenEngine.java | 24 ++++++++++ .../index/engine/FrozenEngineTests.java | 8 ++-- 15 files changed, 101 insertions(+), 40 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json index 42a47253de7..24961a57cf3 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json @@ -57,6 +57,11 @@ "type": "boolean", "description": "Verbose mode. Display column headers", "default": false + }, + "include_unloaded_segments": { + "type": "boolean", + "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory", + "default": false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json index 223e5a14ff6..270c379baa7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json @@ -52,6 +52,11 @@ "type": "boolean", "description": "Whether to report the aggregated disk usage of each one of the Lucene index files (only applies if segment stats are requested)", "default": false + }, + "include_unloaded_segments": { + "type": "boolean", + "description": "If set to true segment stats will include stats for segments that are not currently loaded into memory", + "default": false } } }, diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index 1bf7342be95..04ec9f452d2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -207,7 +207,7 @@ public class CommonStats implements Writeable, ToXContentFragment { completion = indexShard.completionStats(flags.completionDataFields()); break; case Segments: - segments = indexShard.segmentStats(flags.includeSegmentFileSizes()); + segments = indexShard.segmentStats(flags.includeSegmentFileSizes(), flags.includeUnloadedSegments()); break; case Translog: translog = indexShard.translogStats(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index a53cc0b339d..cf09a6ff7a0 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.stats; +import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -38,6 +39,7 @@ public class CommonStatsFlags implements Writeable, Cloneable { private String[] fieldDataFields = null; private String[] completionDataFields = null; private boolean includeSegmentFileSizes = false; + private boolean includeUnloadedSegments = false; /** * @param flags flags to set. If no flags are supplied, default flags will be set. @@ -62,6 +64,9 @@ public class CommonStatsFlags implements Writeable, Cloneable { fieldDataFields = in.readStringArray(); completionDataFields = in.readStringArray(); includeSegmentFileSizes = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + includeUnloadedSegments = in.readBoolean(); + } } @Override @@ -77,6 +82,9 @@ public class CommonStatsFlags implements Writeable, Cloneable { out.writeStringArrayNullable(fieldDataFields); out.writeStringArrayNullable(completionDataFields); out.writeBoolean(includeSegmentFileSizes); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeBoolean(includeUnloadedSegments); + } } /** @@ -89,6 +97,7 @@ public class CommonStatsFlags implements Writeable, Cloneable { fieldDataFields = null; completionDataFields = null; includeSegmentFileSizes = false; + includeUnloadedSegments = false; return this; } @@ -102,6 +111,7 @@ public class CommonStatsFlags implements Writeable, Cloneable { fieldDataFields = null; completionDataFields = null; includeSegmentFileSizes = false; + includeUnloadedSegments = false; return this; } @@ -170,6 +180,15 @@ public class CommonStatsFlags implements Writeable, Cloneable { return this; } + public CommonStatsFlags includeUnloadedSegments(boolean includeUnloadedSegments) { + this.includeUnloadedSegments = includeUnloadedSegments; + return this; + } + + public boolean includeUnloadedSegments() { + return this.includeUnloadedSegments; + } + public boolean includeSegmentFileSizes() { return this.includeSegmentFileSizes; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index cead47520bb..55b1ce6e699 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -821,7 +821,7 @@ public abstract class Engine implements Closeable { /** * Global stats on segments. */ - public SegmentsStats segmentsStats(boolean includeSegmentFileSizes) { + public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { ensureOpen(); Set segmentName = new HashSet<>(); SegmentsStats stats = new SegmentsStats(); @@ -845,7 +845,7 @@ public abstract class Engine implements Closeable { return stats; } - private void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) { + protected void fillSegmentStats(SegmentReader segmentReader, boolean includeSegmentFileSizes, SegmentsStats stats) { stats.add(1, segmentReader.ramBytesUsed()); stats.addTermsMemoryInBytes(guardedRamBytesUsed(segmentReader.getPostingsReader())); stats.addStoredFieldsMemoryInBytes(guardedRamBytesUsed(segmentReader.getFieldsReader())); diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java index 72a938e725e..583eb897c46 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java +++ b/server/src/main/java/org/elasticsearch/index/engine/SegmentsStats.java @@ -391,4 +391,8 @@ public class SegmentsStats implements Streamable, ToXContentFragment { out.writeLong(entry.value.longValue()); } } + + public void clearFileSizes() { + fileSizes = ImmutableOpenMap.of(); + } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index a43e8465795..a55f886d3c6 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -999,8 +999,8 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl return engine.getMergeStats(); } - public SegmentsStats segmentStats(boolean includeSegmentFileSizes) { - SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes); + public SegmentsStats segmentStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { + SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes, includeUnloadedSegments); segmentsStats.addBitsetMemoryInBytes(shardBitsetFilterCache.getMemorySizeInBytes()); return segmentsStats; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java index 715da019e6d..85d9dc29ea8 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndicesStatsAction.java @@ -121,6 +121,7 @@ public class RestIndicesStatsAction extends BaseRestHandler { if (indicesStatsRequest.segments()) { indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_segment_file_sizes", false)); + indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false)); } return channel -> client.admin().indices().stats(indicesStatsRequest, new RestToXContentListener<>(channel)); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index 24f3ac8522a..e28b85c6299 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -116,6 +116,7 @@ public class RestIndicesAction extends AbstractCatAction { indicesStatsRequest.indices(indices); indicesStatsRequest.indicesOptions(strictExpandIndicesOptions); indicesStatsRequest.all(); + indicesStatsRequest.includeSegmentFileSizes(request.paramAsBoolean("include_unloaded_segments", false)); client.admin().indices().stats(indicesStatsRequest, new RestResponseListener(channel) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index d9808277d17..7f34b07d803 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -272,8 +272,8 @@ public class InternalEngineTests extends EngineTestCase { InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null))) { List segments = engine.segments(false); assertThat(segments.isEmpty(), equalTo(true)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(0L)); - assertThat(engine.segmentsStats(false).getMemoryInBytes(), equalTo(0L)); + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(0L)); + assertThat(engine.segmentsStats(false, false).getMemoryInBytes(), equalTo(0L)); // create two docs and refresh ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null); @@ -287,7 +287,7 @@ public class InternalEngineTests extends EngineTestCase { segments = engine.segments(false); assertThat(segments.size(), equalTo(1)); - SegmentsStats stats = engine.segmentsStats(false); + SegmentsStats stats = engine.segmentsStats(false, false); assertThat(stats.getCount(), equalTo(1L)); assertThat(stats.getTermsMemoryInBytes(), greaterThan(0L)); assertThat(stats.getStoredFieldsMemoryInBytes(), greaterThan(0L)); @@ -306,7 +306,7 @@ public class InternalEngineTests extends EngineTestCase { segments = engine.segments(false); assertThat(segments.size(), equalTo(1)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(1L)); + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(1L)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); assertThat(segments.get(0).getNumDocs(), equalTo(2)); @@ -319,15 +319,15 @@ public class InternalEngineTests extends EngineTestCase { segments = engine.segments(false); assertThat(segments.size(), equalTo(2)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(2L)); - assertThat(engine.segmentsStats(false).getTermsMemoryInBytes(), + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L)); + assertThat(engine.segmentsStats(false, false).getTermsMemoryInBytes(), greaterThan(stats.getTermsMemoryInBytes())); - assertThat(engine.segmentsStats(false).getStoredFieldsMemoryInBytes(), + assertThat(engine.segmentsStats(false, false).getStoredFieldsMemoryInBytes(), greaterThan(stats.getStoredFieldsMemoryInBytes())); - assertThat(engine.segmentsStats(false).getTermVectorsMemoryInBytes(), equalTo(0L)); - assertThat(engine.segmentsStats(false).getNormsMemoryInBytes(), + assertThat(engine.segmentsStats(false, false).getTermVectorsMemoryInBytes(), equalTo(0L)); + assertThat(engine.segmentsStats(false, false).getNormsMemoryInBytes(), greaterThan(stats.getNormsMemoryInBytes())); - assertThat(engine.segmentsStats(false).getDocValuesMemoryInBytes(), + assertThat(engine.segmentsStats(false, false).getDocValuesMemoryInBytes(), greaterThan(stats.getDocValuesMemoryInBytes())); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); @@ -349,7 +349,7 @@ public class InternalEngineTests extends EngineTestCase { segments = engine.segments(false); assertThat(segments.size(), equalTo(2)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(2L)); + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(2L)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); @@ -370,7 +370,7 @@ public class InternalEngineTests extends EngineTestCase { segments = engine.segments(false); assertThat(segments.size(), equalTo(3)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(3L)); + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(3L)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); @@ -397,7 +397,7 @@ public class InternalEngineTests extends EngineTestCase { segments = engine.segments(false); assertThat(segments.size(), equalTo(4)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(4L)); + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); @@ -427,7 +427,7 @@ public class InternalEngineTests extends EngineTestCase { engine.refresh("test"); segments = engine.segments(false); assertThat(segments.size(), equalTo(4)); - assertThat(engine.segmentsStats(false).getCount(), equalTo(4L)); + assertThat(engine.segmentsStats(false, false).getCount(), equalTo(4L)); assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); assertThat(segments.get(0).isCommitted(), equalTo(true)); assertThat(segments.get(0).isSearch(), equalTo(true)); @@ -572,13 +572,13 @@ public class InternalEngineTests extends EngineTestCase { public void testSegmentsStatsIncludingFileSizes() throws Exception { try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE)) { - assertThat(engine.segmentsStats(true).getFileSizes().size(), equalTo(0)); + assertThat(engine.segmentsStats(true, false).getFileSizes().size(), equalTo(0)); ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null); engine.index(indexForDoc(doc)); engine.refresh("test"); - SegmentsStats stats = engine.segmentsStats(true); + SegmentsStats stats = engine.segmentsStats(true, false); assertThat(stats.getFileSizes().size(), greaterThan(0)); assertThat(() -> stats.getFileSizes().valuesIt(), everyItem(greaterThan(0L))); @@ -588,7 +588,7 @@ public class InternalEngineTests extends EngineTestCase { engine.index(indexForDoc(doc2)); engine.refresh("test"); - assertThat(engine.segmentsStats(true).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value)); + assertThat(engine.segmentsStats(true, false).getFileSizes().get(firstEntry.key), greaterThan(firstEntry.value)); } } @@ -3698,23 +3698,23 @@ public class InternalEngineTests extends EngineTestCase { NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get); try (Store store = createStore(newFSDirectory(storeDir)); Engine engine = createEngine(configSupplier.apply(store))) { assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, - engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp()); + engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp()); final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); engine.index(appendOnlyPrimary(doc, true, timestamp1)); - assertEquals(timestamp1, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp()); + assertEquals(timestamp1, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp()); } try (Store store = createStore(newFSDirectory(storeDir)); InternalEngine engine = new InternalEngine(configSupplier.apply(store))) { assertEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, - engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp()); + engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp()); engine.initializeMaxSeqNoOfUpdatesOrDeletes(); engine.recoverFromTranslog(translogHandler, Long.MAX_VALUE); - assertEquals(timestamp1, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp()); + assertEquals(timestamp1, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp()); final ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null); engine.index(appendOnlyPrimary(doc, true, timestamp2)); - assertEquals(maxTimestamp12, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp()); + assertEquals(maxTimestamp12, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp()); globalCheckpoint.set(1); // make sure flush cleans up commits for later. engine.flush(); } @@ -3725,7 +3725,7 @@ public class InternalEngineTests extends EngineTestCase { store.associateIndexWithNewTranslog(translogUUID); } try (Engine engine = new InternalEngine(configSupplier.apply(store))) { - assertEquals(maxTimestamp12, engine.segmentsStats(false).getMaxUnsafeAutoIdTimestamp()); + assertEquals(maxTimestamp12, engine.segmentsStats(false, false).getMaxUnsafeAutoIdTimestamp()); } } } diff --git a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java index f67ce8de142..d24124a1048 100644 --- a/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/index/replication/IndexLevelReplicationTests.java @@ -229,8 +229,8 @@ public class IndexLevelReplicationTests extends ESIndexLevelReplicationTestCase IndexShard replica = shards.addReplica(); shards.recoverReplica(replica); - SegmentsStats segmentsStats = replica.segmentStats(false); - SegmentsStats primarySegmentStats = shards.getPrimary().segmentStats(false); + SegmentsStats segmentsStats = replica.segmentStats(false, false); + SegmentsStats primarySegmentStats = shards.getPrimary().segmentStats(false, false); assertNotEquals(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, primarySegmentStats.getMaxUnsafeAutoIdTimestamp()); assertEquals(primarySegmentStats.getMaxUnsafeAutoIdTimestamp(), segmentsStats.getMaxUnsafeAutoIdTimestamp()); assertNotEquals(Long.MAX_VALUE, segmentsStats.getMaxUnsafeAutoIdTimestamp()); diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index e02fdfce6f7..f94a12be054 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -3314,7 +3314,7 @@ public class IndexShardTests extends IndexShardTestCase { indexDoc(primary, "_doc", "0", "{\"foo\" : \"foo\"}"); primary.refresh("forced refresh"); - SegmentsStats ss = primary.segmentStats(randomBoolean()); + SegmentsStats ss = primary.segmentStats(randomBoolean(), randomBoolean()); CircuitBreaker breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING); assertThat(ss.getMemoryInBytes(), equalTo(breaker.getUsed())); final long preRefreshBytes = ss.getMemoryInBytes(); @@ -3323,13 +3323,13 @@ public class IndexShardTests extends IndexShardTestCase { indexDoc(primary, "_doc", "2", "{\"foo\" : \"baz\"}"); indexDoc(primary, "_doc", "3", "{\"foo\" : \"eggplant\"}"); - ss = primary.segmentStats(randomBoolean()); + ss = primary.segmentStats(randomBoolean(), randomBoolean()); breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING); assertThat(preRefreshBytes, equalTo(breaker.getUsed())); primary.refresh("refresh"); - ss = primary.segmentStats(randomBoolean()); + ss = primary.segmentStats(randomBoolean(), randomBoolean()); breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING); assertThat(breaker.getUsed(), equalTo(ss.getMemoryInBytes())); assertThat(breaker.getUsed(), greaterThan(preRefreshBytes)); @@ -3339,7 +3339,7 @@ public class IndexShardTests extends IndexShardTestCase { // Forces a refresh with the INTERNAL scope ((InternalEngine) primary.getEngine()).writeIndexingBuffer(); - ss = primary.segmentStats(randomBoolean()); + ss = primary.segmentStats(randomBoolean(), randomBoolean()); breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING); assertThat(breaker.getUsed(), equalTo(ss.getMemoryInBytes())); assertThat(breaker.getUsed(), greaterThan(preRefreshBytes)); @@ -3356,7 +3356,7 @@ public class IndexShardTests extends IndexShardTestCase { } primary.refresh("force refresh"); - ss = primary.segmentStats(randomBoolean()); + ss = primary.segmentStats(randomBoolean(), randomBoolean()); breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING); assertThat(breaker.getUsed(), lessThan(postRefreshBytes)); @@ -3447,7 +3447,7 @@ public class IndexShardTests extends IndexShardTestCase { IOUtils.close(searchers); primary.refresh("test"); - SegmentsStats ss = primary.segmentStats(randomBoolean()); + SegmentsStats ss = primary.segmentStats(randomBoolean(), randomBoolean()); CircuitBreaker breaker = primary.circuitBreakerService.getBreaker(CircuitBreaker.ACCOUNTING); long segmentMem = ss.getMemoryInBytes(); long breakerMem = breaker.getUsed(); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index ae6eae86564..fb7b79f4597 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -454,7 +454,7 @@ public class RecoverySourceHandlerTests extends ESTestCase { final StartRecoveryRequest request = getStartRecoveryRequest(); final IndexShard shard = mock(IndexShard.class); when(shard.seqNoStats()).thenReturn(mock(SeqNoStats.class)); - when(shard.segmentStats(anyBoolean())).thenReturn(mock(SegmentsStats.class)); + when(shard.segmentStats(anyBoolean(), anyBoolean())).thenReturn(mock(SegmentsStats.class)); when(shard.isRelocatedPrimary()).thenReturn(true); when(shard.acquireSafeIndexCommit()).thenReturn(mock(Engine.IndexCommitRef.class)); doAnswer(invocation -> { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java index 7fbb58504ef..9543e1a2a04 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java @@ -20,6 +20,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; @@ -68,6 +69,7 @@ import java.util.function.Function; public final class FrozenEngine extends ReadOnlyEngine { public static final Setting INDEX_FROZEN = Setting.boolSetting("index.frozen", false, Setting.Property.IndexScope, Setting.Property.PrivateIndex); + private final SegmentsStats stats; private volatile DirectoryReader lastOpenedReader; private final DirectoryReader canMatchReader; @@ -79,6 +81,13 @@ public final class FrozenEngine extends ReadOnlyEngine { try (DirectoryReader reader = DirectoryReader.open(directory)) { canMatchReader = ElasticsearchDirectoryReader.wrap(new RewriteCachingDirectoryReader(directory, reader.leaves()), config.getShardId()); + // we record the segment stats here - that's what the reader needs when it's open and it give the user + // an idea of what it can save when it's closed + this.stats = new SegmentsStats(); + for (LeafReaderContext ctx : reader.getContext().leaves()) { + SegmentReader segmentReader = Lucene.segmentReader(ctx.reader()); + fillSegmentStats(segmentReader, true, stats); + } success = true; } catch (IOException e) { throw new UncheckedIOException(e); @@ -585,6 +594,21 @@ public final class FrozenEngine extends ReadOnlyEngine { } } + @Override + public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) { + if (includeUnloadedSegments) { + final SegmentsStats stats = new SegmentsStats(); + stats.add(this.stats); + if (includeSegmentFileSizes == false) { + stats.clearFileSizes(); + } + return stats; + } else { + return super.segmentsStats(includeSegmentFileSizes, includeUnloadedSegments); + } + + } + synchronized boolean isReaderOpen() { return lastOpenedReader != null; } // this is mainly for tests diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java index e9cb444183f..17019b0ac18 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/FrozenEngineTests.java @@ -117,16 +117,18 @@ public class FrozenEngineTests extends EngineTestCase { listener.reset(); try (FrozenEngine frozenEngine = new FrozenEngine(engine.engineConfig)) { Engine.Searcher searcher = frozenEngine.acquireSearcher("test"); - SegmentsStats segmentsStats = frozenEngine.segmentsStats(randomBoolean()); + SegmentsStats segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false); assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount()); FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).release(); assertEquals(1, listener.afterRefresh.get()); - segmentsStats = frozenEngine.segmentsStats(randomBoolean()); + segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false); assertEquals(0, segmentsStats.getCount()); + segmentsStats = frozenEngine.segmentsStats(randomBoolean(), true); + assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount()); assertEquals(1, listener.afterRefresh.get()); assertFalse(frozenEngine.isReaderOpen()); FrozenEngine.unwrapLazyReader(searcher.getDirectoryReader()).reset(); - segmentsStats = frozenEngine.segmentsStats(randomBoolean()); + segmentsStats = frozenEngine.segmentsStats(randomBoolean(), false); assertEquals(frozenEngine.segments(randomBoolean()).size(), segmentsStats.getCount()); searcher.close(); }