honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Do not wrap soft-deletes reader for segment stats (#51331) 

IndexWriter might not filter out fully deleted segments if retention
leases exist or the number of the retaining operations is non-zero.
SoftDeletesDirectoryReaderWrapper, however, always filters out fully
deleted segments.

This change uses the original directory reader when calculating segment
stats instead.

Relates #51192
Closes #51303
This commit is contained in:
Nhat Nguyen 2020-01-23 08:26:27 -05:00
parent 4e8ab43a3e
commit acf84b68cb
3 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java
View File

@ -54,7 +54,8 @@ public final class NoOpEngine extends ReadOnlyEngine {
super(config, null, null, true, Function.identity());
this.stats = new SegmentsStats();
Directory directory = store.directory();
try (DirectoryReader reader = openDirectory(directory, config.getIndexSettings())) {
// Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
try (DirectoryReader reader = openDirectory(directory, false)) {
for (LeafReaderContext ctx : reader.getContext().leaves()) {
SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
fillSegmentStats(segmentReader, true, stats);

7
server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java
View File

@ -33,7 +33,6 @@ import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
import org.elasticsearch.common.util.concurrent.ReleasableLock;
import org.elasticsearch.core.internal.io.IOUtils;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.seqno.SeqNoStats;
import org.elasticsearch.index.seqno.SequenceNumbers;
@ -536,9 +535,9 @@ public class ReadOnlyEngine extends Engine {
maxSeqNoOfUpdatesOnPrimary + ">" + getMaxSeqNoOfUpdatesOrDeletes();
}
protected DirectoryReader openDirectory(Directory dir, IndexSettings indexSettings) throws IOException {
final DirectoryReader reader = DirectoryReader.open(dir, OFF_HEAP_READER_ATTRIBUTES);
if (indexSettings.isSoftDeleteEnabled()) {
protected static DirectoryReader openDirectory(Directory directory, boolean wrapSoftDeletes) throws IOException {
final DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES);
if (wrapSoftDeletes) {
return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
} else {
return reader;

13
x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java
View File

@ -21,6 +21,7 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.index.SegmentCommitInfo;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@ -77,9 +78,8 @@ public final class FrozenEngine extends ReadOnlyEngine {
boolean success = false;
Directory directory = store.directory();
try (DirectoryReader reader = openDirectory(directory, config.getIndexSettings())) {
canMatchReader = ElasticsearchDirectoryReader.wrap(new RewriteCachingDirectoryReader(directory, reader.leaves()),
config.getShardId());
// Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
try (DirectoryReader reader = openDirectory(directory, false)) {
// we record the segment stats here - that's what the reader needs when it's open and it give the user
// an idea of what it can save when it's closed
this.stats = new SegmentsStats();
@ -87,6 +87,10 @@ public final class FrozenEngine extends ReadOnlyEngine {
SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
fillSegmentStats(segmentReader, true, stats);
}
final DirectoryReader wrappedReader = config.getIndexSettings().isSoftDeleteEnabled() ?
new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD) : reader;
canMatchReader = ElasticsearchDirectoryReader.wrap(
new RewriteCachingDirectoryReader(directory, wrappedReader.leaves()), config.getShardId());
success = true;
} catch (IOException e) {
throw new UncheckedIOException(e);
@ -167,7 +171,8 @@ public final class FrozenEngine extends ReadOnlyEngine {
for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
listeners.beforeRefresh();
}
final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory(), engineConfig.getIndexSettings());
final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory(),
engineConfig.getIndexSettings().isSoftDeleteEnabled());
reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
processReader(reader);
reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);