honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Account soft deletes in committed segments (#43126) 

This change fixes the delete count issue in segment stats where we don't
account soft-deleted documents from committed segments.

Relates #43103
This commit is contained in:
Nhat Nguyen 2019-06-14 12:43:44 -04:00
parent c3f1e6a542
commit 4b643c50fa
2 changed files with 66 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/src/main/java/org/elasticsearch/index/engine/Engine.java
View File

@ -933,7 +933,7 @@ public abstract class Engine implements Closeable {
/** How much heap is used that would be freed by a refresh. Note that this may throw {@link AlreadyClosedException}. */ /** How much heap is used that would be freed by a refresh. Note that this may throw {@link AlreadyClosedException}. */
public abstract long getIndexBufferRAMBytesUsed(); public abstract long getIndexBufferRAMBytesUsed();
protected Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) { final Segment[] getSegmentInfo(SegmentInfos lastCommittedSegmentInfos, boolean verbose) {
ensureOpen(); ensureOpen();
Map<String, Segment> segments = new HashMap<>(); Map<String, Segment> segments = new HashMap<>();
// first, go over and compute the search ones... // first, go over and compute the search ones...
@ -960,8 +960,8 @@ public abstract class Engine implements Closeable {
segment = new Segment(info.info.name); segment = new Segment(info.info.name);
segment.search = false; segment.search = false;
segment.committed = true; segment.committed = true;
segment.docCount = info.info.maxDoc(); segment.delDocCount = info.getDelCount() + info.getSoftDelCount();
segment.delDocCount = info.getDelCount(); segment.docCount = info.info.maxDoc() - segment.delDocCount;
segment.version = info.info.getVersion(); segment.version = info.info.getVersion();
segment.compound = info.info.getUseCompoundFile(); segment.compound = info.info.getUseCompoundFile();
try { try {

64
server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java
View File

@ -272,7 +272,7 @@ public class InternalEngineTests extends EngineTestCase {
} }
} }
public void testSegments() throws Exception { public void testSegmentsWithoutSoftDeletes() throws Exception {
Settings settings = Settings.builder() Settings settings = Settings.builder()
.put(defaultSettings.getSettings()) .put(defaultSettings.getSettings())
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build(); .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false).build();
@ -600,6 +600,68 @@ public class InternalEngineTests extends EngineTestCase {
} }
} }
public void testSegmentsWithSoftDeletes() throws Exception {
Settings.Builder settings = Settings.builder()
.put(defaultSettings.getSettings())
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true);
final IndexMetaData indexMetaData = IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build();
final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(indexMetaData);
final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
try (Store store = createStore();
InternalEngine engine = createEngine(config(indexSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null,
null, globalCheckpoint::get))) {
assertThat(engine.segments(false), empty());
int numDocsFirstSegment = randomIntBetween(5, 50);
Set<String> liveDocsFirstSegment = new HashSet<>();
for (int i = 0; i < numDocsFirstSegment; i++) {
String id = Integer.toString(i);
ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null);
engine.index(indexForDoc(doc));
liveDocsFirstSegment.add(id);
}
engine.refresh("test");
List<Segment> segments = engine.segments(randomBoolean());
assertThat(segments, hasSize(1));
assertThat(segments.get(0).getNumDocs(), equalTo(liveDocsFirstSegment.size()));
assertThat(segments.get(0).getDeletedDocs(), equalTo(0));
assertFalse(segments.get(0).committed);
int deletes = 0;
int updates = 0;
int appends = 0;
int iterations = scaledRandomIntBetween(1, 50);
for (int i = 0; i < iterations && liveDocsFirstSegment.isEmpty() == false; i++) {
String idToUpdate = randomFrom(liveDocsFirstSegment);
liveDocsFirstSegment.remove(idToUpdate);
ParsedDocument doc = testParsedDocument(idToUpdate, null, testDocument(), B_1, null);
if (randomBoolean()) {
engine.delete(new Engine.Delete(doc.type(), doc.id(), newUid(doc), primaryTerm.get()));
deletes++;
} else {
engine.index(indexForDoc(doc));
updates++;
}
if (randomBoolean()) {
engine.index(indexForDoc(testParsedDocument(UUIDs.randomBase64UUID(), null, testDocument(), B_1, null)));
appends++;
}
}
boolean committed = randomBoolean();
if (committed) {
engine.flush();
}
engine.refresh("test");
segments = engine.segments(randomBoolean());
assertThat(segments, hasSize(2));
assertThat(segments.get(0).getNumDocs(), equalTo(liveDocsFirstSegment.size()));
assertThat(segments.get(0).getDeletedDocs(), equalTo(updates + deletes));
assertThat(segments.get(0).committed, equalTo(committed));
assertThat(segments.get(1).getNumDocs(), equalTo(updates + appends));
assertThat(segments.get(1).getDeletedDocs(), equalTo(deletes)); // delete tombstones
assertThat(segments.get(1).committed, equalTo(committed));
}
}
public void testCommitStats() throws IOException { public void testCommitStats() throws IOException {
final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);