honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Adjust CombinedDeletionPolicy for multiple commits (#27456)" 

The commit looks harmless, unfortunately it can break the engine flush
scheduler and the translog rolling. Both `uncommittedOperations` and
`uncommittedSizeInBytes` are currently calculated based on the minimum
required generation for recovery rather than the translog generation of
the last index commit. This is not correct if other index commits are
reserved for snapshotting even though we are keeping the last index
commit only.

This reverts commit e95d18ec23.
This commit is contained in:
Nhat Nguyen 2017-11-24 15:19:50 -05:00
parent 00867e618d
commit e0e1a92d36
2 changed files with 11 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
View File

@ -71,18 +71,12 @@ class CombinedDeletionPolicy extends IndexDeletionPolicy {
}
private void setLastCommittedTranslogGeneration(List<? extends IndexCommit> commits) throws IOException {
// We need to keep translog since the smallest translog generation of un-deleted commits.
// However, there are commits that are not deleted just because they are being snapshotted (rather than being kept by the policy).
// TODO: We need to distinguish those commits and skip them in calculating the minimum required translog generation.
long minRequiredGen = Long.MAX_VALUE;
for (IndexCommit indexCommit : commits) {
if (indexCommit.isDeleted() == false) {
long translogGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
minRequiredGen = Math.min(translogGen, minRequiredGen);
}
}
assert minRequiredGen != Long.MAX_VALUE : "All commits are deleted";
translogDeletionPolicy.setMinTranslogGenerationForRecovery(minRequiredGen);
// when opening an existing lucene index, we currently always open the last commit.
// we therefore use the translog gen as the one that will be required for recovery
final IndexCommit indexCommit = commits.get(commits.size() - 1);
assert indexCommit.isDeleted() == false : "last commit is deleted";
long minGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
translogDeletionPolicy.setMinTranslogGenerationForRecovery(minGen);
}
public SnapshotDeletionPolicy getIndexDeletionPolicy() {

13
core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
View File

@ -60,23 +60,20 @@ public class CombinedDeletionPolicyTests extends ESTestCase {
EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
List<IndexCommit> commitList = new ArrayList<>();
long count = randomIntBetween(10, 20);
long minGen = Long.MAX_VALUE;
long lastGen = 0;
for (int i = 0; i < count; i++) {
long lastGen = randomIntBetween(10, 20000);
minGen = Math.min(minGen, lastGen);
lastGen += randomIntBetween(10, 20000);
commitList.add(mockIndexCommitWithTranslogGen(lastGen));
}
combinedDeletionPolicy.onInit(commitList);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen);
commitList.clear();
minGen = Long.MAX_VALUE;
for (int i = 0; i < count; i++) {
long lastGen = randomIntBetween(10, 20000);
minGen = Math.min(minGen, lastGen);
lastGen += randomIntBetween(10, 20000);
commitList.add(mockIndexCommitWithTranslogGen(lastGen));
}
combinedDeletionPolicy.onCommit(commitList);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen);
}
IndexCommit mockIndexCommitWithTranslogGen(long gen) throws IOException {