honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adjust CombinedDeletionPolicy for multiple commits (#27456) 

Today, we keep only the last index commit and use only it to calculate
the minimum required translog generation. This may no longer be correct
as we introduced a new deletion policy which keeps multiple index
commits. This change adjusts the CombinedDeletionPolicy so that it can
work correctly with a new index deletion policy.

Relates to #10708, #27367
This commit is contained in:
Nhat Nguyen 2017-11-23 11:34:50 -05:00 committed by GitHub
parent d1b1d711df
commit e95d18ec23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
core/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java
View File

@ -71,12 +71,18 @@ class CombinedDeletionPolicy extends IndexDeletionPolicy {
} }
private void setLastCommittedTranslogGeneration(List<? extends IndexCommit> commits) throws IOException { private void setLastCommittedTranslogGeneration(List<? extends IndexCommit> commits) throws IOException {
// when opening an existing lucene index, we currently always open the last commit. // We need to keep translog since the smallest translog generation of un-deleted commits.
// we therefore use the translog gen as the one that will be required for recovery // However, there are commits that are not deleted just because they are being snapshotted (rather than being kept by the policy).
final IndexCommit indexCommit = commits.get(commits.size() - 1); // TODO: We need to distinguish those commits and skip them in calculating the minimum required translog generation.
assert indexCommit.isDeleted() == false : "last commit is deleted"; long minRequiredGen = Long.MAX_VALUE;
long minGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)); for (IndexCommit indexCommit : commits) {
translogDeletionPolicy.setMinTranslogGenerationForRecovery(minGen); if (indexCommit.isDeleted() == false) {
long translogGen = Long.parseLong(indexCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
minRequiredGen = Math.min(translogGen, minRequiredGen);
}
}
assert minRequiredGen != Long.MAX_VALUE : "All commits are deleted";
translogDeletionPolicy.setMinTranslogGenerationForRecovery(minRequiredGen);
} }
public SnapshotDeletionPolicy getIndexDeletionPolicy() { public SnapshotDeletionPolicy getIndexDeletionPolicy() {

13
core/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java
View File

@ -60,20 +60,23 @@ public class CombinedDeletionPolicyTests extends ESTestCase {
EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG); EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG);
List<IndexCommit> commitList = new ArrayList<>(); List<IndexCommit> commitList = new ArrayList<>();
long count = randomIntBetween(10, 20); long count = randomIntBetween(10, 20);
long lastGen = 0; long minGen = Long.MAX_VALUE;
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
lastGen += randomIntBetween(10, 20000); long lastGen = randomIntBetween(10, 20000);
minGen = Math.min(minGen, lastGen);
commitList.add(mockIndexCommitWithTranslogGen(lastGen)); commitList.add(mockIndexCommitWithTranslogGen(lastGen));
} }
combinedDeletionPolicy.onInit(commitList); combinedDeletionPolicy.onInit(commitList);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen); verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen);
commitList.clear(); commitList.clear();
minGen = Long.MAX_VALUE;
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
lastGen += randomIntBetween(10, 20000); long lastGen = randomIntBetween(10, 20000);
minGen = Math.min(minGen, lastGen);
commitList.add(mockIndexCommitWithTranslogGen(lastGen)); commitList.add(mockIndexCommitWithTranslogGen(lastGen));
} }
combinedDeletionPolicy.onCommit(commitList); combinedDeletionPolicy.onCommit(commitList);
verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(lastGen); verify(translogDeletionPolicy, times(1)).setMinTranslogGenerationForRecovery(minGen);
} }
IndexCommit mockIndexCommitWithTranslogGen(long gen) throws IOException { IndexCommit mockIndexCommitWithTranslogGen(long gen) throws IOException {