honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TEST: Corrupt some translog files used in recovery (#27887) 

Currently, method corruptTranslogFiles corrupts some translog files
whose translog_gen are at least the min_required_translog_gen from the
translog checkpoint. However this condition is not enough for
recoverFromTranslog to be always failed.  If we corrupt only translog
operations from only translog files whose translog_gen are smaller than
the min_translog_gen of a recovering index commit, recoverFromTranslog
will be ok as we won't read translog operations from those files.

This commit makes sure corruptTranslogFiles to corrupt some translog 
files that will be used in recoverFromTranslog.

Closes #27538
This commit is contained in:
Nhat Nguyen 2017-12-19 12:01:49 -05:00 committed by GitHub
parent 25b0a7b20f
commit 6b0d90b9d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
core/src/test/java/org/elasticsearch/index/translog/TestTranslog.java
View File

@ -22,6 +22,9 @@ package org.elasticsearch.index.translog;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers; import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Logger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.NIOFSDirectory;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
@ -32,13 +35,13 @@ import java.nio.file.Path;
import java.nio.file.StandardOpenOption; import java.nio.file.StandardOpenOption;
import java.util.Collection; import java.util.Collection;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Random; import java.util.Random;
import java.util.Set; import java.util.Set;
import java.util.TreeSet; import java.util.TreeSet;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import static org.elasticsearch.index.translog.Translog.CHECKPOINT_FILE_NAME;
import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.not;
@ -56,18 +59,16 @@ public class TestTranslog {
*/ */
public static Set<Path> corruptTranslogFiles(Logger logger, Random random, Collection<Path> translogDirs) throws IOException { public static Set<Path> corruptTranslogFiles(Logger logger, Random random, Collection<Path> translogDirs) throws IOException {
Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic
for (Path translogDir : translogDirs) { for (Path translogDir : translogDirs) {
logger.info("--> Translog dir: {}", translogDir);
if (Files.isDirectory(translogDir)) { if (Files.isDirectory(translogDir)) {
final Checkpoint checkpoint = Checkpoint.read(translogDir.resolve(CHECKPOINT_FILE_NAME)); final long minUsedTranslogGen = minTranslogGenUsedInRecovery(translogDir.getParent().resolve("index"));
final long minTranslogGeneration = checkpoint.minTranslogGeneration; logger.info("--> Translog dir [{}], minUsedTranslogGen [{}]", translogDir, minUsedTranslogGen);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) { try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {
for (Path item : stream) { for (Path item : stream) {
if (Files.isRegularFile(item)) { if (Files.isRegularFile(item)) {
// Makes sure that we will corrupt tlog files that are referenced by the Checkpoint. // Makes sure that we will corrupt tlog files that are referenced by the Checkpoint.
final Matcher matcher = TRANSLOG_FILE_PATTERN.matcher(item.getFileName().toString()); final Matcher matcher = TRANSLOG_FILE_PATTERN.matcher(item.getFileName().toString());
if (matcher.matches() && Long.parseLong(matcher.group(1)) >= minTranslogGeneration) { if (matcher.matches() && Long.parseLong(matcher.group(1)) >= minUsedTranslogGen) {
candidates.add(item); candidates.add(item);
} }
} }
@ -83,7 +84,7 @@ public class TestTranslog {
Path fileToCorrupt = RandomPicks.randomFrom(random, candidates); Path fileToCorrupt = RandomPicks.randomFrom(random, candidates);
try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) { try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
// read // read
raf.position(RandomNumbers.randomIntBetween(random, 0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1))); raf.position(RandomNumbers.randomLongBetween(random, 0, raf.size() - 1));
long filePointer = raf.position(); long filePointer = raf.position();
ByteBuffer bb = ByteBuffer.wrap(new byte[1]); ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
raf.read(bb); raf.read(bb);
@ -107,4 +108,16 @@ public class TestTranslog {
assertThat("no translog file corrupted", corruptedFiles, not(empty())); assertThat("no translog file corrupted", corruptedFiles, not(empty()));
return corruptedFiles; return corruptedFiles;
} }
/**
* Lists all existing commits in a given index path, then read the minimum translog generation that will be used in recoverFromTranslog.
*/
private static long minTranslogGenUsedInRecovery(Path indexPath) throws IOException {
try (NIOFSDirectory directory = new NIOFSDirectory(indexPath)) {
final List<IndexCommit> commits = DirectoryReader.listCommits(directory);
// TODO: We should call CombinedDeletionPolicy to get a correct recovering commit.
final IndexCommit recoveringCommit = commits.get(commits.size() - 1);
return Long.parseLong(recoveringCommit.getUserData().get(Translog.TRANSLOG_GENERATION_KEY));
}
}
} }