honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TEST: makes sure to corrupt referenced tlog files (#27546) 

Method TruncateTranslogIT#corruptTranslogFiles corrupts some random
existing *.tlog files in a translog directory. However, this may not
actually corrupt translog at all if it corrupts only tlog files which
are not referenced by the Checkpoint (eg. their translog generations are
smaller the Checkpoint).

This commit makes sure that we corrupt some tlog files which are
referenced by the Checkpoint.

Closes #27538
This commit is contained in:
Nhat Nguyen 2017-11-27 20:18:58 -05:00 committed by GitHub
parent 0eb87e5d57
commit 000f62c1d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 119 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
core/src/test/java/org/elasticsearch/index/store/CorruptedTranslogIT.java
View File

@ -36,6 +36,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MockEngineFactoryPlugin;
import org.elasticsearch.index.translog.TestTranslog;
import org.elasticsearch.monitor.fs.FsInfo;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
@ -43,23 +44,17 @@ import org.elasticsearch.test.engine.MockEngineSupport;
import org.elasticsearch.test.transport.MockTransportService;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.util.CollectionUtils.iterableAsArrayList;
import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.notNullValue;
/**
* Integration test for corrupted translog files
@ -119,51 +114,14 @@ public class CorruptedTranslogIT extends ESIntegTestCase {
assertTrue(shardRouting.assignedToNode());
String nodeId = shardRouting.currentNodeId();
NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(nodeId).setFs(true).get();
Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
Set<Path> translogDirs = new HashSet<>();
for (FsInfo.Path fsPath : nodeStatses.getNodes().get(0).getFs()) {
String path = fsPath.getPath();
final String relativeDataLocationPath = "indices/"+ test.getUUID() +"/" + Integer.toString(shardRouting.getId()) + "/translog";
Path file = PathUtils.get(path).resolve(relativeDataLocationPath);
if (Files.exists(file)) {
logger.info("--> path: {}", file);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
for (Path item : stream) {
logger.info("--> File: {}", item);
if (Files.isRegularFile(item) && item.getFileName().toString().startsWith("translog-")) {
files.add(item);
}
}
}
}
String relativeDataLocationPath = "indices/" + test.getUUID() + "/" + Integer.toString(shardRouting.getId()) + "/translog";
Path translogDir = PathUtils.get(path).resolve(relativeDataLocationPath);
translogDirs.add(translogDir);
}
Path fileToCorrupt = null;
if (!files.isEmpty()) {
int corruptions = randomIntBetween(5, 20);
for (int i = 0; i < corruptions; i++) {
fileToCorrupt = RandomPicks.randomFrom(random(), files);
try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
// read
raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1)));
long filePointer = raf.position();
ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
raf.read(bb);
bb.flip();
// corrupt
byte oldValue = bb.get(0);
byte newValue = (byte) (oldValue + 1);
bb.put(0, newValue);
// rewrite
raf.position(filePointer);
raf.write(bb);
logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}",
fileToCorrupt, filePointer, Integer.toHexString(oldValue),
Integer.toHexString(newValue), fileToCorrupt);
}
}
}
assertThat("no file corrupted", fileToCorrupt, notNullValue());
TestTranslog.corruptTranslogFiles(logger, random(), translogDirs);
}
/** Disables translog flushing for the specified index */

110
core/src/test/java/org/elasticsearch/index/translog/TestTranslog.java Normal file
View File

@ -0,0 +1,110 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.translog;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Collection;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.elasticsearch.index.translog.Translog.CHECKPOINT_FILE_NAME;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.not;
/**
* Helpers for testing translog.
*/
public class TestTranslog {
static final Pattern TRANSLOG_FILE_PATTERN = Pattern.compile("translog-(\\d+)\\.tlog");
/**
* Corrupts some translog files (translog-N.tlog) from the given translog directories.
*
* @return a collection of tlog files that have been corrupted.
*/
public static Set<Path> corruptTranslogFiles(Logger logger, Random random, Collection<Path> translogDirs) throws IOException {
Set<Path> candidates = new TreeSet<>(); // TreeSet makes sure iteration order is deterministic
for (Path translogDir : translogDirs) {
logger.info("--> Translog dir: {}", translogDir);
if (Files.isDirectory(translogDir)) {
final Checkpoint checkpoint = Checkpoint.read(translogDir.resolve(CHECKPOINT_FILE_NAME));
final long minTranslogGeneration = checkpoint.minTranslogGeneration;
try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {
for (Path item : stream) {
if (Files.isRegularFile(item)) {
// Makes sure that we will corrupt tlog files that are referenced by the Checkpoint.
final Matcher matcher = TRANSLOG_FILE_PATTERN.matcher(item.getFileName().toString());
if (matcher.matches() && Long.parseLong(matcher.group(1)) >= minTranslogGeneration) {
candidates.add(item);
}
}
}
}
}
}
Set<Path> corruptedFiles = new HashSet<>();
if (!candidates.isEmpty()) {
int corruptions = RandomNumbers.randomIntBetween(random, 5, 20);
for (int i = 0; i < corruptions; i++) {
Path fileToCorrupt = RandomPicks.randomFrom(random, candidates);
try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
// read
raf.position(RandomNumbers.randomIntBetween(random, 0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1)));
long filePointer = raf.position();
ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
raf.read(bb);
bb.flip();
// corrupt
byte oldValue = bb.get(0);
byte newValue = (byte) (oldValue + 1);
bb.put(0, newValue);
// rewrite
raf.position(filePointer);
raf.write(bb);
logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}",
fileToCorrupt, filePointer, Integer.toHexString(oldValue),
Integer.toHexString(newValue), fileToCorrupt);
}
corruptedFiles.add(fileToCorrupt);
}
}
assertThat("no translog file corrupted", corruptedFiles, not(empty()));
return corruptedFiles;
}
}

54
core/src/test/java/org/elasticsearch/index/translog/TruncateTranslogIT.java
View File

@ -60,12 +60,8 @@ import org.elasticsearch.test.engine.MockEngineSupport;
import org.elasticsearch.test.transport.MockTransportService;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
@ -79,7 +75,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcke
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.notNullValue;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0)
public class TruncateTranslogIT extends ESIntegTestCase {
@ -273,7 +268,7 @@ public class TruncateTranslogIT extends ESIntegTestCase {
// Corrupt the translog file(s)
logger.info("--> corrupting translog");
corruptTranslogFiles(translogDirs);
TestTranslog.corruptTranslogFiles(logger, random(), translogDirs);
// Restart the single node
logger.info("--> starting node");
@ -354,52 +349,7 @@ public class TruncateTranslogIT extends ESIntegTestCase {
private void corruptRandomTranslogFiles(String indexName) throws IOException {
Set<Path> translogDirs = getTranslogDirs(indexName);
corruptTranslogFiles(translogDirs);
}
private void corruptTranslogFiles(Set<Path> translogDirs) throws IOException {
Set<Path> files = new TreeSet<>(); // treeset makes sure iteration order is deterministic
for (Path translogDir : translogDirs) {
if (Files.isDirectory(translogDir)) {
logger.info("--> path: {}", translogDir);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(translogDir)) {
for (Path item : stream) {
logger.info("--> File: {}", item);
if (Files.isRegularFile(item) && item.getFileName().toString().startsWith("translog-")) {
files.add(item);
}
}
}
}
}
Path fileToCorrupt = null;
if (!files.isEmpty()) {
int corruptions = randomIntBetween(5, 20);
for (int i = 0; i < corruptions; i++) {
fileToCorrupt = RandomPicks.randomFrom(random(), files);
try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
// read
raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1)));
long filePointer = raf.position();
ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
raf.read(bb);
bb.flip();
// corrupt
byte oldValue = bb.get(0);
byte newValue = (byte) (oldValue + 1);
bb.put(0, newValue);
// rewrite
raf.position(filePointer);
raf.write(bb);
logger.info("--> corrupting file {} -- flipping at position {} from {} to {} file: {}",
fileToCorrupt, filePointer, Integer.toHexString(oldValue),
Integer.toHexString(newValue), fileToCorrupt);
}
}
}
assertThat("no file corrupted", fileToCorrupt, notNullValue());
TestTranslog.corruptTranslogFiles(logger, random(), translogDirs);
}
/** Disables translog flushing for the specified index */