honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[GATEWAY] Cut over MetaDataStateFormat to Path API 

Closes #8609
This commit is contained in:
Simon Willnauer 2014-11-22 21:39:14 +01:00
parent 82868e9cf2
commit 9e7b15b8f3
4 changed files with 160 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java
View File

@ -23,9 +23,11 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
import org.elasticsearch.bootstrap.Elasticsearch;
import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
@ -212,7 +214,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
IndexMetaData currentIndexMetaData;
if (currentMetaData == null) {
// a new event..., check from the state stored
currentIndexMetaData = loadIndexState(indexMetaData.index());
try {
currentIndexMetaData = loadIndexState(indexMetaData.index());
} catch (IOException ex) {
throw new ElasticsearchException("failed to load index state", ex);
}
} else {
currentIndexMetaData = currentMetaData.index(indexMetaData.index());
}
@ -336,7 +342,12 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
if (autoImportDangled.shouldImport() && !danglingIndices.isEmpty()) {
final List<IndexMetaData> dangled = Lists.newArrayList();
for (String indexName : danglingIndices.keySet()) {
IndexMetaData indexMetaData = loadIndexState(indexName);
IndexMetaData indexMetaData;
try {
indexMetaData = loadIndexState(indexName);
} catch (IOException ex) {
throw new ElasticsearchException("failed to load index state", ex);
}
if (indexMetaData == null) {
logger.debug("failed to find state for dangling index [{}]", indexName);
continue;
@ -417,7 +428,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
final boolean deleteOldFiles = previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version();
final MetaDataStateFormat<IndexMetaData> writer = indexStateFormat(format, formatParams, deleteOldFiles);
try {
writer.write(indexMetaData, INDEX_STATE_FILE_PREFIX, indexMetaData.version(), nodeEnv.indexLocations(new Index(indexMetaData.index())));
writer.write(indexMetaData, INDEX_STATE_FILE_PREFIX, indexMetaData.version(), nodeEnv.indexPaths(new Index(indexMetaData.index())));
} catch (Throwable ex) {
logger.warn("[{}]: failed to write index state", ex, indexMetaData.index());
throw new IOException("failed to write state for [" + indexMetaData.index() + "]", ex);
@ -428,7 +439,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
logger.trace("{} writing state, reason [{}]", GLOBAL_STATE_LOG_TYPE, reason);
final MetaDataStateFormat<MetaData> writer = globalStateFormat(format, gatewayModeFormatParams, true);
try {
writer.write(metaData, GLOBAL_STATE_FILE_PREFIX, metaData.version(), nodeEnv.nodeDataLocations());
writer.write(metaData, GLOBAL_STATE_FILE_PREFIX, metaData.version(), nodeEnv.nodeDataPaths());
} catch (Throwable ex) {
logger.warn("{}: failed to write global state", ex, GLOBAL_STATE_LOG_TYPE);
throw new IOException("failed to write global state", ex);
@ -457,12 +468,12 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
}
@Nullable
private IndexMetaData loadIndexState(String index) {
return MetaDataStateFormat.loadLatestState(logger, indexStateFormat(format, formatParams, true), INDEX_STATE_FILE_PATTERN, "[" + index + "]", nodeEnv.indexLocations(new Index(index)));
private IndexMetaData loadIndexState(String index) throws IOException {
return MetaDataStateFormat.loadLatestState(logger, indexStateFormat(format, formatParams, true), INDEX_STATE_FILE_PATTERN, "[" + index + "]", nodeEnv.indexPaths(new Index(index)));
}
private MetaData loadGlobalState() {
return MetaDataStateFormat.loadLatestState(logger, globalStateFormat(format, gatewayModeFormatParams, true), GLOBAL_STATE_FILE_PATTERN, GLOBAL_STATE_LOG_TYPE, nodeEnv.nodeDataLocations());
private MetaData loadGlobalState() throws IOException {
return MetaDataStateFormat.loadLatestState(logger, globalStateFormat(format, gatewayModeFormatParams, true), GLOBAL_STATE_FILE_PATTERN, GLOBAL_STATE_LOG_TYPE, nodeEnv.nodeDataPaths());
}
@ -641,4 +652,5 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
this.future = future;
}
}
}

115
src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java
View File

@ -38,14 +38,8 @@ import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.io.*;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
@ -95,11 +89,11 @@ public abstract class MetaDataStateFormat<T> {
* @param locations the locations where the state should be written to.
* @throws IOException if an IOException occurs
*/
public final void write(final T state, final String prefix, final long version, final File... locations) throws IOException {
public final void write(final T state, final String prefix, final long version, final Path... locations) throws IOException {
Preconditions.checkArgument(locations != null, "Locations must not be null");
Preconditions.checkArgument(locations.length > 0, "One or more locations required");
String fileName = prefix + version + STATE_FILE_EXTENSION;
Path stateLocation = Paths.get(locations[0].getPath(), STATE_DIR_NAME);
Path stateLocation = locations[0].resolve(STATE_DIR_NAME);
Files.createDirectories(stateLocation);
final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp");
final Path finalStatePath = stateLocation.resolve(fileName);
@ -127,7 +121,7 @@ public abstract class MetaDataStateFormat<T> {
Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE);
IOUtils.fsync(stateLocation, true);
for (int i = 1; i < locations.length; i++) {
stateLocation = Paths.get(locations[i].getPath(), STATE_DIR_NAME);
stateLocation = locations[i].resolve(STATE_DIR_NAME);
Files.createDirectories(stateLocation);
Path tmpPath = stateLocation.resolve(fileName + ".tmp");
Path finalPath = stateLocation.resolve(fileName);
@ -167,9 +161,9 @@ public abstract class MetaDataStateFormat<T> {
* Reads the state from a given file and compares the expected version against the actual version of
* the state.
*/
public final T read(File file, long expectedVersion) throws IOException {
try (Directory dir = newDirectory(file.getParentFile())) {
try (final IndexInput indexInput = dir.openInput(file.getName(), IOContext.DEFAULT)) {
public final T read(Path file, long expectedVersion) throws IOException {
try (Directory dir = newDirectory(file.getParent())) {
try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
// We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
CodecUtil.checksumEntireFile(indexInput);
CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
@ -192,25 +186,25 @@ public abstract class MetaDataStateFormat<T> {
}
}
protected Directory newDirectory(File dir) throws IOException {
return new SimpleFSDirectory(dir.toPath());
protected Directory newDirectory(Path dir) throws IOException {
return new SimpleFSDirectory(dir);
}
private void cleanupOldFiles(String prefix, String fileName, File[] locations) throws IOException {
private void cleanupOldFiles(final String prefix, final String currentStateFile, Path[] locations) throws IOException {
final DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
final String entryFileName = entry.getFileName().toString();
return Files.isRegularFile(entry)
&& entryFileName.startsWith(prefix) // only state files
&& currentStateFile.equals(entryFileName) == false; // keep the current state file around
}
};
// now clean up the old files
for (File dataLocation : locations) {
final File[] files = new File(dataLocation, STATE_DIR_NAME).listFiles();
if (files != null) {
for (File file : files) {
if (!file.getName().startsWith(prefix)) {
continue;
}
if (file.getName().equals(fileName)) {
continue;
}
if (Files.exists(file.toPath())) {
Files.delete(file.toPath());
}
for (Path dataLocation : locations) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dataLocation.resolve(STATE_DIR_NAME), filter)) {
for (Path stateFile : stream) {
Files.deleteIfExists(stateFile);
}
}
}
@ -231,29 +225,27 @@ public abstract class MetaDataStateFormat<T> {
* @param dataLocations the data-locations to try.
* @return the latest state or <code>null</code> if no state was found.
*/
public static <T> T loadLatestState(ESLogger logger, MetaDataStateFormat<T> format, Pattern pattern, String stateType, File... dataLocations) {
List<FileAndVersion> files = new ArrayList<>();
public static <T> T loadLatestState(ESLogger logger, MetaDataStateFormat<T> format, Pattern pattern, String stateType, Path... dataLocations) throws IOException {
List<PathAndVersion> files = new ArrayList<>();
long maxVersion = -1;
boolean maxVersionIsLegacy = true;
if (dataLocations != null) { // select all eligable files first
for (File dataLocation : dataLocations) {
File stateDir = new File(dataLocation, MetaDataStateFormat.STATE_DIR_NAME);
if (!stateDir.exists() || !stateDir.isDirectory()) {
for (Path dataLocation : dataLocations) {
final Path stateDir = dataLocation.resolve(STATE_DIR_NAME);
if (!Files.exists(stateDir) || !Files.isDirectory(stateDir)) {
continue;
}
// now, iterate over the current versions, and find latest one
File[] stateFiles = stateDir.listFiles();
if (stateFiles == null) {
continue;
}
for (File stateFile : stateFiles) {
final Matcher matcher = pattern.matcher(stateFile.getName());
if (matcher.matches()) {
final long version = Long.parseLong(matcher.group(1));
maxVersion = Math.max(maxVersion, version);
final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false;
maxVersionIsLegacy &= legacy; // on purpose, see NOTE below
files.add(new FileAndVersion(stateFile, version, legacy));
try (DirectoryStream<Path> paths = Files.newDirectoryStream(stateDir)) { // we don't pass a glob since we need the group part for parsing
for (Path stateFile : paths) {
final Matcher matcher = pattern.matcher(stateFile.getFileName().toString());
if (matcher.matches()) {
final long version = Long.parseLong(matcher.group(1));
maxVersion = Math.max(maxVersion, version);
final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false;
maxVersionIsLegacy &= legacy; // on purpose, see NOTE below
files.add(new PathAndVersion(stateFile, version, legacy));
}
}
}
}
@ -265,22 +257,22 @@ public abstract class MetaDataStateFormat<T> {
// new format (ie. legacy == false) then we know that the latest version state ought to use this new format.
// In case the state file with the latest version does not use the new format while older state files do,
// the list below will be empty and loading the state will fail
for (FileAndVersion fileAndVersion : Collections2.filter(files, new VersionAndLegacyPredicate(maxVersion, maxVersionIsLegacy))) {
for (PathAndVersion pathAndVersion : Collections2.filter(files, new VersionAndLegacyPredicate(maxVersion, maxVersionIsLegacy))) {
try {
final File stateFile = fileAndVersion.file;
final long version = fileAndVersion.version;
final Path stateFile = pathAndVersion.file;
final long version = pathAndVersion.version;
final XContentParser parser;
if (fileAndVersion.legacy) { // read the legacy format -- plain XContent
try (FileInputStream stream = new FileInputStream(stateFile)) {
if (pathAndVersion.legacy) { // read the legacy format -- plain XContent
try (InputStream stream = Files.newInputStream(stateFile)) {
final byte[] data = Streams.copyToByteArray(stream);
if (data.length == 0) {
logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.getAbsolutePath());
logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.toAbsolutePath());
continue;
}
parser = XContentHelper.createParser(data, 0, data.length);
state = format.fromXContent(parser);
if (state == null) {
logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.getAbsolutePath());
logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.toAbsolutePath());
}
}
} else {
@ -289,7 +281,7 @@ public abstract class MetaDataStateFormat<T> {
return state;
} catch (Throwable e) {
exceptions.add(e);
logger.debug("{}: failed to read [{}], ignoring...", e, fileAndVersion.file.getAbsolutePath(), stateType);
logger.debug("{}: failed to read [{}], ignoring...", e, pathAndVersion.file.toAbsolutePath(), stateType);
}
}
// if we reach this something went wrong
@ -302,10 +294,10 @@ public abstract class MetaDataStateFormat<T> {
}
/**
* Filters out all {@link FileAndVersion} instances with a different version than
* Filters out all {@link org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat.PathAndVersion} instances with a different version than
* the given one.
*/
private static final class VersionAndLegacyPredicate implements Predicate<FileAndVersion> {
private static final class VersionAndLegacyPredicate implements Predicate<PathAndVersion> {
private final long version;
private final boolean legacy;
@ -315,7 +307,7 @@ public abstract class MetaDataStateFormat<T> {
}
@Override
public boolean apply(FileAndVersion input) {
public boolean apply(PathAndVersion input) {
return input.version == version && input.legacy == legacy;
}
}
@ -324,12 +316,12 @@ public abstract class MetaDataStateFormat<T> {
* Internal struct-like class that holds the parsed state version, the file
* and a flag if the file is a legacy state ie. pre 1.5
*/
private static class FileAndVersion {
final File file;
private static class PathAndVersion {
final Path file;
final long version;
final boolean legacy;
private FileAndVersion(File file, long version, boolean legacy) {
private PathAndVersion(Path file, long version, boolean legacy) {
this.file = file;
this.version = version;
this.legacy = legacy;
@ -347,5 +339,4 @@ public abstract class MetaDataStateFormat<T> {
}
IOUtils.rm(stateDirectories);
}
}

6
src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java
View File

@ -186,14 +186,14 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste
return shardsState;
}
private ShardStateInfo loadShardStateInfo(ShardId shardId) {
return MetaDataStateFormat.loadLatestState(logger, newShardStateInfoFormat(false), SHARD_STATE_FILE_PATTERN, shardId.toString(), nodeEnv.shardLocations(shardId));
private ShardStateInfo loadShardStateInfo(ShardId shardId) throws IOException {
return MetaDataStateFormat.loadLatestState(logger, newShardStateInfoFormat(false), SHARD_STATE_FILE_PATTERN, shardId.toString(), nodeEnv.shardPaths(shardId));
}
private void writeShardState(String reason, ShardId shardId, ShardStateInfo shardStateInfo, @Nullable ShardStateInfo previousStateInfo) throws Exception {
logger.trace("{} writing shard state, reason [{}]", shardId, reason);
final boolean deleteOldFiles = previousStateInfo != null && previousStateInfo.version != shardStateInfo.version;
newShardStateInfoFormat(deleteOldFiles).write(shardStateInfo, SHARD_STATE_FILE_PREFIX, shardStateInfo.version, nodeEnv.shardLocations(shardId));
newShardStateInfoFormat(deleteOldFiles).write(shardStateInfo, SHARD_STATE_FILE_PREFIX, shardStateInfo.version, nodeEnv.shardPaths(shardId));
}
private MetaDataStateFormat<ShardStateInfo> newShardStateInfoFormat(boolean deleteOldFiles) {

162
src/test/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormatTest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gateway.local.state.meta;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.google.common.collect.Iterators;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.ChecksumIndexInput;
@ -45,12 +46,11 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.net.URISyntaxException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
@ -93,31 +93,31 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
assertThat(resource, notNullValue());
Path dst = tmp.resolve("global-3.st");
Files.copy(resource, dst);
MetaData read = format.read(dst.toFile(), 3);
MetaData read = format.read(dst, 3);
assertThat(read, notNullValue());
assertThat(read.uuid(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg"));
// indices are empty since they are serialized separately
}
public void testReadWriteState() throws IOException {
File[] dirs = new File[randomIntBetween(1, 5)];
Path[] dirs = new Path[randomIntBetween(1, 5)];
for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST);
dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
}
final boolean deleteOldFiles = randomBoolean();
Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles);
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, "foo-", version, dirs);
for (File file : dirs) {
File[] list = file.listFiles();
for (Path file : dirs) {
Path[] list = content(file);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true));
list = stateDir.listFiles();
assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
Path stateDir = list[0];
assertThat(Files.isDirectory(stateDir), is(true));
list = content(stateDir);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo("foo-" + version + ".st"));
assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st"));
DummyState read = format.read(list[0], version);
assertThat(read, equalTo(state));
}
@ -125,24 +125,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
format.write(state2, "foo-", version2, dirs);
for (File file : dirs) {
File[] list = file.listFiles();
for (Path file : dirs) {
Path[] list = content(file);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true));
list = stateDir.listFiles();
assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
Path stateDir = list[0];
assertThat(Files.isDirectory(stateDir), is(true));
list = content(stateDir);
assertEquals(list.length, deleteOldFiles ? 1 : 2);
if (deleteOldFiles) {
assertThat(list[0].getName(), equalTo("foo-" + version2 + ".st"));
assertThat(list[0].getFileName().toString(), equalTo("foo-" + version2 + ".st"));
DummyState read = format.read(list[0], version2);
assertThat(read, equalTo(state2));
} else {
assertThat(list[0].getName(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st")));
assertThat(list[1].getName(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st")));
DummyState read = format.read(new File(stateDir, "foo-" + version2 + ".st"), version2);
assertThat(list[0].getFileName().toString(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st")));
assertThat(list[1].getFileName().toString(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st")));
DummyState read = format.read(stateDir.resolve("foo-" + version2 + ".st"), version2);
assertThat(read, equalTo(state2));
read = format.read(new File(stateDir, "foo-" + version + ".st"), version);
read = format.read(stateDir.resolve("foo-" + version + ".st"), version);
assertThat(read, equalTo(state));
}
@ -151,24 +151,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
@Test
public void testVersionMismatch() throws IOException {
File[] dirs = new File[randomIntBetween(1, 5)];
Path[] dirs = new Path[randomIntBetween(1, 5)];
for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST);
dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
}
final boolean deleteOldFiles = randomBoolean();
Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles);
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, "foo-", version, dirs);
for (File file : dirs) {
File[] list = file.listFiles();
for (Path file : dirs) {
Path[] list = content(file);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true));
list = stateDir.listFiles();
assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
Path stateDir = list[0];
assertThat(Files.isDirectory(stateDir), is(true));
list = content(stateDir);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo("foo-" + version + ".st"));
assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st"));
try {
format.read(list[0], between(version+1, Integer.MAX_VALUE));
fail("corruption expected");
@ -181,24 +181,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
}
public void testCorruption() throws IOException {
File[] dirs = new File[randomIntBetween(1, 5)];
Path[] dirs = new Path[randomIntBetween(1, 5)];
for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST);
dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
}
final boolean deleteOldFiles = randomBoolean();
Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles);
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2);
format.write(state, "foo-", version, dirs);
for (File file : dirs) {
File[] list = file.listFiles();
for (Path file : dirs) {
Path[] list = content(file);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true));
list = stateDir.listFiles();
assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
Path stateDir = list[0];
assertThat(Files.isDirectory(stateDir), is(true));
list = content(stateDir);
assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo("foo-" + version + ".st"));
assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st"));
DummyState read = format.read(list[0], version);
assertThat(read, equalTo(state));
// now corrupt it
@ -212,25 +212,25 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
}
}
public static void corruptFile(File file, ESLogger logger) throws IOException {
File fileToCorrupt = file;
try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParentFile().toPath())) {
public static void corruptFile(Path file, ESLogger logger) throws IOException {
Path fileToCorrupt = file;
try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
long checksumBeforeCorruption;
try (IndexInput input = dir.openInput(fileToCorrupt.getName(), IOContext.DEFAULT)) {
try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
}
try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt, "rw")) {
try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt.toAbsolutePath().toString(), "rw")) {
raf.seek(randomIntBetween(0, (int)Math.min(Integer.MAX_VALUE, raf.length()-1)));
long filePointer = raf.getFilePointer();
byte b = raf.readByte();
raf.seek(filePointer);
raf.writeByte(~b);
raf.getFD().sync();
logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getName(), filePointer, Integer.toHexString(b), Integer.toHexString(~b));
logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(b), Integer.toHexString(~b));
}
long checksumAfterCorruption;
long actualChecksumAfterCorruption;
try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getName(), IOContext.DEFAULT)) {
try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
assertThat(input.getFilePointer(), is(0l));
input.seek(input.length() - 8); // one long is the checksum... 8 bytes
checksumAfterCorruption = input.getChecksum();
@ -240,7 +240,7 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]");
msg.append(" after: [").append(checksumAfterCorruption).append("]");
msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]");
msg.append(" file: ").append(fileToCorrupt.getName()).append(" length: ").append(dir.fileLength(fileToCorrupt.getName()));
msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
logger.debug(msg.toString());
assumeTrue("Checksum collision - " + msg.toString(),
checksumAfterCorruption != checksumBeforeCorruption // collision
@ -252,13 +252,13 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
public void testLatestVersionDoesNotUseLegacy() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean());
final File[] dirs = new File[2];
dirs[0] = newTempDir(LifecycleScope.TEST);
dirs[1] = newTempDir(LifecycleScope.TEST);
for (File dir : dirs) {
Files.createDirectories(new File(dir, MetaDataStateFormat.STATE_DIR_NAME).toPath());
final Path[] dirs = new Path[2];
dirs[0] = newTempDir(LifecycleScope.TEST).toPath();
dirs[1] = newTempDir(LifecycleScope.TEST).toPath();
for (Path dir : dirs) {
Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME));
}
final File dir1 = randomFrom(dirs);
final Path dir1 = randomFrom(dirs);
final int v1 = randomInt(10);
// write a first state file in the new format
format.write(randomMeta(), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, v1, dir1);
@ -266,9 +266,9 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
// write older state files in the old format but with a newer version
final int numLegacyFiles = randomIntBetween(1, 5);
for (int i = 0; i < numLegacyFiles; ++i) {
final File dir2 = randomFrom(dirs);
final Path dir2 = randomFrom(dirs);
final int v2 = v1 + 1 + randomInt(10);
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), new FileOutputStream(new File(new File(dir2, MetaDataStateFormat.STATE_DIR_NAME), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v2)))) {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v2)))) {
xcontentBuilder.startObject();
MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params);
xcontentBuilder.endObject();
@ -287,23 +287,23 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
public void testPrefersNewerFormat() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean());
final File[] dirs = new File[2];
dirs[0] = newTempDir(LifecycleScope.TEST);
dirs[1] = newTempDir(LifecycleScope.TEST);
for (File dir : dirs) {
Files.createDirectories(new File(dir, MetaDataStateFormat.STATE_DIR_NAME).toPath());
final Path[] dirs = new Path[2];
dirs[0] = newTempDir(LifecycleScope.TEST).toPath();
dirs[1] = newTempDir(LifecycleScope.TEST).toPath();
for (Path dir : dirs) {
Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME));
}
final File dir1 = randomFrom(dirs);
final Path dir1 = randomFrom(dirs);
final long v = randomInt(10);
MetaData meta = randomMeta();
String uuid = meta.uuid();
// write a first state file in the old format
final File dir2 = randomFrom(dirs);
final Path dir2 = randomFrom(dirs);
MetaData meta2 = randomMeta();
assertFalse(meta2.uuid().equals(uuid));
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), new FileOutputStream(new File(new File(dir2, MetaDataStateFormat.STATE_DIR_NAME), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v)))) {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v)))) {
xcontentBuilder.startObject();
MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params);
xcontentBuilder.endObject();
@ -319,25 +319,25 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
@Test
public void testLoadState() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
final File[] dirs = new File[randomIntBetween(1, 5)];
final Path[] dirs = new Path[randomIntBetween(1, 5)];
int numStates = randomIntBetween(1, 5);
int numLegacy = randomIntBetween(0, numStates);
List<MetaData> meta = new ArrayList<>();
for (int i = 0; i < numStates; i++) {
meta.add(randomMeta());
}
Set<File> corruptedFiles = new HashSet<>();
Set<Path> corruptedFiles = new HashSet<>();
MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean());
for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST);
Files.createDirectories(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME).toPath());
dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
for (int j = 0; j < numLegacy; j++) {
XContentType type = format.format();
if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {
File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-"+j);
Files.createFile(file.toPath()); // randomly create 0-byte files -- there is extra logic to skip them
Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
} else {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, new FileOutputStream(new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + j)))) {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
xcontentBuilder.startObject();
MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, params);
xcontentBuilder.endObject();
@ -347,16 +347,16 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
for (int j = numLegacy; j < numStates; j++) {
format.write(meta.get(j), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, j, dirs[i]);
if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { // corrupt a file that we do not necessarily need here....
File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + j + ".st");
Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st");
corruptedFiles.add(file);
MetaDataStateFormatTest.corruptFile(file, logger);
}
}
}
List<File> dirList = Arrays.asList(dirs);
List<Path> dirList = Arrays.asList(dirs);
Collections.shuffle(dirList, getRandom());
MetaData loadedMetaData = MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new File[0]));
MetaData loadedMetaData = MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new Path[0]));
MetaData latestMetaData = meta.get(numStates-1);
assertThat(loadedMetaData.uuid(), not(equalTo("_na_")));
assertThat(loadedMetaData.uuid(), equalTo(latestMetaData.uuid()));
@ -373,14 +373,14 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
// now corrupt all the latest ones and make sure we fail to load the state
if (numStates > numLegacy) {
for (int i = 0; i < dirs.length; i++) {
File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + (numStates-1) + ".st");
Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (numStates-1) + ".st");
if (corruptedFiles.contains(file)) {
continue;
}
MetaDataStateFormatTest.corruptFile(file, logger);
}
try {
MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new File[0]));
MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new Path[0]));
fail("latest version can not be read");
} catch (ElasticsearchException ex) {
assertThat(ex.getCause(), instanceOf(CorruptStateException.class));
@ -422,7 +422,7 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
}
@Override
protected Directory newDirectory(File dir) throws IOException {
protected Directory newDirectory(Path dir) throws IOException {
MockDirectoryWrapper mock = new MockDirectoryWrapper(getRandom(), super.newDirectory(dir));
closeAfterSuite(new CloseableDirectory(mock, suiteFailureMarker));
return mock;
@ -559,4 +559,10 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
}
}
}
public Path[] content(Path dir) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
return Iterators.toArray(stream.iterator(), Path.class);
}
}
}