honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[GATEWAY] Cut over MetaDataStateFormat to Path API 

Closes #8609
This commit is contained in:
Simon Willnauer 2014-11-22 21:39:14 +01:00
parent 82868e9cf2
commit 9e7b15b8f3
4 changed files with 160 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java
View File

@ -23,9 +23,11 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps; import com.google.common.collect.Maps;
import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version; import org.elasticsearch.Version;
import org.elasticsearch.bootstrap.Elasticsearch;
import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterChangedEvent;
import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.ClusterStateListener;
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction; import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
@ -212,7 +214,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
IndexMetaData currentIndexMetaData; IndexMetaData currentIndexMetaData;
if (currentMetaData == null) { if (currentMetaData == null) {
// a new event..., check from the state stored // a new event..., check from the state stored
currentIndexMetaData = loadIndexState(indexMetaData.index()); try {
currentIndexMetaData = loadIndexState(indexMetaData.index());
} catch (IOException ex) {
throw new ElasticsearchException("failed to load index state", ex);
}
} else { } else {
currentIndexMetaData = currentMetaData.index(indexMetaData.index()); currentIndexMetaData = currentMetaData.index(indexMetaData.index());
} }
@ -336,7 +342,12 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
if (autoImportDangled.shouldImport() && !danglingIndices.isEmpty()) { if (autoImportDangled.shouldImport() && !danglingIndices.isEmpty()) {
final List<IndexMetaData> dangled = Lists.newArrayList(); final List<IndexMetaData> dangled = Lists.newArrayList();
for (String indexName : danglingIndices.keySet()) { for (String indexName : danglingIndices.keySet()) {
IndexMetaData indexMetaData = loadIndexState(indexName); IndexMetaData indexMetaData;
try {
indexMetaData = loadIndexState(indexName);
} catch (IOException ex) {
throw new ElasticsearchException("failed to load index state", ex);
}
if (indexMetaData == null) { if (indexMetaData == null) {
logger.debug("failed to find state for dangling index [{}]", indexName); logger.debug("failed to find state for dangling index [{}]", indexName);
continue; continue;
@ -417,7 +428,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
final boolean deleteOldFiles = previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version(); final boolean deleteOldFiles = previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version();
final MetaDataStateFormat<IndexMetaData> writer = indexStateFormat(format, formatParams, deleteOldFiles); final MetaDataStateFormat<IndexMetaData> writer = indexStateFormat(format, formatParams, deleteOldFiles);
try { try {
writer.write(indexMetaData, INDEX_STATE_FILE_PREFIX, indexMetaData.version(), nodeEnv.indexLocations(new Index(indexMetaData.index()))); writer.write(indexMetaData, INDEX_STATE_FILE_PREFIX, indexMetaData.version(), nodeEnv.indexPaths(new Index(indexMetaData.index())));
} catch (Throwable ex) { } catch (Throwable ex) {
logger.warn("[{}]: failed to write index state", ex, indexMetaData.index()); logger.warn("[{}]: failed to write index state", ex, indexMetaData.index());
throw new IOException("failed to write state for [" + indexMetaData.index() + "]", ex); throw new IOException("failed to write state for [" + indexMetaData.index() + "]", ex);
@ -428,7 +439,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
logger.trace("{} writing state, reason [{}]", GLOBAL_STATE_LOG_TYPE, reason); logger.trace("{} writing state, reason [{}]", GLOBAL_STATE_LOG_TYPE, reason);
final MetaDataStateFormat<MetaData> writer = globalStateFormat(format, gatewayModeFormatParams, true); final MetaDataStateFormat<MetaData> writer = globalStateFormat(format, gatewayModeFormatParams, true);
try { try {
writer.write(metaData, GLOBAL_STATE_FILE_PREFIX, metaData.version(), nodeEnv.nodeDataLocations()); writer.write(metaData, GLOBAL_STATE_FILE_PREFIX, metaData.version(), nodeEnv.nodeDataPaths());
} catch (Throwable ex) { } catch (Throwable ex) {
logger.warn("{}: failed to write global state", ex, GLOBAL_STATE_LOG_TYPE); logger.warn("{}: failed to write global state", ex, GLOBAL_STATE_LOG_TYPE);
throw new IOException("failed to write global state", ex); throw new IOException("failed to write global state", ex);
@ -457,12 +468,12 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
} }
@Nullable @Nullable
private IndexMetaData loadIndexState(String index) { private IndexMetaData loadIndexState(String index) throws IOException {
return MetaDataStateFormat.loadLatestState(logger, indexStateFormat(format, formatParams, true), INDEX_STATE_FILE_PATTERN, "[" + index + "]", nodeEnv.indexLocations(new Index(index))); return MetaDataStateFormat.loadLatestState(logger, indexStateFormat(format, formatParams, true), INDEX_STATE_FILE_PATTERN, "[" + index + "]", nodeEnv.indexPaths(new Index(index)));
} }
private MetaData loadGlobalState() { private MetaData loadGlobalState() throws IOException {
return MetaDataStateFormat.loadLatestState(logger, globalStateFormat(format, gatewayModeFormatParams, true), GLOBAL_STATE_FILE_PATTERN, GLOBAL_STATE_LOG_TYPE, nodeEnv.nodeDataLocations()); return MetaDataStateFormat.loadLatestState(logger, globalStateFormat(format, gatewayModeFormatParams, true), GLOBAL_STATE_FILE_PATTERN, GLOBAL_STATE_LOG_TYPE, nodeEnv.nodeDataPaths());
} }
@ -641,4 +652,5 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
this.future = future; this.future = future;
} }
} }
} }

115
src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java
View File

@ -38,14 +38,8 @@ import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import java.io.File; import java.io.*;
import java.io.FileInputStream; import java.nio.file.*;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
@ -95,11 +89,11 @@ public abstract class MetaDataStateFormat<T> {
* @param locations the locations where the state should be written to. * @param locations the locations where the state should be written to.
* @throws IOException if an IOException occurs * @throws IOException if an IOException occurs
*/ */
public final void write(final T state, final String prefix, final long version, final File... locations) throws IOException { public final void write(final T state, final String prefix, final long version, final Path... locations) throws IOException {
Preconditions.checkArgument(locations != null, "Locations must not be null"); Preconditions.checkArgument(locations != null, "Locations must not be null");
Preconditions.checkArgument(locations.length > 0, "One or more locations required"); Preconditions.checkArgument(locations.length > 0, "One or more locations required");
String fileName = prefix + version + STATE_FILE_EXTENSION; String fileName = prefix + version + STATE_FILE_EXTENSION;
Path stateLocation = Paths.get(locations[0].getPath(), STATE_DIR_NAME); Path stateLocation = locations[0].resolve(STATE_DIR_NAME);
Files.createDirectories(stateLocation); Files.createDirectories(stateLocation);
final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp"); final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp");
final Path finalStatePath = stateLocation.resolve(fileName); final Path finalStatePath = stateLocation.resolve(fileName);
@ -127,7 +121,7 @@ public abstract class MetaDataStateFormat<T> {
Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE); Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE);
IOUtils.fsync(stateLocation, true); IOUtils.fsync(stateLocation, true);
for (int i = 1; i < locations.length; i++) { for (int i = 1; i < locations.length; i++) {
stateLocation = Paths.get(locations[i].getPath(), STATE_DIR_NAME); stateLocation = locations[i].resolve(STATE_DIR_NAME);
Files.createDirectories(stateLocation); Files.createDirectories(stateLocation);
Path tmpPath = stateLocation.resolve(fileName + ".tmp"); Path tmpPath = stateLocation.resolve(fileName + ".tmp");
Path finalPath = stateLocation.resolve(fileName); Path finalPath = stateLocation.resolve(fileName);
@ -167,9 +161,9 @@ public abstract class MetaDataStateFormat<T> {
* Reads the state from a given file and compares the expected version against the actual version of * Reads the state from a given file and compares the expected version against the actual version of
* the state. * the state.
*/ */
public final T read(File file, long expectedVersion) throws IOException { public final T read(Path file, long expectedVersion) throws IOException {
try (Directory dir = newDirectory(file.getParentFile())) { try (Directory dir = newDirectory(file.getParent())) {
try (final IndexInput indexInput = dir.openInput(file.getName(), IOContext.DEFAULT)) { try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
// We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
CodecUtil.checksumEntireFile(indexInput); CodecUtil.checksumEntireFile(indexInput);
CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION);
@ -192,25 +186,25 @@ public abstract class MetaDataStateFormat<T> {
} }
} }
protected Directory newDirectory(File dir) throws IOException { protected Directory newDirectory(Path dir) throws IOException {
return new SimpleFSDirectory(dir.toPath()); return new SimpleFSDirectory(dir);
} }
private void cleanupOldFiles(String prefix, String fileName, File[] locations) throws IOException { private void cleanupOldFiles(final String prefix, final String currentStateFile, Path[] locations) throws IOException {
final DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
final String entryFileName = entry.getFileName().toString();
return Files.isRegularFile(entry)
&& entryFileName.startsWith(prefix) // only state files
&& currentStateFile.equals(entryFileName) == false; // keep the current state file around
}
};
// now clean up the old files // now clean up the old files
for (File dataLocation : locations) { for (Path dataLocation : locations) {
final File[] files = new File(dataLocation, STATE_DIR_NAME).listFiles(); try (DirectoryStream<Path> stream = Files.newDirectoryStream(dataLocation.resolve(STATE_DIR_NAME), filter)) {
if (files != null) { for (Path stateFile : stream) {
for (File file : files) { Files.deleteIfExists(stateFile);
if (!file.getName().startsWith(prefix)) {
continue;
}
if (file.getName().equals(fileName)) {
continue;
}
if (Files.exists(file.toPath())) {
Files.delete(file.toPath());
}
} }
} }
} }
@ -231,29 +225,27 @@ public abstract class MetaDataStateFormat<T> {
* @param dataLocations the data-locations to try. * @param dataLocations the data-locations to try.
* @return the latest state or <code>null</code> if no state was found. * @return the latest state or <code>null</code> if no state was found.
*/ */
public static <T> T loadLatestState(ESLogger logger, MetaDataStateFormat<T> format, Pattern pattern, String stateType, File... dataLocations) { public static <T> T loadLatestState(ESLogger logger, MetaDataStateFormat<T> format, Pattern pattern, String stateType, Path... dataLocations) throws IOException {
List<FileAndVersion> files = new ArrayList<>(); List<PathAndVersion> files = new ArrayList<>();
long maxVersion = -1; long maxVersion = -1;
boolean maxVersionIsLegacy = true; boolean maxVersionIsLegacy = true;
if (dataLocations != null) { // select all eligable files first if (dataLocations != null) { // select all eligable files first
for (File dataLocation : dataLocations) { for (Path dataLocation : dataLocations) {
File stateDir = new File(dataLocation, MetaDataStateFormat.STATE_DIR_NAME); final Path stateDir = dataLocation.resolve(STATE_DIR_NAME);
if (!stateDir.exists() || !stateDir.isDirectory()) { if (!Files.exists(stateDir) || !Files.isDirectory(stateDir)) {
continue; continue;
} }
// now, iterate over the current versions, and find latest one // now, iterate over the current versions, and find latest one
File[] stateFiles = stateDir.listFiles(); try (DirectoryStream<Path> paths = Files.newDirectoryStream(stateDir)) { // we don't pass a glob since we need the group part for parsing
if (stateFiles == null) { for (Path stateFile : paths) {
continue; final Matcher matcher = pattern.matcher(stateFile.getFileName().toString());
} if (matcher.matches()) {
for (File stateFile : stateFiles) { final long version = Long.parseLong(matcher.group(1));
final Matcher matcher = pattern.matcher(stateFile.getName()); maxVersion = Math.max(maxVersion, version);
if (matcher.matches()) { final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false;
final long version = Long.parseLong(matcher.group(1)); maxVersionIsLegacy &= legacy; // on purpose, see NOTE below
maxVersion = Math.max(maxVersion, version); files.add(new PathAndVersion(stateFile, version, legacy));
final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false; }
maxVersionIsLegacy &= legacy; // on purpose, see NOTE below
files.add(new FileAndVersion(stateFile, version, legacy));
} }
} }
} }
@ -265,22 +257,22 @@ public abstract class MetaDataStateFormat<T> {
// new format (ie. legacy == false) then we know that the latest version state ought to use this new format. // new format (ie. legacy == false) then we know that the latest version state ought to use this new format.
// In case the state file with the latest version does not use the new format while older state files do, // In case the state file with the latest version does not use the new format while older state files do,
// the list below will be empty and loading the state will fail // the list below will be empty and loading the state will fail
for (FileAndVersion fileAndVersion : Collections2.filter(files, new VersionAndLegacyPredicate(maxVersion, maxVersionIsLegacy))) { for (PathAndVersion pathAndVersion : Collections2.filter(files, new VersionAndLegacyPredicate(maxVersion, maxVersionIsLegacy))) {
try { try {
final File stateFile = fileAndVersion.file; final Path stateFile = pathAndVersion.file;
final long version = fileAndVersion.version; final long version = pathAndVersion.version;
final XContentParser parser; final XContentParser parser;
if (fileAndVersion.legacy) { // read the legacy format -- plain XContent if (pathAndVersion.legacy) { // read the legacy format -- plain XContent
try (FileInputStream stream = new FileInputStream(stateFile)) { try (InputStream stream = Files.newInputStream(stateFile)) {
final byte[] data = Streams.copyToByteArray(stream); final byte[] data = Streams.copyToByteArray(stream);
if (data.length == 0) { if (data.length == 0) {
logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.getAbsolutePath()); logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.toAbsolutePath());
continue; continue;
} }
parser = XContentHelper.createParser(data, 0, data.length); parser = XContentHelper.createParser(data, 0, data.length);
state = format.fromXContent(parser); state = format.fromXContent(parser);
if (state == null) { if (state == null) {
logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.getAbsolutePath()); logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.toAbsolutePath());
} }
} }
} else { } else {
@ -289,7 +281,7 @@ public abstract class MetaDataStateFormat<T> {
return state; return state;
} catch (Throwable e) { } catch (Throwable e) {
exceptions.add(e); exceptions.add(e);
logger.debug("{}: failed to read [{}], ignoring...", e, fileAndVersion.file.getAbsolutePath(), stateType); logger.debug("{}: failed to read [{}], ignoring...", e, pathAndVersion.file.toAbsolutePath(), stateType);
} }
} }
// if we reach this something went wrong // if we reach this something went wrong
@ -302,10 +294,10 @@ public abstract class MetaDataStateFormat<T> {
} }
/** /**
* Filters out all {@link FileAndVersion} instances with a different version than * Filters out all {@link org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat.PathAndVersion} instances with a different version than
* the given one. * the given one.
*/ */
private static final class VersionAndLegacyPredicate implements Predicate<FileAndVersion> { private static final class VersionAndLegacyPredicate implements Predicate<PathAndVersion> {
private final long version; private final long version;
private final boolean legacy; private final boolean legacy;
@ -315,7 +307,7 @@ public abstract class MetaDataStateFormat<T> {
} }
@Override @Override
public boolean apply(FileAndVersion input) { public boolean apply(PathAndVersion input) {
return input.version == version && input.legacy == legacy; return input.version == version && input.legacy == legacy;
} }
} }
@ -324,12 +316,12 @@ public abstract class MetaDataStateFormat<T> {
* Internal struct-like class that holds the parsed state version, the file * Internal struct-like class that holds the parsed state version, the file
* and a flag if the file is a legacy state ie. pre 1.5 * and a flag if the file is a legacy state ie. pre 1.5
*/ */
private static class FileAndVersion { private static class PathAndVersion {
final File file; final Path file;
final long version; final long version;
final boolean legacy; final boolean legacy;
private FileAndVersion(File file, long version, boolean legacy) { private PathAndVersion(Path file, long version, boolean legacy) {
this.file = file; this.file = file;
this.version = version; this.version = version;
this.legacy = legacy; this.legacy = legacy;
@ -347,5 +339,4 @@ public abstract class MetaDataStateFormat<T> {
} }
IOUtils.rm(stateDirectories); IOUtils.rm(stateDirectories);
} }
} }

6
src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java
View File

@ -186,14 +186,14 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste
return shardsState; return shardsState;
} }
private ShardStateInfo loadShardStateInfo(ShardId shardId) { private ShardStateInfo loadShardStateInfo(ShardId shardId) throws IOException {
return MetaDataStateFormat.loadLatestState(logger, newShardStateInfoFormat(false), SHARD_STATE_FILE_PATTERN, shardId.toString(), nodeEnv.shardLocations(shardId)); return MetaDataStateFormat.loadLatestState(logger, newShardStateInfoFormat(false), SHARD_STATE_FILE_PATTERN, shardId.toString(), nodeEnv.shardPaths(shardId));
} }
private void writeShardState(String reason, ShardId shardId, ShardStateInfo shardStateInfo, @Nullable ShardStateInfo previousStateInfo) throws Exception { private void writeShardState(String reason, ShardId shardId, ShardStateInfo shardStateInfo, @Nullable ShardStateInfo previousStateInfo) throws Exception {
logger.trace("{} writing shard state, reason [{}]", shardId, reason); logger.trace("{} writing shard state, reason [{}]", shardId, reason);
final boolean deleteOldFiles = previousStateInfo != null && previousStateInfo.version != shardStateInfo.version; final boolean deleteOldFiles = previousStateInfo != null && previousStateInfo.version != shardStateInfo.version;
newShardStateInfoFormat(deleteOldFiles).write(shardStateInfo, SHARD_STATE_FILE_PREFIX, shardStateInfo.version, nodeEnv.shardLocations(shardId)); newShardStateInfoFormat(deleteOldFiles).write(shardStateInfo, SHARD_STATE_FILE_PREFIX, shardStateInfo.version, nodeEnv.shardPaths(shardId));
} }
private MetaDataStateFormat<ShardStateInfo> newShardStateInfoFormat(boolean deleteOldFiles) { private MetaDataStateFormat<ShardStateInfo> newShardStateInfoFormat(boolean deleteOldFiles) {

162
src/test/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormatTest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gateway.local.state.meta; package org.elasticsearch.gateway.local.state.meta;
import com.carrotsearch.randomizedtesting.LifecycleScope; import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.google.common.collect.Iterators;
import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput;
@ -45,12 +46,11 @@ import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import java.io.Closeable; import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream; import java.io.InputStream;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
@ -93,31 +93,31 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
assertThat(resource, notNullValue()); assertThat(resource, notNullValue());
Path dst = tmp.resolve("global-3.st"); Path dst = tmp.resolve("global-3.st");
Files.copy(resource, dst); Files.copy(resource, dst);
MetaData read = format.read(dst.toFile(), 3); MetaData read = format.read(dst, 3);
assertThat(read, notNullValue()); assertThat(read, notNullValue());
assertThat(read.uuid(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg")); assertThat(read.uuid(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg"));
// indices are empty since they are serialized separately // indices are empty since they are serialized separately
} }
public void testReadWriteState() throws IOException { public void testReadWriteState() throws IOException {
File[] dirs = new File[randomIntBetween(1, 5)]; Path[] dirs = new Path[randomIntBetween(1, 5)];
for (int i = 0; i < dirs.length; i++) { for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST); dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
} }
final boolean deleteOldFiles = randomBoolean(); final boolean deleteOldFiles = randomBoolean();
Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles); Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles);
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2); int version = between(0, Integer.MAX_VALUE/2);
format.write(state, "foo-", version, dirs); format.write(state, "foo-", version, dirs);
for (File file : dirs) { for (Path file : dirs) {
File[] list = file.listFiles(); Path[] list = content(file);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0]; Path stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true)); assertThat(Files.isDirectory(stateDir), is(true));
list = stateDir.listFiles(); list = content(stateDir);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo("foo-" + version + ".st")); assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st"));
DummyState read = format.read(list[0], version); DummyState read = format.read(list[0], version);
assertThat(read, equalTo(state)); assertThat(read, equalTo(state));
} }
@ -125,24 +125,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
format.write(state2, "foo-", version2, dirs); format.write(state2, "foo-", version2, dirs);
for (File file : dirs) { for (Path file : dirs) {
File[] list = file.listFiles(); Path[] list = content(file);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0]; Path stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true)); assertThat(Files.isDirectory(stateDir), is(true));
list = stateDir.listFiles(); list = content(stateDir);
assertEquals(list.length, deleteOldFiles ? 1 : 2); assertEquals(list.length, deleteOldFiles ? 1 : 2);
if (deleteOldFiles) { if (deleteOldFiles) {
assertThat(list[0].getName(), equalTo("foo-" + version2 + ".st")); assertThat(list[0].getFileName().toString(), equalTo("foo-" + version2 + ".st"));
DummyState read = format.read(list[0], version2); DummyState read = format.read(list[0], version2);
assertThat(read, equalTo(state2)); assertThat(read, equalTo(state2));
} else { } else {
assertThat(list[0].getName(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st"))); assertThat(list[0].getFileName().toString(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st")));
assertThat(list[1].getName(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st"))); assertThat(list[1].getFileName().toString(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st")));
DummyState read = format.read(new File(stateDir, "foo-" + version2 + ".st"), version2); DummyState read = format.read(stateDir.resolve("foo-" + version2 + ".st"), version2);
assertThat(read, equalTo(state2)); assertThat(read, equalTo(state2));
read = format.read(new File(stateDir, "foo-" + version + ".st"), version); read = format.read(stateDir.resolve("foo-" + version + ".st"), version);
assertThat(read, equalTo(state)); assertThat(read, equalTo(state));
} }
@ -151,24 +151,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
@Test @Test
public void testVersionMismatch() throws IOException { public void testVersionMismatch() throws IOException {
File[] dirs = new File[randomIntBetween(1, 5)]; Path[] dirs = new Path[randomIntBetween(1, 5)];
for (int i = 0; i < dirs.length; i++) { for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST); dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
} }
final boolean deleteOldFiles = randomBoolean(); final boolean deleteOldFiles = randomBoolean();
Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles); Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles);
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2); int version = between(0, Integer.MAX_VALUE/2);
format.write(state, "foo-", version, dirs); format.write(state, "foo-", version, dirs);
for (File file : dirs) { for (Path file : dirs) {
File[] list = file.listFiles(); Path[] list = content(file);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0]; Path stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true)); assertThat(Files.isDirectory(stateDir), is(true));
list = stateDir.listFiles(); list = content(stateDir);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo("foo-" + version + ".st")); assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st"));
try { try {
format.read(list[0], between(version+1, Integer.MAX_VALUE)); format.read(list[0], between(version+1, Integer.MAX_VALUE));
fail("corruption expected"); fail("corruption expected");
@ -181,24 +181,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
} }
public void testCorruption() throws IOException { public void testCorruption() throws IOException {
File[] dirs = new File[randomIntBetween(1, 5)]; Path[] dirs = new Path[randomIntBetween(1, 5)];
for (int i = 0; i < dirs.length; i++) { for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST); dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
} }
final boolean deleteOldFiles = randomBoolean(); final boolean deleteOldFiles = randomBoolean();
Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles); Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles);
DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean());
int version = between(0, Integer.MAX_VALUE/2); int version = between(0, Integer.MAX_VALUE/2);
format.write(state, "foo-", version, dirs); format.write(state, "foo-", version, dirs);
for (File file : dirs) { for (Path file : dirs) {
File[] list = file.listFiles(); Path[] list = content(file);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME));
File stateDir = list[0]; Path stateDir = list[0];
assertThat(stateDir.isDirectory(), is(true)); assertThat(Files.isDirectory(stateDir), is(true));
list = stateDir.listFiles(); list = content(stateDir);
assertEquals(list.length, 1); assertEquals(list.length, 1);
assertThat(list[0].getName(), equalTo("foo-" + version + ".st")); assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st"));
DummyState read = format.read(list[0], version); DummyState read = format.read(list[0], version);
assertThat(read, equalTo(state)); assertThat(read, equalTo(state));
// now corrupt it // now corrupt it
@ -212,25 +212,25 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
} }
} }
public static void corruptFile(File file, ESLogger logger) throws IOException { public static void corruptFile(Path file, ESLogger logger) throws IOException {
File fileToCorrupt = file; Path fileToCorrupt = file;
try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParentFile().toPath())) { try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
long checksumBeforeCorruption; long checksumBeforeCorruption;
try (IndexInput input = dir.openInput(fileToCorrupt.getName(), IOContext.DEFAULT)) { try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
} }
try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt, "rw")) { try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt.toAbsolutePath().toString(), "rw")) {
raf.seek(randomIntBetween(0, (int)Math.min(Integer.MAX_VALUE, raf.length()-1))); raf.seek(randomIntBetween(0, (int)Math.min(Integer.MAX_VALUE, raf.length()-1)));
long filePointer = raf.getFilePointer(); long filePointer = raf.getFilePointer();
byte b = raf.readByte(); byte b = raf.readByte();
raf.seek(filePointer); raf.seek(filePointer);
raf.writeByte(~b); raf.writeByte(~b);
raf.getFD().sync(); raf.getFD().sync();
logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getName(), filePointer, Integer.toHexString(b), Integer.toHexString(~b)); logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(b), Integer.toHexString(~b));
} }
long checksumAfterCorruption; long checksumAfterCorruption;
long actualChecksumAfterCorruption; long actualChecksumAfterCorruption;
try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getName(), IOContext.DEFAULT)) { try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
assertThat(input.getFilePointer(), is(0l)); assertThat(input.getFilePointer(), is(0l));
input.seek(input.length() - 8); // one long is the checksum... 8 bytes input.seek(input.length() - 8); // one long is the checksum... 8 bytes
checksumAfterCorruption = input.getChecksum(); checksumAfterCorruption = input.getChecksum();
@ -240,7 +240,7 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]");
msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]");
msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]");
msg.append(" file: ").append(fileToCorrupt.getName()).append(" length: ").append(dir.fileLength(fileToCorrupt.getName())); msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
logger.debug(msg.toString()); logger.debug(msg.toString());
assumeTrue("Checksum collision - " + msg.toString(), assumeTrue("Checksum collision - " + msg.toString(),
checksumAfterCorruption != checksumBeforeCorruption // collision checksumAfterCorruption != checksumBeforeCorruption // collision
@ -252,13 +252,13 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
public void testLatestVersionDoesNotUseLegacy() throws IOException { public void testLatestVersionDoesNotUseLegacy() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS; final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean()); MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean());
final File[] dirs = new File[2]; final Path[] dirs = new Path[2];
dirs[0] = newTempDir(LifecycleScope.TEST); dirs[0] = newTempDir(LifecycleScope.TEST).toPath();
dirs[1] = newTempDir(LifecycleScope.TEST); dirs[1] = newTempDir(LifecycleScope.TEST).toPath();
for (File dir : dirs) { for (Path dir : dirs) {
Files.createDirectories(new File(dir, MetaDataStateFormat.STATE_DIR_NAME).toPath()); Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME));
} }
final File dir1 = randomFrom(dirs); final Path dir1 = randomFrom(dirs);
final int v1 = randomInt(10); final int v1 = randomInt(10);
// write a first state file in the new format // write a first state file in the new format
format.write(randomMeta(), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, v1, dir1); format.write(randomMeta(), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, v1, dir1);
@ -266,9 +266,9 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
// write older state files in the old format but with a newer version // write older state files in the old format but with a newer version
final int numLegacyFiles = randomIntBetween(1, 5); final int numLegacyFiles = randomIntBetween(1, 5);
for (int i = 0; i < numLegacyFiles; ++i) { for (int i = 0; i < numLegacyFiles; ++i) {
final File dir2 = randomFrom(dirs); final Path dir2 = randomFrom(dirs);
final int v2 = v1 + 1 + randomInt(10); final int v2 = v1 + 1 + randomInt(10);
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), new FileOutputStream(new File(new File(dir2, MetaDataStateFormat.STATE_DIR_NAME), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v2)))) { try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v2)))) {
xcontentBuilder.startObject(); xcontentBuilder.startObject();
MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params); MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params);
xcontentBuilder.endObject(); xcontentBuilder.endObject();
@ -287,23 +287,23 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
public void testPrefersNewerFormat() throws IOException { public void testPrefersNewerFormat() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS; final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean()); MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean());
final File[] dirs = new File[2]; final Path[] dirs = new Path[2];
dirs[0] = newTempDir(LifecycleScope.TEST); dirs[0] = newTempDir(LifecycleScope.TEST).toPath();
dirs[1] = newTempDir(LifecycleScope.TEST); dirs[1] = newTempDir(LifecycleScope.TEST).toPath();
for (File dir : dirs) { for (Path dir : dirs) {
Files.createDirectories(new File(dir, MetaDataStateFormat.STATE_DIR_NAME).toPath()); Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME));
} }
final File dir1 = randomFrom(dirs); final Path dir1 = randomFrom(dirs);
final long v = randomInt(10); final long v = randomInt(10);
MetaData meta = randomMeta(); MetaData meta = randomMeta();
String uuid = meta.uuid(); String uuid = meta.uuid();
// write a first state file in the old format // write a first state file in the old format
final File dir2 = randomFrom(dirs); final Path dir2 = randomFrom(dirs);
MetaData meta2 = randomMeta(); MetaData meta2 = randomMeta();
assertFalse(meta2.uuid().equals(uuid)); assertFalse(meta2.uuid().equals(uuid));
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), new FileOutputStream(new File(new File(dir2, MetaDataStateFormat.STATE_DIR_NAME), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v)))) { try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v)))) {
xcontentBuilder.startObject(); xcontentBuilder.startObject();
MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params); MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params);
xcontentBuilder.endObject(); xcontentBuilder.endObject();
@ -319,25 +319,25 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
@Test @Test
public void testLoadState() throws IOException { public void testLoadState() throws IOException {
final ToXContent.Params params = ToXContent.EMPTY_PARAMS; final ToXContent.Params params = ToXContent.EMPTY_PARAMS;
final File[] dirs = new File[randomIntBetween(1, 5)]; final Path[] dirs = new Path[randomIntBetween(1, 5)];
int numStates = randomIntBetween(1, 5); int numStates = randomIntBetween(1, 5);
int numLegacy = randomIntBetween(0, numStates); int numLegacy = randomIntBetween(0, numStates);
List<MetaData> meta = new ArrayList<>(); List<MetaData> meta = new ArrayList<>();
for (int i = 0; i < numStates; i++) { for (int i = 0; i < numStates; i++) {
meta.add(randomMeta()); meta.add(randomMeta());
} }
Set<File> corruptedFiles = new HashSet<>(); Set<Path> corruptedFiles = new HashSet<>();
MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean()); MetaDataStateFormat<MetaData> format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean());
for (int i = 0; i < dirs.length; i++) { for (int i = 0; i < dirs.length; i++) {
dirs[i] = newTempDir(LifecycleScope.TEST); dirs[i] = newTempDir(LifecycleScope.TEST).toPath();
Files.createDirectories(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME).toPath()); Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME));
for (int j = 0; j < numLegacy; j++) { for (int j = 0; j < numLegacy; j++) {
XContentType type = format.format(); XContentType type = format.format();
if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) {
File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-"+j); Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j);
Files.createFile(file.toPath()); // randomly create 0-byte files -- there is extra logic to skip them Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them
} else { } else {
try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, new FileOutputStream(new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + j)))) { try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) {
xcontentBuilder.startObject(); xcontentBuilder.startObject();
MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, params); MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, params);
xcontentBuilder.endObject(); xcontentBuilder.endObject();
@ -347,16 +347,16 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
for (int j = numLegacy; j < numStates; j++) { for (int j = numLegacy; j < numStates; j++) {
format.write(meta.get(j), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, j, dirs[i]); format.write(meta.get(j), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, j, dirs[i]);
if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { // corrupt a file that we do not necessarily need here.... if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { // corrupt a file that we do not necessarily need here....
File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + j + ".st"); Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st");
corruptedFiles.add(file); corruptedFiles.add(file);
MetaDataStateFormatTest.corruptFile(file, logger); MetaDataStateFormatTest.corruptFile(file, logger);
} }
} }
} }
List<File> dirList = Arrays.asList(dirs); List<Path> dirList = Arrays.asList(dirs);
Collections.shuffle(dirList, getRandom()); Collections.shuffle(dirList, getRandom());
MetaData loadedMetaData = MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new File[0])); MetaData loadedMetaData = MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new Path[0]));
MetaData latestMetaData = meta.get(numStates-1); MetaData latestMetaData = meta.get(numStates-1);
assertThat(loadedMetaData.uuid(), not(equalTo("_na_"))); assertThat(loadedMetaData.uuid(), not(equalTo("_na_")));
assertThat(loadedMetaData.uuid(), equalTo(latestMetaData.uuid())); assertThat(loadedMetaData.uuid(), equalTo(latestMetaData.uuid()));
@ -373,14 +373,14 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
// now corrupt all the latest ones and make sure we fail to load the state // now corrupt all the latest ones and make sure we fail to load the state
if (numStates > numLegacy) { if (numStates > numLegacy) {
for (int i = 0; i < dirs.length; i++) { for (int i = 0; i < dirs.length; i++) {
File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + (numStates-1) + ".st"); Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (numStates-1) + ".st");
if (corruptedFiles.contains(file)) { if (corruptedFiles.contains(file)) {
continue; continue;
} }
MetaDataStateFormatTest.corruptFile(file, logger); MetaDataStateFormatTest.corruptFile(file, logger);
} }
try { try {
MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new File[0])); MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new Path[0]));
fail("latest version can not be read"); fail("latest version can not be read");
} catch (ElasticsearchException ex) { } catch (ElasticsearchException ex) {
assertThat(ex.getCause(), instanceOf(CorruptStateException.class)); assertThat(ex.getCause(), instanceOf(CorruptStateException.class));
@ -422,7 +422,7 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
} }
@Override @Override
protected Directory newDirectory(File dir) throws IOException { protected Directory newDirectory(Path dir) throws IOException {
MockDirectoryWrapper mock = new MockDirectoryWrapper(getRandom(), super.newDirectory(dir)); MockDirectoryWrapper mock = new MockDirectoryWrapper(getRandom(), super.newDirectory(dir));
closeAfterSuite(new CloseableDirectory(mock, suiteFailureMarker)); closeAfterSuite(new CloseableDirectory(mock, suiteFailureMarker));
return mock; return mock;
@ -559,4 +559,10 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase {
} }
} }
} }
public Path[] content(Path dir) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
return Iterators.toArray(stream.iterator(), Path.class);
}
}
} }