From 9e7b15b8f3b8b1f2405b233a042d842d80f64028 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Sat, 22 Nov 2014 21:39:14 +0100 Subject: [PATCH] [GATEWAY] Cut over MetaDataStateFormat to Path API Closes #8609 --- .../state/meta/LocalGatewayMetaState.java | 28 ++- .../local/state/meta/MetaDataStateFormat.java | 115 ++++++------- .../state/shards/LocalGatewayShardsState.java | 6 +- .../state/meta/MetaDataStateFormatTest.java | 162 +++++++++--------- 4 files changed, 160 insertions(+), 151 deletions(-) diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java index 803f57c3afc..fa1d4b2b08a 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -23,9 +23,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.IOUtils; +import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.Version; +import org.elasticsearch.bootstrap.Elasticsearch; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction; @@ -212,7 +214,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS IndexMetaData currentIndexMetaData; if (currentMetaData == null) { // a new event..., check from the state stored - currentIndexMetaData = loadIndexState(indexMetaData.index()); + try { + currentIndexMetaData = loadIndexState(indexMetaData.index()); + } catch (IOException ex) { + throw new ElasticsearchException("failed to load index state", ex); + } } else { currentIndexMetaData = currentMetaData.index(indexMetaData.index()); } @@ -336,7 +342,12 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS if (autoImportDangled.shouldImport() && !danglingIndices.isEmpty()) { final List dangled = Lists.newArrayList(); for (String indexName : danglingIndices.keySet()) { - IndexMetaData indexMetaData = loadIndexState(indexName); + IndexMetaData indexMetaData; + try { + indexMetaData = loadIndexState(indexName); + } catch (IOException ex) { + throw new ElasticsearchException("failed to load index state", ex); + } if (indexMetaData == null) { logger.debug("failed to find state for dangling index [{}]", indexName); continue; @@ -417,7 +428,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS final boolean deleteOldFiles = previousIndexMetaData != null && previousIndexMetaData.version() != indexMetaData.version(); final MetaDataStateFormat writer = indexStateFormat(format, formatParams, deleteOldFiles); try { - writer.write(indexMetaData, INDEX_STATE_FILE_PREFIX, indexMetaData.version(), nodeEnv.indexLocations(new Index(indexMetaData.index()))); + writer.write(indexMetaData, INDEX_STATE_FILE_PREFIX, indexMetaData.version(), nodeEnv.indexPaths(new Index(indexMetaData.index()))); } catch (Throwable ex) { logger.warn("[{}]: failed to write index state", ex, indexMetaData.index()); throw new IOException("failed to write state for [" + indexMetaData.index() + "]", ex); @@ -428,7 +439,7 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS logger.trace("{} writing state, reason [{}]", GLOBAL_STATE_LOG_TYPE, reason); final MetaDataStateFormat writer = globalStateFormat(format, gatewayModeFormatParams, true); try { - writer.write(metaData, GLOBAL_STATE_FILE_PREFIX, metaData.version(), nodeEnv.nodeDataLocations()); + writer.write(metaData, GLOBAL_STATE_FILE_PREFIX, metaData.version(), nodeEnv.nodeDataPaths()); } catch (Throwable ex) { logger.warn("{}: failed to write global state", ex, GLOBAL_STATE_LOG_TYPE); throw new IOException("failed to write global state", ex); @@ -457,12 +468,12 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS } @Nullable - private IndexMetaData loadIndexState(String index) { - return MetaDataStateFormat.loadLatestState(logger, indexStateFormat(format, formatParams, true), INDEX_STATE_FILE_PATTERN, "[" + index + "]", nodeEnv.indexLocations(new Index(index))); + private IndexMetaData loadIndexState(String index) throws IOException { + return MetaDataStateFormat.loadLatestState(logger, indexStateFormat(format, formatParams, true), INDEX_STATE_FILE_PATTERN, "[" + index + "]", nodeEnv.indexPaths(new Index(index))); } - private MetaData loadGlobalState() { - return MetaDataStateFormat.loadLatestState(logger, globalStateFormat(format, gatewayModeFormatParams, true), GLOBAL_STATE_FILE_PATTERN, GLOBAL_STATE_LOG_TYPE, nodeEnv.nodeDataLocations()); + private MetaData loadGlobalState() throws IOException { + return MetaDataStateFormat.loadLatestState(logger, globalStateFormat(format, gatewayModeFormatParams, true), GLOBAL_STATE_FILE_PATTERN, GLOBAL_STATE_LOG_TYPE, nodeEnv.nodeDataPaths()); } @@ -641,4 +652,5 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS this.future = future; } } + } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java index b5792a67613..8de74a3f6e8 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java @@ -38,14 +38,8 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; +import java.io.*; +import java.nio.file.*; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; @@ -95,11 +89,11 @@ public abstract class MetaDataStateFormat { * @param locations the locations where the state should be written to. * @throws IOException if an IOException occurs */ - public final void write(final T state, final String prefix, final long version, final File... locations) throws IOException { + public final void write(final T state, final String prefix, final long version, final Path... locations) throws IOException { Preconditions.checkArgument(locations != null, "Locations must not be null"); Preconditions.checkArgument(locations.length > 0, "One or more locations required"); String fileName = prefix + version + STATE_FILE_EXTENSION; - Path stateLocation = Paths.get(locations[0].getPath(), STATE_DIR_NAME); + Path stateLocation = locations[0].resolve(STATE_DIR_NAME); Files.createDirectories(stateLocation); final Path tmpStatePath = stateLocation.resolve(fileName + ".tmp"); final Path finalStatePath = stateLocation.resolve(fileName); @@ -127,7 +121,7 @@ public abstract class MetaDataStateFormat { Files.move(tmpStatePath, finalStatePath, StandardCopyOption.ATOMIC_MOVE); IOUtils.fsync(stateLocation, true); for (int i = 1; i < locations.length; i++) { - stateLocation = Paths.get(locations[i].getPath(), STATE_DIR_NAME); + stateLocation = locations[i].resolve(STATE_DIR_NAME); Files.createDirectories(stateLocation); Path tmpPath = stateLocation.resolve(fileName + ".tmp"); Path finalPath = stateLocation.resolve(fileName); @@ -167,9 +161,9 @@ public abstract class MetaDataStateFormat { * Reads the state from a given file and compares the expected version against the actual version of * the state. */ - public final T read(File file, long expectedVersion) throws IOException { - try (Directory dir = newDirectory(file.getParentFile())) { - try (final IndexInput indexInput = dir.openInput(file.getName(), IOContext.DEFAULT)) { + public final T read(Path file, long expectedVersion) throws IOException { + try (Directory dir = newDirectory(file.getParent())) { + try (final IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) { // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here. CodecUtil.checksumEntireFile(indexInput); CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, STATE_FILE_VERSION, STATE_FILE_VERSION); @@ -192,25 +186,25 @@ public abstract class MetaDataStateFormat { } } - protected Directory newDirectory(File dir) throws IOException { - return new SimpleFSDirectory(dir.toPath()); + protected Directory newDirectory(Path dir) throws IOException { + return new SimpleFSDirectory(dir); } - private void cleanupOldFiles(String prefix, String fileName, File[] locations) throws IOException { + private void cleanupOldFiles(final String prefix, final String currentStateFile, Path[] locations) throws IOException { + final DirectoryStream.Filter filter = new DirectoryStream.Filter() { + @Override + public boolean accept(Path entry) throws IOException { + final String entryFileName = entry.getFileName().toString(); + return Files.isRegularFile(entry) + && entryFileName.startsWith(prefix) // only state files + && currentStateFile.equals(entryFileName) == false; // keep the current state file around + } + }; // now clean up the old files - for (File dataLocation : locations) { - final File[] files = new File(dataLocation, STATE_DIR_NAME).listFiles(); - if (files != null) { - for (File file : files) { - if (!file.getName().startsWith(prefix)) { - continue; - } - if (file.getName().equals(fileName)) { - continue; - } - if (Files.exists(file.toPath())) { - Files.delete(file.toPath()); - } + for (Path dataLocation : locations) { + try (DirectoryStream stream = Files.newDirectoryStream(dataLocation.resolve(STATE_DIR_NAME), filter)) { + for (Path stateFile : stream) { + Files.deleteIfExists(stateFile); } } } @@ -231,29 +225,27 @@ public abstract class MetaDataStateFormat { * @param dataLocations the data-locations to try. * @return the latest state or null if no state was found. */ - public static T loadLatestState(ESLogger logger, MetaDataStateFormat format, Pattern pattern, String stateType, File... dataLocations) { - List files = new ArrayList<>(); + public static T loadLatestState(ESLogger logger, MetaDataStateFormat format, Pattern pattern, String stateType, Path... dataLocations) throws IOException { + List files = new ArrayList<>(); long maxVersion = -1; boolean maxVersionIsLegacy = true; if (dataLocations != null) { // select all eligable files first - for (File dataLocation : dataLocations) { - File stateDir = new File(dataLocation, MetaDataStateFormat.STATE_DIR_NAME); - if (!stateDir.exists() || !stateDir.isDirectory()) { + for (Path dataLocation : dataLocations) { + final Path stateDir = dataLocation.resolve(STATE_DIR_NAME); + if (!Files.exists(stateDir) || !Files.isDirectory(stateDir)) { continue; } // now, iterate over the current versions, and find latest one - File[] stateFiles = stateDir.listFiles(); - if (stateFiles == null) { - continue; - } - for (File stateFile : stateFiles) { - final Matcher matcher = pattern.matcher(stateFile.getName()); - if (matcher.matches()) { - final long version = Long.parseLong(matcher.group(1)); - maxVersion = Math.max(maxVersion, version); - final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false; - maxVersionIsLegacy &= legacy; // on purpose, see NOTE below - files.add(new FileAndVersion(stateFile, version, legacy)); + try (DirectoryStream paths = Files.newDirectoryStream(stateDir)) { // we don't pass a glob since we need the group part for parsing + for (Path stateFile : paths) { + final Matcher matcher = pattern.matcher(stateFile.getFileName().toString()); + if (matcher.matches()) { + final long version = Long.parseLong(matcher.group(1)); + maxVersion = Math.max(maxVersion, version); + final boolean legacy = MetaDataStateFormat.STATE_FILE_EXTENSION.equals(matcher.group(2)) == false; + maxVersionIsLegacy &= legacy; // on purpose, see NOTE below + files.add(new PathAndVersion(stateFile, version, legacy)); + } } } } @@ -265,22 +257,22 @@ public abstract class MetaDataStateFormat { // new format (ie. legacy == false) then we know that the latest version state ought to use this new format. // In case the state file with the latest version does not use the new format while older state files do, // the list below will be empty and loading the state will fail - for (FileAndVersion fileAndVersion : Collections2.filter(files, new VersionAndLegacyPredicate(maxVersion, maxVersionIsLegacy))) { + for (PathAndVersion pathAndVersion : Collections2.filter(files, new VersionAndLegacyPredicate(maxVersion, maxVersionIsLegacy))) { try { - final File stateFile = fileAndVersion.file; - final long version = fileAndVersion.version; + final Path stateFile = pathAndVersion.file; + final long version = pathAndVersion.version; final XContentParser parser; - if (fileAndVersion.legacy) { // read the legacy format -- plain XContent - try (FileInputStream stream = new FileInputStream(stateFile)) { + if (pathAndVersion.legacy) { // read the legacy format -- plain XContent + try (InputStream stream = Files.newInputStream(stateFile)) { final byte[] data = Streams.copyToByteArray(stream); if (data.length == 0) { - logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.getAbsolutePath()); + logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.toAbsolutePath()); continue; } parser = XContentHelper.createParser(data, 0, data.length); state = format.fromXContent(parser); if (state == null) { - logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.getAbsolutePath()); + logger.debug("{}: no data for [{}], ignoring...", stateType, stateFile.toAbsolutePath()); } } } else { @@ -289,7 +281,7 @@ public abstract class MetaDataStateFormat { return state; } catch (Throwable e) { exceptions.add(e); - logger.debug("{}: failed to read [{}], ignoring...", e, fileAndVersion.file.getAbsolutePath(), stateType); + logger.debug("{}: failed to read [{}], ignoring...", e, pathAndVersion.file.toAbsolutePath(), stateType); } } // if we reach this something went wrong @@ -302,10 +294,10 @@ public abstract class MetaDataStateFormat { } /** - * Filters out all {@link FileAndVersion} instances with a different version than + * Filters out all {@link org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat.PathAndVersion} instances with a different version than * the given one. */ - private static final class VersionAndLegacyPredicate implements Predicate { + private static final class VersionAndLegacyPredicate implements Predicate { private final long version; private final boolean legacy; @@ -315,7 +307,7 @@ public abstract class MetaDataStateFormat { } @Override - public boolean apply(FileAndVersion input) { + public boolean apply(PathAndVersion input) { return input.version == version && input.legacy == legacy; } } @@ -324,12 +316,12 @@ public abstract class MetaDataStateFormat { * Internal struct-like class that holds the parsed state version, the file * and a flag if the file is a legacy state ie. pre 1.5 */ - private static class FileAndVersion { - final File file; + private static class PathAndVersion { + final Path file; final long version; final boolean legacy; - private FileAndVersion(File file, long version, boolean legacy) { + private PathAndVersion(Path file, long version, boolean legacy) { this.file = file; this.version = version; this.legacy = legacy; @@ -347,5 +339,4 @@ public abstract class MetaDataStateFormat { } IOUtils.rm(stateDirectories); } - } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java index 32640d73491..75127dbf8a4 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -186,14 +186,14 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste return shardsState; } - private ShardStateInfo loadShardStateInfo(ShardId shardId) { - return MetaDataStateFormat.loadLatestState(logger, newShardStateInfoFormat(false), SHARD_STATE_FILE_PATTERN, shardId.toString(), nodeEnv.shardLocations(shardId)); + private ShardStateInfo loadShardStateInfo(ShardId shardId) throws IOException { + return MetaDataStateFormat.loadLatestState(logger, newShardStateInfoFormat(false), SHARD_STATE_FILE_PATTERN, shardId.toString(), nodeEnv.shardPaths(shardId)); } private void writeShardState(String reason, ShardId shardId, ShardStateInfo shardStateInfo, @Nullable ShardStateInfo previousStateInfo) throws Exception { logger.trace("{} writing shard state, reason [{}]", shardId, reason); final boolean deleteOldFiles = previousStateInfo != null && previousStateInfo.version != shardStateInfo.version; - newShardStateInfoFormat(deleteOldFiles).write(shardStateInfo, SHARD_STATE_FILE_PREFIX, shardStateInfo.version, nodeEnv.shardLocations(shardId)); + newShardStateInfoFormat(deleteOldFiles).write(shardStateInfo, SHARD_STATE_FILE_PREFIX, shardStateInfo.version, nodeEnv.shardPaths(shardId)); } private MetaDataStateFormat newShardStateInfoFormat(boolean deleteOldFiles) { diff --git a/src/test/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormatTest.java b/src/test/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormatTest.java index bf58726e178..9a9c2df8168 100644 --- a/src/test/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormatTest.java +++ b/src/test/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormatTest.java @@ -19,6 +19,7 @@ package org.elasticsearch.gateway.local.state.meta; import com.carrotsearch.randomizedtesting.LifecycleScope; +import com.google.common.collect.Iterators; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.ChecksumIndexInput; @@ -45,12 +46,11 @@ import org.junit.Assert; import org.junit.Test; import java.io.Closeable; -import java.io.File; -import java.io.FileOutputStream; import java.io.InputStream; import java.io.IOException; import java.io.RandomAccessFile; import java.net.URISyntaxException; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; @@ -93,31 +93,31 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { assertThat(resource, notNullValue()); Path dst = tmp.resolve("global-3.st"); Files.copy(resource, dst); - MetaData read = format.read(dst.toFile(), 3); + MetaData read = format.read(dst, 3); assertThat(read, notNullValue()); assertThat(read.uuid(), equalTo("3O1tDF1IRB6fSJ-GrTMUtg")); // indices are empty since they are serialized separately } public void testReadWriteState() throws IOException { - File[] dirs = new File[randomIntBetween(1, 5)]; + Path[] dirs = new Path[randomIntBetween(1, 5)]; for (int i = 0; i < dirs.length; i++) { - dirs[i] = newTempDir(LifecycleScope.TEST); + dirs[i] = newTempDir(LifecycleScope.TEST).toPath(); } final boolean deleteOldFiles = randomBoolean(); Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); format.write(state, "foo-", version, dirs); - for (File file : dirs) { - File[] list = file.listFiles(); + for (Path file : dirs) { + Path[] list = content(file); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); - File stateDir = list[0]; - assertThat(stateDir.isDirectory(), is(true)); - list = stateDir.listFiles(); + assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); + Path stateDir = list[0]; + assertThat(Files.isDirectory(stateDir), is(true)); + list = content(stateDir); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo("foo-" + version + ".st")); + assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st")); DummyState read = format.read(list[0], version); assertThat(read, equalTo(state)); } @@ -125,24 +125,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { DummyState state2 = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); format.write(state2, "foo-", version2, dirs); - for (File file : dirs) { - File[] list = file.listFiles(); + for (Path file : dirs) { + Path[] list = content(file); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); - File stateDir = list[0]; - assertThat(stateDir.isDirectory(), is(true)); - list = stateDir.listFiles(); + assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); + Path stateDir = list[0]; + assertThat(Files.isDirectory(stateDir), is(true)); + list = content(stateDir); assertEquals(list.length, deleteOldFiles ? 1 : 2); if (deleteOldFiles) { - assertThat(list[0].getName(), equalTo("foo-" + version2 + ".st")); + assertThat(list[0].getFileName().toString(), equalTo("foo-" + version2 + ".st")); DummyState read = format.read(list[0], version2); assertThat(read, equalTo(state2)); } else { - assertThat(list[0].getName(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st"))); - assertThat(list[1].getName(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st"))); - DummyState read = format.read(new File(stateDir, "foo-" + version2 + ".st"), version2); + assertThat(list[0].getFileName().toString(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st"))); + assertThat(list[1].getFileName().toString(), anyOf(equalTo("foo-" + version + ".st"), equalTo("foo-" + version2 + ".st"))); + DummyState read = format.read(stateDir.resolve("foo-" + version2 + ".st"), version2); assertThat(read, equalTo(state2)); - read = format.read(new File(stateDir, "foo-" + version + ".st"), version); + read = format.read(stateDir.resolve("foo-" + version + ".st"), version); assertThat(read, equalTo(state)); } @@ -151,24 +151,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { @Test public void testVersionMismatch() throws IOException { - File[] dirs = new File[randomIntBetween(1, 5)]; + Path[] dirs = new Path[randomIntBetween(1, 5)]; for (int i = 0; i < dirs.length; i++) { - dirs[i] = newTempDir(LifecycleScope.TEST); + dirs[i] = newTempDir(LifecycleScope.TEST).toPath(); } final boolean deleteOldFiles = randomBoolean(); Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); format.write(state, "foo-", version, dirs); - for (File file : dirs) { - File[] list = file.listFiles(); + for (Path file : dirs) { + Path[] list = content(file); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); - File stateDir = list[0]; - assertThat(stateDir.isDirectory(), is(true)); - list = stateDir.listFiles(); + assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); + Path stateDir = list[0]; + assertThat(Files.isDirectory(stateDir), is(true)); + list = content(stateDir); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo("foo-" + version + ".st")); + assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st")); try { format.read(list[0], between(version+1, Integer.MAX_VALUE)); fail("corruption expected"); @@ -181,24 +181,24 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { } public void testCorruption() throws IOException { - File[] dirs = new File[randomIntBetween(1, 5)]; + Path[] dirs = new Path[randomIntBetween(1, 5)]; for (int i = 0; i < dirs.length; i++) { - dirs[i] = newTempDir(LifecycleScope.TEST); + dirs[i] = newTempDir(LifecycleScope.TEST).toPath(); } final boolean deleteOldFiles = randomBoolean(); Format format = new Format(randomFrom(XContentType.values()), deleteOldFiles); DummyState state = new DummyState(randomRealisticUnicodeOfCodepointLengthBetween(1, 1000), randomInt(), randomLong(), randomDouble(), randomBoolean()); int version = between(0, Integer.MAX_VALUE/2); format.write(state, "foo-", version, dirs); - for (File file : dirs) { - File[] list = file.listFiles(); + for (Path file : dirs) { + Path[] list = content(file); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); - File stateDir = list[0]; - assertThat(stateDir.isDirectory(), is(true)); - list = stateDir.listFiles(); + assertThat(list[0].getFileName().toString(), equalTo(MetaDataStateFormat.STATE_DIR_NAME)); + Path stateDir = list[0]; + assertThat(Files.isDirectory(stateDir), is(true)); + list = content(stateDir); assertEquals(list.length, 1); - assertThat(list[0].getName(), equalTo("foo-" + version + ".st")); + assertThat(list[0].getFileName().toString(), equalTo("foo-" + version + ".st")); DummyState read = format.read(list[0], version); assertThat(read, equalTo(state)); // now corrupt it @@ -212,25 +212,25 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { } } - public static void corruptFile(File file, ESLogger logger) throws IOException { - File fileToCorrupt = file; - try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParentFile().toPath())) { + public static void corruptFile(Path file, ESLogger logger) throws IOException { + Path fileToCorrupt = file; + try (final SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) { long checksumBeforeCorruption; - try (IndexInput input = dir.openInput(fileToCorrupt.getName(), IOContext.DEFAULT)) { + try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { checksumBeforeCorruption = CodecUtil.retrieveChecksum(input); } - try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt, "rw")) { + try (RandomAccessFile raf = new RandomAccessFile(fileToCorrupt.toAbsolutePath().toString(), "rw")) { raf.seek(randomIntBetween(0, (int)Math.min(Integer.MAX_VALUE, raf.length()-1))); long filePointer = raf.getFilePointer(); byte b = raf.readByte(); raf.seek(filePointer); raf.writeByte(~b); raf.getFD().sync(); - logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getName(), filePointer, Integer.toHexString(b), Integer.toHexString(~b)); + logger.debug("Corrupting file {} -- flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(b), Integer.toHexString(~b)); } long checksumAfterCorruption; long actualChecksumAfterCorruption; - try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getName(), IOContext.DEFAULT)) { + try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) { assertThat(input.getFilePointer(), is(0l)); input.seek(input.length() - 8); // one long is the checksum... 8 bytes checksumAfterCorruption = input.getChecksum(); @@ -240,7 +240,7 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]"); msg.append(" after: [").append(checksumAfterCorruption).append("]"); msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]"); - msg.append(" file: ").append(fileToCorrupt.getName()).append(" length: ").append(dir.fileLength(fileToCorrupt.getName())); + msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString())); logger.debug(msg.toString()); assumeTrue("Checksum collision - " + msg.toString(), checksumAfterCorruption != checksumBeforeCorruption // collision @@ -252,13 +252,13 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { public void testLatestVersionDoesNotUseLegacy() throws IOException { final ToXContent.Params params = ToXContent.EMPTY_PARAMS; MetaDataStateFormat format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean()); - final File[] dirs = new File[2]; - dirs[0] = newTempDir(LifecycleScope.TEST); - dirs[1] = newTempDir(LifecycleScope.TEST); - for (File dir : dirs) { - Files.createDirectories(new File(dir, MetaDataStateFormat.STATE_DIR_NAME).toPath()); + final Path[] dirs = new Path[2]; + dirs[0] = newTempDir(LifecycleScope.TEST).toPath(); + dirs[1] = newTempDir(LifecycleScope.TEST).toPath(); + for (Path dir : dirs) { + Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME)); } - final File dir1 = randomFrom(dirs); + final Path dir1 = randomFrom(dirs); final int v1 = randomInt(10); // write a first state file in the new format format.write(randomMeta(), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, v1, dir1); @@ -266,9 +266,9 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { // write older state files in the old format but with a newer version final int numLegacyFiles = randomIntBetween(1, 5); for (int i = 0; i < numLegacyFiles; ++i) { - final File dir2 = randomFrom(dirs); + final Path dir2 = randomFrom(dirs); final int v2 = v1 + 1 + randomInt(10); - try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), new FileOutputStream(new File(new File(dir2, MetaDataStateFormat.STATE_DIR_NAME), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v2)))) { + try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v2)))) { xcontentBuilder.startObject(); MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params); xcontentBuilder.endObject(); @@ -287,23 +287,23 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { public void testPrefersNewerFormat() throws IOException { final ToXContent.Params params = ToXContent.EMPTY_PARAMS; MetaDataStateFormat format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean()); - final File[] dirs = new File[2]; - dirs[0] = newTempDir(LifecycleScope.TEST); - dirs[1] = newTempDir(LifecycleScope.TEST); - for (File dir : dirs) { - Files.createDirectories(new File(dir, MetaDataStateFormat.STATE_DIR_NAME).toPath()); + final Path[] dirs = new Path[2]; + dirs[0] = newTempDir(LifecycleScope.TEST).toPath(); + dirs[1] = newTempDir(LifecycleScope.TEST).toPath(); + for (Path dir : dirs) { + Files.createDirectories(dir.resolve(MetaDataStateFormat.STATE_DIR_NAME)); } - final File dir1 = randomFrom(dirs); + final Path dir1 = randomFrom(dirs); final long v = randomInt(10); MetaData meta = randomMeta(); String uuid = meta.uuid(); // write a first state file in the old format - final File dir2 = randomFrom(dirs); + final Path dir2 = randomFrom(dirs); MetaData meta2 = randomMeta(); assertFalse(meta2.uuid().equals(uuid)); - try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), new FileOutputStream(new File(new File(dir2, MetaDataStateFormat.STATE_DIR_NAME), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v)))) { + try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(format.format(), Files.newOutputStream(dir2.resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve(LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX + v)))) { xcontentBuilder.startObject(); MetaData.Builder.toXContent(randomMeta(), xcontentBuilder, params); xcontentBuilder.endObject(); @@ -319,25 +319,25 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { @Test public void testLoadState() throws IOException { final ToXContent.Params params = ToXContent.EMPTY_PARAMS; - final File[] dirs = new File[randomIntBetween(1, 5)]; + final Path[] dirs = new Path[randomIntBetween(1, 5)]; int numStates = randomIntBetween(1, 5); int numLegacy = randomIntBetween(0, numStates); List meta = new ArrayList<>(); for (int i = 0; i < numStates; i++) { meta.add(randomMeta()); } - Set corruptedFiles = new HashSet<>(); + Set corruptedFiles = new HashSet<>(); MetaDataStateFormat format = LocalGatewayMetaState.globalStateFormat(randomFrom(XContentType.values()), params, randomBoolean()); for (int i = 0; i < dirs.length; i++) { - dirs[i] = newTempDir(LifecycleScope.TEST); - Files.createDirectories(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME).toPath()); + dirs[i] = newTempDir(LifecycleScope.TEST).toPath(); + Files.createDirectories(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME)); for (int j = 0; j < numLegacy; j++) { XContentType type = format.format(); if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { - File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-"+j); - Files.createFile(file.toPath()); // randomly create 0-byte files -- there is extra logic to skip them + Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-"+j); + Files.createFile(file); // randomly create 0-byte files -- there is extra logic to skip them } else { - try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, new FileOutputStream(new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + j)))) { + try (XContentBuilder xcontentBuilder = XContentFactory.contentBuilder(type, Files.newOutputStream(dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j)))) { xcontentBuilder.startObject(); MetaData.Builder.toXContent(meta.get(j), xcontentBuilder, params); xcontentBuilder.endObject(); @@ -347,16 +347,16 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { for (int j = numLegacy; j < numStates; j++) { format.write(meta.get(j), LocalGatewayMetaState.GLOBAL_STATE_FILE_PREFIX, j, dirs[i]); if (randomBoolean() && (j < numStates - 1 || dirs.length > 0 && i != 0)) { // corrupt a file that we do not necessarily need here.... - File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + j + ".st"); + Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + j + ".st"); corruptedFiles.add(file); MetaDataStateFormatTest.corruptFile(file, logger); } } } - List dirList = Arrays.asList(dirs); + List dirList = Arrays.asList(dirs); Collections.shuffle(dirList, getRandom()); - MetaData loadedMetaData = MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new File[0])); + MetaData loadedMetaData = MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new Path[0])); MetaData latestMetaData = meta.get(numStates-1); assertThat(loadedMetaData.uuid(), not(equalTo("_na_"))); assertThat(loadedMetaData.uuid(), equalTo(latestMetaData.uuid())); @@ -373,14 +373,14 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { // now corrupt all the latest ones and make sure we fail to load the state if (numStates > numLegacy) { for (int i = 0; i < dirs.length; i++) { - File file = new File(new File(dirs[i], MetaDataStateFormat.STATE_DIR_NAME), "global-" + (numStates-1) + ".st"); + Path file = dirs[i].resolve(MetaDataStateFormat.STATE_DIR_NAME).resolve("global-" + (numStates-1) + ".st"); if (corruptedFiles.contains(file)) { continue; } MetaDataStateFormatTest.corruptFile(file, logger); } try { - MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new File[0])); + MetaDataStateFormat.loadLatestState(logger, format, LocalGatewayMetaState.GLOBAL_STATE_FILE_PATTERN, "foobar", dirList.toArray(new Path[0])); fail("latest version can not be read"); } catch (ElasticsearchException ex) { assertThat(ex.getCause(), instanceOf(CorruptStateException.class)); @@ -422,7 +422,7 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { } @Override - protected Directory newDirectory(File dir) throws IOException { + protected Directory newDirectory(Path dir) throws IOException { MockDirectoryWrapper mock = new MockDirectoryWrapper(getRandom(), super.newDirectory(dir)); closeAfterSuite(new CloseableDirectory(mock, suiteFailureMarker)); return mock; @@ -559,4 +559,10 @@ public class MetaDataStateFormatTest extends ElasticsearchTestCase { } } } + + public Path[] content(Path dir) throws IOException { + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + return Iterators.toArray(stream.iterator(), Path.class); + } + } }