From 95171e2bc2e5de3fc34529062c3f0a07e9388105 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 6 Nov 2014 15:12:16 +0100 Subject: [PATCH] [CORE] Cut over to Path API for file deletion Today we use the File API for file deletion as well as recursive directory deletions. This API returns a boolean if operations are successful while hiding the actual reason why they failed. The Path API throws and actual exception that might provide better insights and debug information. Closes #8366 --- dev-tools/forbidden/all-signatures.txt | 2 + .../common/blobstore/BlobContainer.java | 2 +- .../common/blobstore/BlobStore.java | 4 +- .../common/blobstore/fs/FsBlobContainer.java | 10 +- .../common/blobstore/fs/FsBlobStore.java | 6 +- .../blobstore/url/URLBlobContainer.java | 2 +- .../http/client/HttpDownloadHelper.java | 5 +- .../common/io/FileSystemUtils.java | 100 +++++------------- .../gateway/local/LocalGateway.java | 7 +- .../state/meta/LocalGatewayMetaState.java | 26 ++++- .../local/state/meta/MetaDataStateFormat.java | 4 +- .../state/shards/LocalGatewayShardsState.java | 8 +- .../gateway/none/NoneGateway.java | 7 +- .../gateway/local/LocalIndexShardGateway.java | 7 +- .../index/store/fs/FsIndexStore.java | 7 +- .../index/translog/fs/FsTranslog.java | 7 +- .../index/translog/fs/RafReference.java | 12 ++- .../indices/store/IndicesStore.java | 7 +- .../elasticsearch/plugins/PluginManager.java | 36 +++++-- .../benchmark/fs/FsAppendBenchmark.java | 5 +- .../allocation/ClusterRerouteTests.java | 3 +- .../index/mapper/FileBasedMappingsTests.java | 3 +- .../translog/fs/FsBufferedTranslogTests.java | 7 +- .../translog/fs/FsSimpleTranslogTests.java | 7 +- .../plugins/PluginManagerTests.java | 17 +-- .../script/ScriptServiceTests.java | 5 +- .../SharedClusterSnapshotRestoreTests.java | 9 +- .../mockstore/BlobContainerWrapper.java | 4 +- .../snapshots/mockstore/BlobStoreWrapper.java | 4 +- .../snapshots/mockstore/MockRepository.java | 4 +- .../fullrestart/FullRestartStressTest.java | 7 +- .../RollingRestartStressTest.java | 8 +- .../test/InternalTestCluster.java | 20 ++-- .../watcher/FileWatcherTest.java | 35 +++--- 34 files changed, 244 insertions(+), 153 deletions(-) diff --git a/dev-tools/forbidden/all-signatures.txt b/dev-tools/forbidden/all-signatures.txt index 5f13cea4af7..bb7a0d44360 100644 --- a/dev-tools/forbidden/all-signatures.txt +++ b/dev-tools/forbidden/all-signatures.txt @@ -1,3 +1,5 @@ @defaultMessage Convert to URI java.net.URL#getPath() java.net.URL#getFile() + +java.io.File#delete() @ use Files.delete for real exception, IOUtils.deleteFilesIgnoringExceptions if you dont care diff --git a/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java b/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java index bac1e1ed719..3ff5158a39a 100644 --- a/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java +++ b/src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java @@ -51,7 +51,7 @@ public interface BlobContainer { */ OutputStream createOutput(String blobName) throws IOException; - boolean deleteBlob(String blobName) throws IOException; + void deleteBlob(String blobName) throws IOException; void deleteBlobsByPrefix(String blobNamePrefix) throws IOException; diff --git a/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java b/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java index eb91d484851..df3a8c6be8d 100644 --- a/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java +++ b/src/main/java/org/elasticsearch/common/blobstore/BlobStore.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.common.blobstore; +import java.io.IOException; + /** * */ @@ -25,7 +27,7 @@ public interface BlobStore { BlobContainer blobContainer(BlobPath path); - void delete(BlobPath path); + void delete(BlobPath path) throws IOException; void close(); } diff --git a/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java index 12119bce910..bd66ad3b564 100644 --- a/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java +++ b/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java @@ -30,6 +30,8 @@ import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.io.FileSystemUtils; import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; /** * @@ -65,8 +67,12 @@ public class FsBlobContainer extends AbstractBlobContainer { return builder.immutableMap(); } - public boolean deleteBlob(String blobName) throws IOException { - return new File(path, blobName).delete(); + @Override + public void deleteBlob(String blobName) throws IOException { + Path blobPath = new File(path, blobName).toPath(); + if (Files.exists(blobPath)) { + Files.delete(blobPath); + } } @Override diff --git a/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java b/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java index d63f63fbb72..d96de4ab26b 100644 --- a/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java +++ b/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.blobstore.fs; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; @@ -30,6 +31,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import java.io.File; +import java.io.IOException; /** * @@ -74,8 +76,8 @@ public class FsBlobStore extends AbstractComponent implements BlobStore { } @Override - public void delete(BlobPath path) { - FileSystemUtils.deleteRecursively(buildPath(path)); + public void delete(BlobPath path) throws IOException { + IOUtils.rm(buildPath(path).toPath()); } @Override diff --git a/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java b/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java index 5dd633e9114..e7ee3803402 100644 --- a/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java +++ b/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java @@ -73,7 +73,7 @@ public class URLBlobContainer extends AbstractBlobContainer { * This operation is not supported by URLBlobContainer */ @Override - public boolean deleteBlob(String blobName) throws IOException { + public void deleteBlob(String blobName) throws IOException { throw new UnsupportedOperationException("URL repository is read only"); } diff --git a/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java b/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java index 81353c2134b..aeb44a0a828 100644 --- a/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java +++ b/src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java @@ -22,6 +22,7 @@ package org.elasticsearch.common.http.client; import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchTimeoutException; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.unit.TimeValue; import java.io.*; @@ -346,7 +347,7 @@ public class HttpDownloadHelper { // Try to delete the garbage we'd otherwise leave // behind. IOUtils.closeWhileHandlingException(os, is); - dest.delete(); + FileSystemUtils.deleteFilesIgnoringExceptions(dest.toPath()); } else { IOUtils.close(os, is); } @@ -385,7 +386,7 @@ public class HttpDownloadHelper { } else { IOUtils.closeWhileHandlingException(is, os); if (dest != null && dest.exists()) { - dest.delete(); + FileSystemUtils.deleteFilesIgnoringExceptions(dest.toPath()); } } } diff --git a/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index d7a9156a5f3..30257bd6aa4 100644 --- a/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -78,84 +78,34 @@ public class FileSystemUtils { } /** - * Deletes the given files recursively. if deleteRoots is set to true - * the given root files will be deleted as well. Otherwise only their content is deleted. + * Returns an array of {@link Path} build from the correspondent element + * in the input array using {@link java.io.File#toPath()} + * @param files the files to get paths for */ - public static boolean deleteRecursively(File[] roots, boolean deleteRoots) { - - boolean deleted = true; - for (File root : roots) { - deleted &= deleteRecursively(root, deleteRoots); + public static Path[] toPaths(File... files) { + Path[] paths = new Path[files.length]; + for (int i = 0; i < files.length; i++) { + paths[i] = files[i].toPath(); } - return deleted; + return paths; } /** - * Deletes all subdirectories of the given roots recursively. + * Deletes all subdirectories in the given path recursively + * @throws java.lang.IllegalArgumentException if the given path is not a directory */ - public static boolean deleteSubDirectories(File[] roots) { - - boolean deleted = true; - for (File root : roots) { - if (root.isDirectory()) { - File[] files = root.listFiles(new FileFilter() { - @Override - public boolean accept(File pathname) { - return pathname.isDirectory(); - } - }); - deleted &= deleteRecursively(files, true); - } - - } - return deleted; - } - - /** - * Deletes the given files recursively including the given roots. - */ - public static boolean deleteRecursively(File... roots) { - return deleteRecursively(roots, true); - } - - /** - * Delete the supplied {@link java.io.File} - for directories, - * recursively delete any nested directories or files as well. - * - * @param root the root File to delete - * @param deleteRoot whether or not to delete the root itself or just the content of the root. - * @return true if the File was deleted, - * otherwise false - */ - public static boolean deleteRecursively(File root, boolean deleteRoot) { - if (root != null && root.exists()) { - if (root.isDirectory()) { - File[] children = root.listFiles(); - if (children != null) { - for (File aChildren : children) { - deleteRecursively(aChildren, true); + public static void deleteSubDirectories(Path... paths) throws IOException { + for (Path path : paths) { + try (DirectoryStream stream = Files.newDirectoryStream(path)) { + for (Path subPath : stream) { + if (Files.isDirectory(subPath)) { + IOUtils.rm(subPath); } } } - - if (deleteRoot) { - return root.delete(); - } else { - return true; - } } - return false; } - /** - * Ensure that any writes to the given file is written to the storage device that contains it. - * @param fileToSync the file to fsync - * @param isDir if true, the given file is a directory (we open for read and ignore IOExceptions, - * because not all file systems and operating systems allow to fsync on a directory) - */ - public static void syncFile(File fileToSync, boolean isDir) throws IOException { - IOUtils.fsync(fileToSync.toPath(), isDir); - } /** * Check that a directory exists, is a directory and is readable @@ -181,11 +131,19 @@ public class FileSystemUtils { private FileSystemUtils() {} - public static void tryDeleteFile(File file) { - try { - file.delete(); - } catch (SecurityException e1) { - // ignore + /** + * Temporary solution until LUCENE-6051 is fixed + * @see org.apache.lucene.util.IOUtils#deleteFilesIgnoringExceptions(java.nio.file.Path...) + */ + public static void deleteFilesIgnoringExceptions(Path... files) { + for (Path name : files) { + if (name != null) { + try { + Files.delete(name); + } catch (Throwable ignored) { + // ignore + } + } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index d980c93a84b..9ecefc87022 100644 --- a/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -22,6 +22,7 @@ package org.elasticsearch.gateway.local; import com.carrotsearch.hppc.ObjectFloatOpenHashMap; import com.carrotsearch.hppc.ObjectOpenHashSet; import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.cluster.*; @@ -197,7 +198,11 @@ public class LocalGateway extends AbstractLifecycleComponent implements @Override public void reset() throws Exception { - FileSystemUtils.deleteRecursively(nodeEnv.nodeDataLocations()); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.nodeDataLocations())); + } catch (Exception ex) { + logger.debug("failed to delete shard locations", ex); + } } @Override diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java index 7e276443430..1280a5d0475 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java @@ -21,6 +21,7 @@ package org.elasticsearch.gateway.local.state.meta; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.Version; @@ -49,6 +50,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.nio.file.Files; import java.util.List; import java.util.Map; import java.util.Set; @@ -243,7 +245,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS if (!newMetaData.hasIndex(current.index())) { logger.debug("[{}] deleting index that is no longer part of the metadata (indices: [{}])", current.index(), newMetaData.indices().keys()); if (nodeEnv.hasNodeFile()) { - FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index()))); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(current.index())))); + } catch (Exception ex) { + logger.debug("[{}] failed to delete index", ex, current.index()); + } } try { nodeIndexDeletedAction.nodeIndexStoreDeleted(event.state(), current.index(), event.state().nodes().localNodeId()); @@ -280,7 +286,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS if (indexMetaData != null) { if (danglingTimeout.millis() == 0) { logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, timeout set to 0, deleting now", indexName); - FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(indexName))); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(indexName)))); + } catch (Exception ex) { + logger.debug("[{}] failed to delete dangling index", ex, indexName); + } } else { logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, scheduling to delete in [{}], auto import to cluster state [{}]", indexName, danglingTimeout, autoImportDangled); danglingIndices.put(indexName, new DanglingIndex(indexName, threadPool.schedule(danglingTimeout, ThreadPool.Names.SAME, new RemoveDanglingIndex(indexName)))); @@ -517,7 +527,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS if (!name.startsWith("metadata-")) { continue; } - stateFile.delete(); + try { + Files.delete(stateFile.toPath()); + } catch (Exception ex) { + logger.debug("failed to delete file " + stateFile, ex); + } } } @@ -576,7 +590,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS return; } logger.warn("[{}] deleting dangling index", index); - FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(index))); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(index)))); + } catch (Exception ex) { + logger.debug("failed to delete dangling index", ex); + } } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java b/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java index 8dbff01cd8d..1126807f0f9 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java @@ -208,7 +208,9 @@ public abstract class MetaDataStateFormat { if (file.getName().equals(fileName)) { continue; } - Files.delete(file.toPath()); + if (Files.exists(file.toPath())) { + Files.delete(file.toPath()); + } } } } diff --git a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java index 139fa35cd33..32640d73491 100644 --- a/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java +++ b/src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java @@ -37,6 +37,7 @@ import org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat; import org.elasticsearch.index.shard.ShardId; import java.io.*; +import java.nio.file.Files; import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -317,7 +318,12 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste if (!name.startsWith("shards-")) { continue; } - stateFile.delete(); + try { + Files.delete(stateFile.toPath()); + } catch (Exception ex) { + logger.debug("Failed to delete state file {}", ex, stateFile); + } + } } diff --git a/src/main/java/org/elasticsearch/gateway/none/NoneGateway.java b/src/main/java/org/elasticsearch/gateway/none/NoneGateway.java index 5a9abdeb5e2..7ecbc05e9e3 100644 --- a/src/main/java/org/elasticsearch/gateway/none/NoneGateway.java +++ b/src/main/java/org/elasticsearch/gateway/none/NoneGateway.java @@ -19,6 +19,7 @@ package org.elasticsearch.gateway.none; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction; @@ -117,7 +118,11 @@ public class NoneGateway extends AbstractLifecycleComponent implements if (!newMetaData.hasIndex(current.index())) { logger.debug("[{}] deleting index that is no longer part of the metadata (indices: [{}])", current.index(), newMetaData.indices().keys()); if (nodeEnv.hasNodeFile()) { - FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index()))); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(current.index())))); + } catch (Exception ex) { + logger.debug("failed to delete shard locations", ex); + } } try { nodeIndexDeletedAction.nodeIndexStoreDeleted(event.state(), current.index(), event.state().nodes().localNodeId()); diff --git a/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java b/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java index 24bc1eee3b3..25a09186fb5 100644 --- a/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java +++ b/src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java @@ -51,6 +51,7 @@ import org.elasticsearch.threadpool.ThreadPool; import java.io.EOFException; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.Arrays; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -290,7 +291,11 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen } indexShard.performRecoveryFinalization(true); - recoveringTranslogFile.delete(); + try { + Files.delete(recoveringTranslogFile.toPath()); + } catch (Exception ex) { + logger.debug("Failed to delete recovering translog file {}", ex, recoveringTranslogFile); + } for (final String type : typesToUpdate) { final CountDownLatch latch = new CountDownLatch(1); diff --git a/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java b/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java index ad552b86229..e4e6bb53408 100644 --- a/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java +++ b/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.store.fs; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; @@ -81,7 +82,11 @@ public abstract class FsIndexStore extends AbstractIndexStore { if (indexService.hasShard(shardId.id())) { throw new ElasticsearchIllegalStateException(shardId + " allocated, can't be deleted"); } - FileSystemUtils.deleteRecursively(shardLocations(shardId)); + try { + IOUtils.rm(FileSystemUtils.toPaths(shardLocations(shardId))); + } catch (Exception ex) { + logger.debug("failed to delete shard locations", ex); + } } public File[] shardLocations(ShardId shardId) { diff --git a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java index 1ad6f1635a6..cf02e592150 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.translog.*; import java.io.File; import java.io.IOException; import java.nio.channels.ClosedChannelException; +import java.nio.file.Files; import java.util.Collections; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.ReadWriteLock; @@ -215,9 +216,9 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog continue; } try { - file.delete(); - } catch (Exception e) { - // ignore + Files.delete(file.toPath()); + } catch (Exception ex) { + logger.debug("failed to delete " + file, ex); } } } diff --git a/src/main/java/org/elasticsearch/index/translog/fs/RafReference.java b/src/main/java/org/elasticsearch/index/translog/fs/RafReference.java index ee8752357e6..35def429548 100644 --- a/src/main/java/org/elasticsearch/index/translog/fs/RafReference.java +++ b/src/main/java/org/elasticsearch/index/translog/fs/RafReference.java @@ -19,11 +19,15 @@ package org.elasticsearch.index.translog.fs; +import org.apache.lucene.util.IOUtils; +import org.elasticsearch.common.io.FileSystemUtils; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.util.concurrent.atomic.AtomicInteger; /** @@ -70,12 +74,14 @@ public class RafReference { if (refCount.decrementAndGet() <= 0) { try { raf.close(); - if (delete) { - file.delete(); - } } catch (IOException e) { // ignore + } finally { + if (delete) { + FileSystemUtils.deleteFilesIgnoringExceptions(file.toPath()); + } } + } } } diff --git a/src/main/java/org/elasticsearch/indices/store/IndicesStore.java b/src/main/java/org/elasticsearch/indices/store/IndicesStore.java index bc5aa05ad62..1fd365cbe99 100644 --- a/src/main/java/org/elasticsearch/indices/store/IndicesStore.java +++ b/src/main/java/org/elasticsearch/indices/store/IndicesStore.java @@ -20,6 +20,7 @@ package org.elasticsearch.indices.store; import org.apache.lucene.store.StoreRateLimiting; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.Version; import org.elasticsearch.cluster.*; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -323,7 +324,11 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe File[] shardLocations = nodeEnv.shardLocations(shardId); if (FileSystemUtils.exists(shardLocations)) { logger.debug("{} deleting shard that is no longer used", shardId); - FileSystemUtils.deleteRecursively(shardLocations); + try { + IOUtils.rm(FileSystemUtils.toPaths(shardLocations)); + } catch (Exception ex) { + logger.debug("failed to delete shard locations", ex); + } } } } else { diff --git a/src/main/java/org/elasticsearch/plugins/PluginManager.java b/src/main/java/org/elasticsearch/plugins/PluginManager.java index dd49772dafb..b4f7017ea73 100644 --- a/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -21,6 +21,7 @@ package org.elasticsearch.plugins; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchIllegalStateException; import org.elasticsearch.ElasticsearchTimeoutException; @@ -32,6 +33,7 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.env.Environment; +import org.elasticsearch.index.Index; import org.elasticsearch.node.internal.InternalSettingsPreparer; import javax.net.ssl.HttpsURLConnection; @@ -222,12 +224,20 @@ public class PluginManager { // ignore } } - pluginFile.delete(); + try { + Files.delete(pluginFile.toPath()); + } catch (Exception ex) { + log("Failed to delete plugin file" + pluginFile + " " + ex); + } } if (FileSystemUtils.hasExtensions(extractLocation, ".java")) { debug("Plugin installation assumed to be site plugin, but contains source code, aborting installation..."); - FileSystemUtils.deleteRecursively(extractLocation); + try { + IOUtils.rm(extractLocation.toPath()); + } catch(Exception ex) { + debug("Failed to remove site plugin from path " + extractLocation + " - " + ex.getMessage()); + } throw new IllegalArgumentException("Plugin installation assumed to be site plugin, but contains source code, aborting installation."); } @@ -237,7 +247,9 @@ public class PluginManager { if (binFile.exists() && binFile.isDirectory()) { File toLocation = pluginHandle.binDir(environment); debug("Found bin, moving to " + toLocation.getAbsolutePath()); - FileSystemUtils.deleteRecursively(toLocation); + if (toLocation.exists()) { + IOUtils.rm(toLocation.toPath()); + } if (!binFile.renameTo(toLocation)) { throw new IOException("Could not move ["+ binFile.getAbsolutePath() + "] to [" + toLocation.getAbsolutePath() + "]"); } @@ -294,27 +306,33 @@ public class PluginManager { File pluginToDelete = pluginHandle.extractedDir(environment); if (pluginToDelete.exists()) { debug("Removing: " + pluginToDelete.getPath()); - if (!FileSystemUtils.deleteRecursively(pluginToDelete, true)) { + try { + IOUtils.rm(pluginToDelete.toPath()); + } catch (IOException ex){ throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " + - pluginToDelete.toString()); + pluginToDelete.toString(), ex); } removed = true; } pluginToDelete = pluginHandle.distroFile(environment); if (pluginToDelete.exists()) { debug("Removing: " + pluginToDelete.getPath()); - if (!pluginToDelete.delete()) { + try { + Files.delete(pluginToDelete.toPath()); + } catch (Exception ex) { throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " + - pluginToDelete.toString()); + pluginToDelete.toString(), ex); } removed = true; } File binLocation = pluginHandle.binDir(environment); if (binLocation.exists()) { debug("Removing: " + binLocation.getPath()); - if (!FileSystemUtils.deleteRecursively(binLocation)) { + try { + IOUtils.rm(binLocation.toPath()); + } catch (IOException ex){ throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " + - binLocation.toString()); + binLocation.toString(), ex); } removed = true; } diff --git a/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java b/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java index f8ec2cf1efa..89acf1148f5 100644 --- a/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java @@ -18,13 +18,16 @@ */ package org.elasticsearch.benchmark.fs; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.unit.ByteSizeValue; import java.io.File; import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.file.Paths; import java.util.Random; /** @@ -33,7 +36,7 @@ import java.util.Random; public class FsAppendBenchmark { public static void main(String[] args) throws Exception { - new File("work/test.log").delete(); + FileSystemUtils.deleteFilesIgnoringExceptions(Paths.get("work/test.log")); RandomAccessFile raf = new RandomAccessFile("work/test.log", "rw"); raf.setLength(0); diff --git a/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteTests.java b/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteTests.java index 406883fd719..cfea1fd5f84 100644 --- a/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteTests.java +++ b/src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.allocation; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse; @@ -196,7 +197,7 @@ public class ClusterRerouteTests extends ElasticsearchIntegrationTest { logger.info("--> deleting the shard data [{}] ", Arrays.toString(shardLocation)); assertThat(FileSystemUtils.exists(shardLocation), equalTo(true)); // verify again after cluster was shut down - assertThat(FileSystemUtils.deleteRecursively(shardLocation), equalTo(true)); + IOUtils.rm(FileSystemUtils.toPaths(shardLocation)); logger.info("--> starting nodes back, will not allocate the shard since it has no data, but the index will be there"); node_1 = internalCluster().startNode(commonSettings); diff --git a/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java b/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java index cc82ea32510..ff2d78a1336 100644 --- a/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper; import com.google.common.collect.ImmutableSet; import com.google.common.io.Files; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -98,7 +99,7 @@ public class FileBasedMappingsTests extends ElasticsearchTestCase { assertEquals(ImmutableSet.of("f", "g", "h"), properties.keySet()); } } finally { - FileSystemUtils.deleteRecursively(configDir); + IOUtils.rm(configDir.toPath()); } } diff --git a/src/test/java/org/elasticsearch/index/translog/fs/FsBufferedTranslogTests.java b/src/test/java/org/elasticsearch/index/translog/fs/FsBufferedTranslogTests.java index 346d8cfd18b..4485bd9e4a0 100644 --- a/src/test/java/org/elasticsearch/index/translog/fs/FsBufferedTranslogTests.java +++ b/src/test/java/org/elasticsearch/index/translog/fs/FsBufferedTranslogTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.translog.fs; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.index.translog.AbstractSimpleTranslogTests; @@ -26,6 +27,8 @@ import org.elasticsearch.index.translog.Translog; import org.junit.AfterClass; import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; /** * @@ -49,7 +52,7 @@ public class FsBufferedTranslogTests extends AbstractSimpleTranslogTests { } @AfterClass - public static void cleanup() { - FileSystemUtils.deleteRecursively(new File("data/fs-buf-translog"), true); + public static void cleanup() throws IOException { + IOUtils.rm(Paths.get("data/fs-buf-translog")); } } diff --git a/src/test/java/org/elasticsearch/index/translog/fs/FsSimpleTranslogTests.java b/src/test/java/org/elasticsearch/index/translog/fs/FsSimpleTranslogTests.java index ceb02640e5e..231fb0ecdf4 100644 --- a/src/test/java/org/elasticsearch/index/translog/fs/FsSimpleTranslogTests.java +++ b/src/test/java/org/elasticsearch/index/translog/fs/FsSimpleTranslogTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.translog.fs; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.index.translog.AbstractSimpleTranslogTests; @@ -26,6 +27,8 @@ import org.elasticsearch.index.translog.Translog; import org.junit.AfterClass; import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; /** * @@ -45,7 +48,7 @@ public class FsSimpleTranslogTests extends AbstractSimpleTranslogTests { } @AfterClass - public static void cleanup() { - FileSystemUtils.deleteRecursively(new File("data/fs-simple-translog"), true); + public static void cleanup() throws IOException { + IOUtils.rm(Paths.get("data/fs-simple-translog")); } } diff --git a/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java b/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java index efa839c7957..561ac735076 100644 --- a/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java +++ b/src/test/java/org/elasticsearch/plugins/PluginManagerTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.plugins; import com.google.common.base.Predicate; import org.apache.http.impl.client.HttpClients; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalArgumentException; import org.elasticsearch.ElasticsearchTimeoutException; @@ -46,6 +47,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; import java.net.URI; +import java.nio.file.Paths; import java.util.concurrent.TimeUnit; import static org.elasticsearch.common.io.FileSystemUtilsTests.assertFileContent; @@ -65,12 +67,12 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { private static final String PLUGIN_DIR = "plugins"; @After - public void afterTest() { + public void afterTest() throws IOException { deletePluginsFolder(); } @Before - public void beforeTest() { + public void beforeTest() throws IOException { deletePluginsFolder(); } @@ -123,8 +125,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { assertThat(toolFile.canExecute(), is(true)); } finally { // we need to clean up the copied dirs - FileSystemUtils.deleteRecursively(pluginBinDir); - FileSystemUtils.deleteRecursively(pluginConfigDir); + IOUtils.rm(pluginBinDir.toPath(), pluginConfigDir.toPath()); } } @@ -206,7 +207,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { assertFileContent(pluginConfigDir, "dir/subdir/testsubdir.txt.new", "version2\n"); } finally { // we need to clean up the copied dirs - FileSystemUtils.deleteRecursively(pluginConfigDir); + IOUtils.rm(pluginConfigDir.toPath()); } } @@ -230,7 +231,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { assertDirectoryExists(pluginBinDir); } finally { // we need to clean up the copied dirs - FileSystemUtils.deleteRecursively(pluginBinDir); + IOUtils.rm(pluginBinDir.toPath()); } } @@ -463,8 +464,8 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest { return false; } - private void deletePluginsFolder() { - FileSystemUtils.deleteRecursively(new File(PLUGIN_DIR)); + private void deletePluginsFolder() throws IOException { + IOUtils.rm(Paths.get(PLUGIN_DIR)); } @Test diff --git a/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 091d21f50e3..1308442f83a 100644 --- a/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -31,6 +31,7 @@ import org.junit.Test; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.Map; import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; @@ -73,8 +74,8 @@ public class ScriptServiceTests extends ElasticsearchTestCase { assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file")); logger.info("--> delete both files"); - assertThat(testFileNoExt.delete(), equalTo(true)); - assertThat(testFileWithExt.delete(), equalTo(true)); + Files.delete(testFileNoExt.toPath()); + Files.delete(testFileWithExt.toPath()); resourceWatcherService.notifyNow(); logger.info("--> verify that file with extension was correctly removed"); diff --git a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java index 9023002a2fd..1aad7819cd2 100644 --- a/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java +++ b/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java @@ -22,6 +22,7 @@ package org.elasticsearch.snapshots; import com.carrotsearch.randomizedtesting.LifecycleScope; import com.google.common.base.Predicate; import com.google.common.collect.ImmutableList; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LuceneTestCase.Slow; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ListenableActionFuture; @@ -45,6 +46,7 @@ import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.metadata.SnapshotMetaData; import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.store.support.AbstractIndexStore; @@ -55,6 +57,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging; import org.junit.Test; import java.io.File; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -732,8 +735,8 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { File testIndex1 = new File(indices, "test-idx-1"); File testIndex2 = new File(indices, "test-idx-2"); File testIndex2Shard0 = new File(testIndex2, "0"); - new File(testIndex1, "snapshot-test-snap-1").delete(); - new File(testIndex2Shard0, "snapshot-test-snap-1").delete(); + FileSystemUtils.deleteFilesIgnoringExceptions(new File(testIndex1, "snapshot-test-snap-1").toPath()); + FileSystemUtils.deleteFilesIgnoringExceptions(new File(testIndex2Shard0, "snapshot-test-snap-1").toPath()); logger.info("--> delete snapshot"); client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); @@ -768,7 +771,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests { logger.info("--> delete index metadata and shard metadata"); File metadata = new File(repo, "metadata-test-snap-1"); - assertThat(metadata.delete(), equalTo(true)); + Files.delete(metadata.toPath()); logger.info("--> delete snapshot"); client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get(); diff --git a/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java b/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java index 055cd3276f1..17c92a1c271 100644 --- a/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java +++ b/src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java @@ -58,8 +58,8 @@ public class BlobContainerWrapper implements BlobContainer { } @Override - public boolean deleteBlob(String blobName) throws IOException { - return delegate.deleteBlob(blobName); + public void deleteBlob(String blobName) throws IOException { + delegate.deleteBlob(blobName); } @Override diff --git a/src/test/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java b/src/test/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java index e5df46eb998..086aac209b5 100644 --- a/src/test/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java +++ b/src/test/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java @@ -22,6 +22,8 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import java.io.IOException; + /** * */ @@ -39,7 +41,7 @@ public class BlobStoreWrapper implements BlobStore { } @Override - public void delete(BlobPath path) { + public void delete(BlobPath path) throws IOException { delegate.delete(path); } diff --git a/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java b/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java index f57177bef01..1e8f0b24da0 100644 --- a/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java +++ b/src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java @@ -262,9 +262,9 @@ public class MockRepository extends FsRepository { } @Override - public boolean deleteBlob(String blobName) throws IOException { + public void deleteBlob(String blobName) throws IOException { maybeIOExceptionOrBlock(blobName); - return super.deleteBlob(blobName); + super.deleteBlob(blobName); } @Override diff --git a/src/test/java/org/elasticsearch/stresstest/fullrestart/FullRestartStressTest.java b/src/test/java/org/elasticsearch/stresstest/fullrestart/FullRestartStressTest.java index 8d06e0c0b5d..2d76d44c62d 100644 --- a/src/test/java/org/elasticsearch/stresstest/fullrestart/FullRestartStressTest.java +++ b/src/test/java/org/elasticsearch/stresstest/fullrestart/FullRestartStressTest.java @@ -19,6 +19,7 @@ package org.elasticsearch.stresstest.fullrestart; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.count.CountResponse; @@ -200,7 +201,11 @@ public class FullRestartStressTest { File[] nodeDatas = ((InternalNode) node).injector().getInstance(NodeEnvironment.class).nodeDataLocations(); node.close(); if (clearNodeWork && !settings.get("gateway.type").equals("local")) { - FileSystemUtils.deleteRecursively(nodeDatas); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeDatas)); + } catch (Exception ex) { + logger.debug("failed to remove node data locations", ex); + } } } diff --git a/src/test/java/org/elasticsearch/stresstest/rollingrestart/RollingRestartStressTest.java b/src/test/java/org/elasticsearch/stresstest/rollingrestart/RollingRestartStressTest.java index 87bfb31cdfa..d44f59cbec5 100644 --- a/src/test/java/org/elasticsearch/stresstest/rollingrestart/RollingRestartStressTest.java +++ b/src/test/java/org/elasticsearch/stresstest/rollingrestart/RollingRestartStressTest.java @@ -19,6 +19,7 @@ package org.elasticsearch.stresstest.rollingrestart; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.count.CountResponse; import org.elasticsearch.action.get.GetResponse; @@ -170,7 +171,12 @@ public class RollingRestartStressTest { File[] nodeData = ((InternalNode) nodes[nodeIndex]).injector().getInstance(NodeEnvironment.class).nodeDataLocations(); nodes[nodeIndex].close(); if (clearNodeData) { - FileSystemUtils.deleteRecursively(nodeData); + try { + IOUtils.rm(FileSystemUtils.toPaths(nodeData)); + } catch (Exception ex) { + logger.debug("Failed to delete node data directories", ex); + + } } try { diff --git a/src/test/java/org/elasticsearch/test/InternalTestCluster.java b/src/test/java/org/elasticsearch/test/InternalTestCluster.java index cc028e632f9..cbc04eb05ce 100644 --- a/src/test/java/org/elasticsearch/test/InternalTestCluster.java +++ b/src/test/java/org/elasticsearch/test/InternalTestCluster.java @@ -99,6 +99,7 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.nio.file.Path; import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -164,7 +165,7 @@ public final class InternalTestCluster extends TestCluster { /* sorted map to make traverse order reproducible, concurrent since we do checks on it not within a sync block */ private final NavigableMap nodes = new TreeMap<>(); - private final Set dataDirToClean = new HashSet<>(); + private final Set dataDirToClean = new HashSet<>(); private final String clusterName; @@ -788,7 +789,7 @@ public final class InternalTestCluster extends TestCluster { if (callback.clearData(name)) { NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, node); if (nodeEnv.hasNodeFile()) { - FileSystemUtils.deleteRecursively(nodeEnv.nodeDataLocations()); + IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.nodeDataLocations())); } } node = (InternalNode) nodeBuilder().settings(node.settings()).settings(newSettings).node(); @@ -953,12 +954,17 @@ public final class InternalTestCluster extends TestCluster { private void wipeDataDirectories() { if (!dataDirToClean.isEmpty()) { - boolean deleted = false; try { - deleted = FileSystemUtils.deleteSubDirectories(dataDirToClean.toArray(new File[dataDirToClean.size()])); + for (Path path : dataDirToClean) { + try { + FileSystemUtils.deleteSubDirectories(path); + logger.info("Successfully wiped data directory for node location: {}", path); + } catch (IOException e) { + logger.info("Failed to wipe data directory for node location: {}", path); + } + } } finally { - logger.info("Wipe data directory for all nodes locations: {} success: {}", this.dataDirToClean, deleted); - this.dataDirToClean.clear(); + dataDirToClean.clear(); } } } @@ -1399,7 +1405,7 @@ public final class InternalTestCluster extends TestCluster { assert !nodeAndClient.node().isClosed(); NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, nodeAndClient.node); if (nodeEnv.hasNodeFile()) { - dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataLocations())); + dataDirToClean.addAll(Arrays.asList(FileSystemUtils.toPaths(nodeEnv.nodeDataLocations()))); } nodes.put(nodeAndClient.name, nodeAndClient); applyDisruptionSchemeToNode(nodeAndClient); diff --git a/src/test/java/org/elasticsearch/watcher/FileWatcherTest.java b/src/test/java/org/elasticsearch/watcher/FileWatcherTest.java index ce401714f97..8d5d0b89745 100644 --- a/src/test/java/org/elasticsearch/watcher/FileWatcherTest.java +++ b/src/test/java/org/elasticsearch/watcher/FileWatcherTest.java @@ -19,17 +19,19 @@ package org.elasticsearch.watcher; import com.carrotsearch.randomizedtesting.LifecycleScope; +import org.apache.lucene.util.IOUtils; import org.elasticsearch.test.ElasticsearchTestCase; import org.junit.Test; import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.io.Files.*; -import static org.elasticsearch.common.io.FileSystemUtils.deleteRecursively; import static org.hamcrest.Matchers.*; /** @@ -114,7 +116,7 @@ public class FileWatcherTest extends ElasticsearchTestCase { fileWatcher.checkAndNotify(); assertThat(changes.notifications(), hasSize(0)); - testFile.delete(); + Files.delete(testFile.toPath()); fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains(equalTo("onFileDeleted: test.txt"))); @@ -160,8 +162,8 @@ public class FileWatcherTest extends ElasticsearchTestCase { fileWatcher.checkAndNotify(); assertThat(changes.notifications(), hasSize(0)); - new File(testDir, "test1.txt").delete(); - new File(testDir, "test2.txt").delete(); + Files.delete(new File(testDir, "test1.txt").toPath()); + Files.delete(new File(testDir, "test2.txt").toPath()); fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains( @@ -173,7 +175,7 @@ public class FileWatcherTest extends ElasticsearchTestCase { fileWatcher.checkAndNotify(); assertThat(changes.notifications(), hasSize(0)); - new File(testDir, "test0.txt").delete(); + Files.delete(new File(testDir, "test0.txt").toPath()); touch(new File(testDir, "test2.txt")); touch(new File(testDir, "test4.txt")); fileWatcher.checkAndNotify(); @@ -187,8 +189,8 @@ public class FileWatcherTest extends ElasticsearchTestCase { changes.notifications().clear(); - new File(testDir, "test3.txt").delete(); - new File(testDir, "test4.txt").delete(); + Files.delete(new File(testDir, "test3.txt").toPath()); + Files.delete(new File(testDir, "test4.txt").toPath()); fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains( equalTo("onFileDeleted: test-dir/test3.txt"), @@ -197,7 +199,9 @@ public class FileWatcherTest extends ElasticsearchTestCase { changes.notifications().clear(); - deleteRecursively(testDir); + if (testDir.exists()) { + IOUtils.rm(testDir.toPath()); + } fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains( @@ -261,7 +265,10 @@ public class FileWatcherTest extends ElasticsearchTestCase { assertThat(changes.notifications(), hasSize(0)); // Delete a directory, check notifications for - deleteRecursively(new File(testDir, "first-level")); + Path path = new File(testDir, "first-level").toPath(); + if (Files.exists(path)) { + IOUtils.rm(path); + } fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains( equalTo("onFileDeleted: test-dir/first-level/file1.txt"), @@ -294,7 +301,9 @@ public class FileWatcherTest extends ElasticsearchTestCase { changes.notifications().clear(); - deleteRecursively(subDir); + if (subDir.exists()) { + IOUtils.rm(subDir.toPath()); + } touch(subDir); fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains( @@ -306,7 +315,7 @@ public class FileWatcherTest extends ElasticsearchTestCase { changes.notifications().clear(); - subDir.delete(); + Files.delete(subDir.toPath()); subDir.mkdir(); fileWatcher.checkAndNotify(); @@ -330,8 +339,8 @@ public class FileWatcherTest extends ElasticsearchTestCase { fileWatcher.init(); changes.notifications().clear(); - new File(testDir, "test0.txt").delete(); - new File(testDir, "test1.txt").delete(); + Files.delete(new File(testDir, "test0.txt").toPath()); + Files.delete(new File(testDir, "test1.txt").toPath()); fileWatcher.checkAndNotify(); assertThat(changes.notifications(), contains( equalTo("onFileDeleted: test-dir/test0.txt"),