honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CORE] Cut over to Path API for file deletion 

Today we use the File API for file deletion as well as recursive
directory deletions. This API returns a boolean if operations
are successful while hiding the actual reason why they failed.
The Path API throws and actual exception that might provide better
insights and debug information.

Closes #8366
This commit is contained in:
Simon Willnauer 2014-11-06 15:12:16 +01:00
parent 9ebce349e1
commit 95171e2bc2
34 changed files with 244 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev-tools/forbidden/all-signatures.txt
View File

@ -1,3 +1,5 @@
@defaultMessage Convert to URI
java.net.URL#getPath()
java.net.URL#getFile()
java.io.File#delete() @ use Files.delete for real exception, IOUtils.deleteFilesIgnoringExceptions if you dont care

2
src/main/java/org/elasticsearch/common/blobstore/BlobContainer.java
View File

@ -51,7 +51,7 @@ public interface BlobContainer {
*/
OutputStream createOutput(String blobName) throws IOException;
boolean deleteBlob(String blobName) throws IOException;
void deleteBlob(String blobName) throws IOException;
void deleteBlobsByPrefix(String blobNamePrefix) throws IOException;

4
src/main/java/org/elasticsearch/common/blobstore/BlobStore.java
View File

@ -18,6 +18,8 @@
*/
package org.elasticsearch.common.blobstore;
import java.io.IOException;
/**
*
*/
@ -25,7 +27,7 @@ public interface BlobStore {
BlobContainer blobContainer(BlobPath path);
void delete(BlobPath path);
void delete(BlobPath path) throws IOException;
void close();
}

10
src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java
View File

@ -30,6 +30,8 @@ import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.io.FileSystemUtils;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
/**
*
@ -65,8 +67,12 @@ public class FsBlobContainer extends AbstractBlobContainer {
return builder.immutableMap();
}
public boolean deleteBlob(String blobName) throws IOException {
return new File(path, blobName).delete();
@Override
public void deleteBlob(String blobName) throws IOException {
Path blobPath = new File(path, blobName).toPath();
if (Files.exists(blobPath)) {
Files.delete(blobPath);
}
}
@Override

6
src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.common.blobstore.fs;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
@ -30,6 +31,7 @@ import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import java.io.File;
import java.io.IOException;
/**
*
@ -74,8 +76,8 @@ public class FsBlobStore extends AbstractComponent implements BlobStore {
}
@Override
public void delete(BlobPath path) {
FileSystemUtils.deleteRecursively(buildPath(path));
public void delete(BlobPath path) throws IOException {
IOUtils.rm(buildPath(path).toPath());
}
@Override

2
src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java
View File

@ -73,7 +73,7 @@ public class URLBlobContainer extends AbstractBlobContainer {
* This operation is not supported by URLBlobContainer
*/
@Override
public boolean deleteBlob(String blobName) throws IOException {
public void deleteBlob(String blobName) throws IOException {
throw new UnsupportedOperationException("URL repository is read only");
}

5
src/main/java/org/elasticsearch/common/http/client/HttpDownloadHelper.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.common.http.client;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchTimeoutException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.unit.TimeValue;
import java.io.*;
@ -346,7 +347,7 @@ public class HttpDownloadHelper {
// Try to delete the garbage we'd otherwise leave
// behind.
IOUtils.closeWhileHandlingException(os, is);
dest.delete();
FileSystemUtils.deleteFilesIgnoringExceptions(dest.toPath());
} else {
IOUtils.close(os, is);
}
@ -385,7 +386,7 @@ public class HttpDownloadHelper {
} else {
IOUtils.closeWhileHandlingException(is, os);
if (dest != null && dest.exists()) {
dest.delete();
FileSystemUtils.deleteFilesIgnoringExceptions(dest.toPath());
}
}
}

100
src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
View File

@ -78,84 +78,34 @@ public class FileSystemUtils {
}
/**
* Deletes the given files recursively. if <tt>deleteRoots</tt> is set to <code>true</code>
* the given root files will be deleted as well. Otherwise only their content is deleted.
* Returns an array of {@link Path} build from the correspondent element
* in the input array using {@link java.io.File#toPath()}
* @param files the files to get paths for
*/
public static boolean deleteRecursively(File[] roots, boolean deleteRoots) {
boolean deleted = true;
for (File root : roots) {
deleted &= deleteRecursively(root, deleteRoots);
public static Path[] toPaths(File... files) {
Path[] paths = new Path[files.length];
for (int i = 0; i < files.length; i++) {
paths[i] = files[i].toPath();
}
return deleted;
return paths;
}
/**
* Deletes all subdirectories of the given roots recursively.
* Deletes all subdirectories in the given path recursively
* @throws java.lang.IllegalArgumentException if the given path is not a directory
*/
public static boolean deleteSubDirectories(File[] roots) {
boolean deleted = true;
for (File root : roots) {
if (root.isDirectory()) {
File[] files = root.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) {
return pathname.isDirectory();
}
});
deleted &= deleteRecursively(files, true);
}
}
return deleted;
}
/**
* Deletes the given files recursively including the given roots.
*/
public static boolean deleteRecursively(File... roots) {
return deleteRecursively(roots, true);
}
/**
* Delete the supplied {@link java.io.File} - for directories,
* recursively delete any nested directories or files as well.
*
* @param root the root <code>File</code> to delete
* @param deleteRoot whether or not to delete the root itself or just the content of the root.
* @return <code>true</code> if the <code>File</code> was deleted,
* otherwise <code>false</code>
*/
public static boolean deleteRecursively(File root, boolean deleteRoot) {
if (root != null && root.exists()) {
if (root.isDirectory()) {
File[] children = root.listFiles();
if (children != null) {
for (File aChildren : children) {
deleteRecursively(aChildren, true);
public static void deleteSubDirectories(Path... paths) throws IOException {
for (Path path : paths) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
for (Path subPath : stream) {
if (Files.isDirectory(subPath)) {
IOUtils.rm(subPath);
}
}
}
if (deleteRoot) {
return root.delete();
} else {
return true;
}
}
return false;
}
/**
* Ensure that any writes to the given file is written to the storage device that contains it.
* @param fileToSync the file to fsync
* @param isDir if true, the given file is a directory (we open for read and ignore IOExceptions,
* because not all file systems and operating systems allow to fsync on a directory)
*/
public static void syncFile(File fileToSync, boolean isDir) throws IOException {
IOUtils.fsync(fileToSync.toPath(), isDir);
}
/**
* Check that a directory exists, is a directory and is readable
@ -181,11 +131,19 @@ public class FileSystemUtils {
private FileSystemUtils() {}
public static void tryDeleteFile(File file) {
try {
file.delete();
} catch (SecurityException e1) {
// ignore
/**
* Temporary solution until LUCENE-6051 is fixed
* @see org.apache.lucene.util.IOUtils#deleteFilesIgnoringExceptions(java.nio.file.Path...)
*/
public static void deleteFilesIgnoringExceptions(Path... files) {
for (Path name : files) {
if (name != null) {
try {
Files.delete(name);
} catch (Throwable ignored) {
// ignore
}
}
}
}

7
src/main/java/org/elasticsearch/gateway/local/LocalGateway.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.gateway.local;
import com.carrotsearch.hppc.ObjectFloatOpenHashMap;
import com.carrotsearch.hppc.ObjectOpenHashSet;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.FailedNodeException;
import org.elasticsearch.cluster.*;
@ -197,7 +198,11 @@ public class LocalGateway extends AbstractLifecycleComponent<Gateway> implements
@Override
public void reset() throws Exception {
FileSystemUtils.deleteRecursively(nodeEnv.nodeDataLocations());
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.nodeDataLocations()));
} catch (Exception ex) {
logger.debug("failed to delete shard locations", ex);
}
}
@Override

26
src/main/java/org/elasticsearch/gateway/local/state/meta/LocalGatewayMetaState.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.gateway.local.state.meta;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.Version;
@ -49,6 +50,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.List;
import java.util.Map;
import java.util.Set;
@ -243,7 +245,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
if (!newMetaData.hasIndex(current.index())) {
logger.debug("[{}] deleting index that is no longer part of the metadata (indices: [{}])", current.index(), newMetaData.indices().keys());
if (nodeEnv.hasNodeFile()) {
FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index())));
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(current.index()))));
} catch (Exception ex) {
logger.debug("[{}] failed to delete index", ex, current.index());
}
}
try {
nodeIndexDeletedAction.nodeIndexStoreDeleted(event.state(), current.index(), event.state().nodes().localNodeId());
@ -280,7 +286,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
if (indexMetaData != null) {
if (danglingTimeout.millis() == 0) {
logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, timeout set to 0, deleting now", indexName);
FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(indexName)));
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(indexName))));
} catch (Exception ex) {
logger.debug("[{}] failed to delete dangling index", ex, indexName);
}
} else {
logger.info("[{}] dangling index, exists on local file system, but not in cluster metadata, scheduling to delete in [{}], auto import to cluster state [{}]", indexName, danglingTimeout, autoImportDangled);
danglingIndices.put(indexName, new DanglingIndex(indexName, threadPool.schedule(danglingTimeout, ThreadPool.Names.SAME, new RemoveDanglingIndex(indexName))));
@ -517,7 +527,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
if (!name.startsWith("metadata-")) {
continue;
}
stateFile.delete();
try {
Files.delete(stateFile.toPath());
} catch (Exception ex) {
logger.debug("failed to delete file " + stateFile, ex);
}
}
}
@ -576,7 +590,11 @@ public class LocalGatewayMetaState extends AbstractComponent implements ClusterS
return;
}
logger.warn("[{}] deleting dangling index", index);
FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(index)));
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(index))));
} catch (Exception ex) {
logger.debug("failed to delete dangling index", ex);
}
}
}
}

4
src/main/java/org/elasticsearch/gateway/local/state/meta/MetaDataStateFormat.java
View File

@ -208,7 +208,9 @@ public abstract class MetaDataStateFormat<T> {
if (file.getName().equals(fileName)) {
continue;
}
Files.delete(file.toPath());
if (Files.exists(file.toPath())) {
Files.delete(file.toPath());
}
}
}
}

8
src/main/java/org/elasticsearch/gateway/local/state/shards/LocalGatewayShardsState.java
View File

@ -37,6 +37,7 @@ import org.elasticsearch.gateway.local.state.meta.MetaDataStateFormat;
import org.elasticsearch.index.shard.ShardId;
import java.io.*;
import java.nio.file.Files;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
@ -317,7 +318,12 @@ public class LocalGatewayShardsState extends AbstractComponent implements Cluste
if (!name.startsWith("shards-")) {
continue;
}
stateFile.delete();
try {
Files.delete(stateFile.toPath());
} catch (Exception ex) {
logger.debug("Failed to delete state file {}", ex, stateFile);
}
}
}

7
src/main/java/org/elasticsearch/gateway/none/NoneGateway.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.gateway.none;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
@ -117,7 +118,11 @@ public class NoneGateway extends AbstractLifecycleComponent<Gateway> implements
if (!newMetaData.hasIndex(current.index())) {
logger.debug("[{}] deleting index that is no longer part of the metadata (indices: [{}])", current.index(), newMetaData.indices().keys());
if (nodeEnv.hasNodeFile()) {
FileSystemUtils.deleteRecursively(nodeEnv.indexLocations(new Index(current.index())));
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.indexLocations(new Index(current.index()))));
} catch (Exception ex) {
logger.debug("failed to delete shard locations", ex);
}
}
try {
nodeIndexDeletedAction.nodeIndexStoreDeleted(event.state(), current.index(), event.state().nodes().localNodeId());

7
src/main/java/org/elasticsearch/index/gateway/local/LocalIndexShardGateway.java
View File

@ -51,6 +51,7 @@ import org.elasticsearch.threadpool.ThreadPool;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Arrays;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
@ -290,7 +291,11 @@ public class LocalIndexShardGateway extends AbstractIndexShardComponent implemen
}
indexShard.performRecoveryFinalization(true);
recoveringTranslogFile.delete();
try {
Files.delete(recoveringTranslogFile.toPath());
} catch (Exception ex) {
logger.debug("Failed to delete recovering translog file {}", ex, recoveringTranslogFile);
}
for (final String type : typesToUpdate) {
final CountDownLatch latch = new CountDownLatch(1);

7
src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.store.fs;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
@ -81,7 +82,11 @@ public abstract class FsIndexStore extends AbstractIndexStore {
if (indexService.hasShard(shardId.id())) {
throw new ElasticsearchIllegalStateException(shardId + " allocated, can't be deleted");
}
FileSystemUtils.deleteRecursively(shardLocations(shardId));
try {
IOUtils.rm(FileSystemUtils.toPaths(shardLocations(shardId)));
} catch (Exception ex) {
logger.debug("failed to delete shard locations", ex);
}
}
public File[] shardLocations(ShardId shardId) {

7
src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java
View File

@ -41,6 +41,7 @@ import org.elasticsearch.index.translog.*;
import java.io.File;
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.nio.file.Files;
import java.util.Collections;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.locks.ReadWriteLock;
@ -215,9 +216,9 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
continue;
}
try {
file.delete();
} catch (Exception e) {
// ignore
Files.delete(file.toPath());
} catch (Exception ex) {
logger.debug("failed to delete " + file, ex);
}
}
}

12
src/main/java/org/elasticsearch/index/translog/fs/RafReference.java
View File

@ -19,11 +19,15 @@
package org.elasticsearch.index.translog.fs;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.io.FileSystemUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.util.concurrent.atomic.AtomicInteger;
/**
@ -70,12 +74,14 @@ public class RafReference {
if (refCount.decrementAndGet() <= 0) {
try {
raf.close();
if (delete) {
file.delete();
}
} catch (IOException e) {
// ignore
} finally {
if (delete) {
FileSystemUtils.deleteFilesIgnoringExceptions(file.toPath());
}
}
}
}
}

7
src/main/java/org/elasticsearch/indices/store/IndicesStore.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.indices.store;
import org.apache.lucene.store.StoreRateLimiting;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.*;
import org.elasticsearch.cluster.node.DiscoveryNode;
@ -323,7 +324,11 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
File[] shardLocations = nodeEnv.shardLocations(shardId);
if (FileSystemUtils.exists(shardLocations)) {
logger.debug("{} deleting shard that is no longer used", shardId);
FileSystemUtils.deleteRecursively(shardLocations);
try {
IOUtils.rm(FileSystemUtils.toPaths(shardLocations));
} catch (Exception ex) {
logger.debug("failed to delete shard locations", ex);
}
}
}
} else {

36
src/main/java/org/elasticsearch/plugins/PluginManager.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.plugins;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.ElasticsearchTimeoutException;
@ -32,6 +33,7 @@ import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.Index;
import org.elasticsearch.node.internal.InternalSettingsPreparer;
import javax.net.ssl.HttpsURLConnection;
@ -222,12 +224,20 @@ public class PluginManager {
// ignore
}
}
pluginFile.delete();
try {
Files.delete(pluginFile.toPath());
} catch (Exception ex) {
log("Failed to delete plugin file" + pluginFile + " " + ex);
}
}
if (FileSystemUtils.hasExtensions(extractLocation, ".java")) {
debug("Plugin installation assumed to be site plugin, but contains source code, aborting installation...");
FileSystemUtils.deleteRecursively(extractLocation);
try {
IOUtils.rm(extractLocation.toPath());
} catch(Exception ex) {
debug("Failed to remove site plugin from path " + extractLocation + " - " + ex.getMessage());
}
throw new IllegalArgumentException("Plugin installation assumed to be site plugin, but contains source code, aborting installation.");
}
@ -237,7 +247,9 @@ public class PluginManager {
if (binFile.exists() && binFile.isDirectory()) {
File toLocation = pluginHandle.binDir(environment);
debug("Found bin, moving to " + toLocation.getAbsolutePath());
FileSystemUtils.deleteRecursively(toLocation);
if (toLocation.exists()) {
IOUtils.rm(toLocation.toPath());
}
if (!binFile.renameTo(toLocation)) {
throw new IOException("Could not move ["+ binFile.getAbsolutePath() + "] to [" + toLocation.getAbsolutePath() + "]");
}
@ -294,27 +306,33 @@ public class PluginManager {
File pluginToDelete = pluginHandle.extractedDir(environment);
if (pluginToDelete.exists()) {
debug("Removing: " + pluginToDelete.getPath());
if (!FileSystemUtils.deleteRecursively(pluginToDelete, true)) {
try {
IOUtils.rm(pluginToDelete.toPath());
} catch (IOException ex){
throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " +
pluginToDelete.toString());
pluginToDelete.toString(), ex);
}
removed = true;
}
pluginToDelete = pluginHandle.distroFile(environment);
if (pluginToDelete.exists()) {
debug("Removing: " + pluginToDelete.getPath());
if (!pluginToDelete.delete()) {
try {
Files.delete(pluginToDelete.toPath());
} catch (Exception ex) {
throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " +
pluginToDelete.toString());
pluginToDelete.toString(), ex);
}
removed = true;
}
File binLocation = pluginHandle.binDir(environment);
if (binLocation.exists()) {
debug("Removing: " + binLocation.getPath());
if (!FileSystemUtils.deleteRecursively(binLocation)) {
try {
IOUtils.rm(binLocation.toPath());
} catch (IOException ex){
throw new IOException("Unable to remove " + pluginHandle.name + ". Check file permissions on " +
binLocation.toString());
binLocation.toString(), ex);
}
removed = true;
}

5
src/test/java/org/elasticsearch/benchmark/fs/FsAppendBenchmark.java
View File

@ -18,13 +18,16 @@
*/
package org.elasticsearch.benchmark.fs;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.StopWatch;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.unit.ByteSizeValue;
import java.io.File;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Paths;
import java.util.Random;
/**
@ -33,7 +36,7 @@ import java.util.Random;
public class FsAppendBenchmark {
public static void main(String[] args) throws Exception {
new File("work/test.log").delete();
FileSystemUtils.deleteFilesIgnoringExceptions(Paths.get("work/test.log"));
RandomAccessFile raf = new RandomAccessFile("work/test.log", "rw");
raf.setLength(0);

3
src/test/java/org/elasticsearch/cluster/allocation/ClusterRerouteTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.cluster.allocation;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteResponse;
@ -196,7 +197,7 @@ public class ClusterRerouteTests extends ElasticsearchIntegrationTest {
logger.info("--> deleting the shard data [{}] ", Arrays.toString(shardLocation));
assertThat(FileSystemUtils.exists(shardLocation), equalTo(true)); // verify again after cluster was shut down
assertThat(FileSystemUtils.deleteRecursively(shardLocation), equalTo(true));
IOUtils.rm(FileSystemUtils.toPaths(shardLocation));
logger.info("--> starting nodes back, will not allocate the shard since it has no data, but the index will be there");
node_1 = internalCluster().startNode(commonSettings);

3
src/test/java/org/elasticsearch/index/mapper/FileBasedMappingsTests.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.index.mapper;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.Files;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@ -98,7 +99,7 @@ public class FileBasedMappingsTests extends ElasticsearchTestCase {
assertEquals(ImmutableSet.of("f", "g", "h"), properties.keySet());
}
} finally {
FileSystemUtils.deleteRecursively(configDir);
IOUtils.rm(configDir.toPath());
}
}

7
src/test/java/org/elasticsearch/index/translog/fs/FsBufferedTranslogTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.translog.fs;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.translog.AbstractSimpleTranslogTests;
@ -26,6 +27,8 @@ import org.elasticsearch.index.translog.Translog;
import org.junit.AfterClass;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
/**
*
@ -49,7 +52,7 @@ public class FsBufferedTranslogTests extends AbstractSimpleTranslogTests {
}
@AfterClass
public static void cleanup() {
FileSystemUtils.deleteRecursively(new File("data/fs-buf-translog"), true);
public static void cleanup() throws IOException {
IOUtils.rm(Paths.get("data/fs-buf-translog"));
}
}

7
src/test/java/org/elasticsearch/index/translog/fs/FsSimpleTranslogTests.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.translog.fs;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.index.translog.AbstractSimpleTranslogTests;
@ -26,6 +27,8 @@ import org.elasticsearch.index.translog.Translog;
import org.junit.AfterClass;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
/**
*
@ -45,7 +48,7 @@ public class FsSimpleTranslogTests extends AbstractSimpleTranslogTests {
}
@AfterClass
public static void cleanup() {
FileSystemUtils.deleteRecursively(new File("data/fs-simple-translog"), true);
public static void cleanup() throws IOException {
IOUtils.rm(Paths.get("data/fs-simple-translog"));
}
}

17
src/test/java/org/elasticsearch/plugins/PluginManagerTests.java
View File

@ -20,6 +20,7 @@ package org.elasticsearch.plugins;
import com.google.common.base.Predicate;
import org.apache.http.impl.client.HttpClients;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchTimeoutException;
@ -46,6 +47,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.file.Paths;
import java.util.concurrent.TimeUnit;
import static org.elasticsearch.common.io.FileSystemUtilsTests.assertFileContent;
@ -65,12 +67,12 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
private static final String PLUGIN_DIR = "plugins";
@After
public void afterTest() {
public void afterTest() throws IOException {
deletePluginsFolder();
}
@Before
public void beforeTest() {
public void beforeTest() throws IOException {
deletePluginsFolder();
}
@ -123,8 +125,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
assertThat(toolFile.canExecute(), is(true));
} finally {
// we need to clean up the copied dirs
FileSystemUtils.deleteRecursively(pluginBinDir);
FileSystemUtils.deleteRecursively(pluginConfigDir);
IOUtils.rm(pluginBinDir.toPath(), pluginConfigDir.toPath());
}
}
@ -206,7 +207,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
assertFileContent(pluginConfigDir, "dir/subdir/testsubdir.txt.new", "version2\n");
} finally {
// we need to clean up the copied dirs
FileSystemUtils.deleteRecursively(pluginConfigDir);
IOUtils.rm(pluginConfigDir.toPath());
}
}
@ -230,7 +231,7 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
assertDirectoryExists(pluginBinDir);
} finally {
// we need to clean up the copied dirs
FileSystemUtils.deleteRecursively(pluginBinDir);
IOUtils.rm(pluginBinDir.toPath());
}
}
@ -463,8 +464,8 @@ public class PluginManagerTests extends ElasticsearchIntegrationTest {
return false;
}
private void deletePluginsFolder() {
FileSystemUtils.deleteRecursively(new File(PLUGIN_DIR));
private void deletePluginsFolder() throws IOException {
IOUtils.rm(Paths.get(PLUGIN_DIR));
}
@Test

5
src/test/java/org/elasticsearch/script/ScriptServiceTests.java
View File

@ -31,6 +31,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Map;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
@ -73,8 +74,8 @@ public class ScriptServiceTests extends ElasticsearchTestCase {
assertThat(compiledScript.compiled(), equalTo((Object) "compiled_test_file"));
logger.info("--> delete both files");
assertThat(testFileNoExt.delete(), equalTo(true));
assertThat(testFileWithExt.delete(), equalTo(true));
Files.delete(testFileNoExt.toPath());
Files.delete(testFileWithExt.toPath());
resourceWatcherService.notifyNow();
logger.info("--> verify that file with extension was correctly removed");

9
src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.snapshots;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.action.ListenableActionFuture;
@ -45,6 +46,7 @@ import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.cluster.metadata.SnapshotMetaData;
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.store.support.AbstractIndexStore;
@ -55,6 +57,7 @@ import org.elasticsearch.test.junit.annotations.TestLogging;
import org.junit.Test;
import java.io.File;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@ -732,8 +735,8 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
File testIndex1 = new File(indices, "test-idx-1");
File testIndex2 = new File(indices, "test-idx-2");
File testIndex2Shard0 = new File(testIndex2, "0");
new File(testIndex1, "snapshot-test-snap-1").delete();
new File(testIndex2Shard0, "snapshot-test-snap-1").delete();
FileSystemUtils.deleteFilesIgnoringExceptions(new File(testIndex1, "snapshot-test-snap-1").toPath());
FileSystemUtils.deleteFilesIgnoringExceptions(new File(testIndex2Shard0, "snapshot-test-snap-1").toPath());
logger.info("--> delete snapshot");
client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();
@ -768,7 +771,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
logger.info("--> delete index metadata and shard metadata");
File metadata = new File(repo, "metadata-test-snap-1");
assertThat(metadata.delete(), equalTo(true));
Files.delete(metadata.toPath());
logger.info("--> delete snapshot");
client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap-1").get();

4
src/test/java/org/elasticsearch/snapshots/mockstore/BlobContainerWrapper.java
View File

@ -58,8 +58,8 @@ public class BlobContainerWrapper implements BlobContainer {
}
@Override
public boolean deleteBlob(String blobName) throws IOException {
return delegate.deleteBlob(blobName);
public void deleteBlob(String blobName) throws IOException {
delegate.deleteBlob(blobName);
}
@Override

4
src/test/java/org/elasticsearch/snapshots/mockstore/BlobStoreWrapper.java
View File

@ -22,6 +22,8 @@ import org.elasticsearch.common.blobstore.BlobContainer;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import java.io.IOException;
/**
*
*/
@ -39,7 +41,7 @@ public class BlobStoreWrapper implements BlobStore {
}
@Override
public void delete(BlobPath path) {
public void delete(BlobPath path) throws IOException {
delegate.delete(path);
}

4
src/test/java/org/elasticsearch/snapshots/mockstore/MockRepository.java
View File

@ -262,9 +262,9 @@ public class MockRepository extends FsRepository {
}
@Override
public boolean deleteBlob(String blobName) throws IOException {
public void deleteBlob(String blobName) throws IOException {
maybeIOExceptionOrBlock(blobName);
return super.deleteBlob(blobName);
super.deleteBlob(blobName);
}
@Override

7
src/test/java/org/elasticsearch/stresstest/fullrestart/FullRestartStressTest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.stresstest.fullrestart;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.count.CountResponse;
@ -200,7 +201,11 @@ public class FullRestartStressTest {
File[] nodeDatas = ((InternalNode) node).injector().getInstance(NodeEnvironment.class).nodeDataLocations();
node.close();
if (clearNodeWork && !settings.get("gateway.type").equals("local")) {
FileSystemUtils.deleteRecursively(nodeDatas);
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeDatas));
} catch (Exception ex) {
logger.debug("failed to remove node data locations", ex);
}
}
}

8
src/test/java/org/elasticsearch/stresstest/rollingrestart/RollingRestartStressTest.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.stresstest.rollingrestart;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.count.CountResponse;
import org.elasticsearch.action.get.GetResponse;
@ -170,7 +171,12 @@ public class RollingRestartStressTest {
File[] nodeData = ((InternalNode) nodes[nodeIndex]).injector().getInstance(NodeEnvironment.class).nodeDataLocations();
nodes[nodeIndex].close();
if (clearNodeData) {
FileSystemUtils.deleteRecursively(nodeData);
try {
IOUtils.rm(FileSystemUtils.toPaths(nodeData));
} catch (Exception ex) {
logger.debug("Failed to delete node data directories", ex);
}
}
try {

20
src/test/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -99,6 +99,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.file.Path;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@ -164,7 +165,7 @@ public final class InternalTestCluster extends TestCluster {
/* sorted map to make traverse order reproducible, concurrent since we do checks on it not within a sync block */
private final NavigableMap<String, NodeAndClient> nodes = new TreeMap<>();
private final Set<File> dataDirToClean = new HashSet<>();
private final Set<Path> dataDirToClean = new HashSet<>();
private final String clusterName;
@ -788,7 +789,7 @@ public final class InternalTestCluster extends TestCluster {
if (callback.clearData(name)) {
NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, node);
if (nodeEnv.hasNodeFile()) {
FileSystemUtils.deleteRecursively(nodeEnv.nodeDataLocations());
IOUtils.rm(FileSystemUtils.toPaths(nodeEnv.nodeDataLocations()));
}
}
node = (InternalNode) nodeBuilder().settings(node.settings()).settings(newSettings).node();
@ -953,12 +954,17 @@ public final class InternalTestCluster extends TestCluster {
private void wipeDataDirectories() {
if (!dataDirToClean.isEmpty()) {
boolean deleted = false;
try {
deleted = FileSystemUtils.deleteSubDirectories(dataDirToClean.toArray(new File[dataDirToClean.size()]));
for (Path path : dataDirToClean) {
try {
FileSystemUtils.deleteSubDirectories(path);
logger.info("Successfully wiped data directory for node location: {}", path);
} catch (IOException e) {
logger.info("Failed to wipe data directory for node location: {}", path);
}
}
} finally {
logger.info("Wipe data directory for all nodes locations: {} success: {}", this.dataDirToClean, deleted);
this.dataDirToClean.clear();
dataDirToClean.clear();
}
}
}
@ -1399,7 +1405,7 @@ public final class InternalTestCluster extends TestCluster {
assert !nodeAndClient.node().isClosed();
NodeEnvironment nodeEnv = getInstanceFromNode(NodeEnvironment.class, nodeAndClient.node);
if (nodeEnv.hasNodeFile()) {
dataDirToClean.addAll(Arrays.asList(nodeEnv.nodeDataLocations()));
dataDirToClean.addAll(Arrays.asList(FileSystemUtils.toPaths(nodeEnv.nodeDataLocations())));
}
nodes.put(nodeAndClient.name, nodeAndClient);
applyDisruptionSchemeToNode(nodeAndClient);

35
src/test/java/org/elasticsearch/watcher/FileWatcherTest.java
View File

@ -19,17 +19,19 @@
package org.elasticsearch.watcher;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import org.apache.lucene.util.IOUtils;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.io.Files.*;
import static org.elasticsearch.common.io.FileSystemUtils.deleteRecursively;
import static org.hamcrest.Matchers.*;
/**
@ -114,7 +116,7 @@ public class FileWatcherTest extends ElasticsearchTestCase {
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), hasSize(0));
testFile.delete();
Files.delete(testFile.toPath());
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(equalTo("onFileDeleted: test.txt")));
@ -160,8 +162,8 @@ public class FileWatcherTest extends ElasticsearchTestCase {
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), hasSize(0));
new File(testDir, "test1.txt").delete();
new File(testDir, "test2.txt").delete();
Files.delete(new File(testDir, "test1.txt").toPath());
Files.delete(new File(testDir, "test2.txt").toPath());
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(
@ -173,7 +175,7 @@ public class FileWatcherTest extends ElasticsearchTestCase {
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), hasSize(0));
new File(testDir, "test0.txt").delete();
Files.delete(new File(testDir, "test0.txt").toPath());
touch(new File(testDir, "test2.txt"));
touch(new File(testDir, "test4.txt"));
fileWatcher.checkAndNotify();
@ -187,8 +189,8 @@ public class FileWatcherTest extends ElasticsearchTestCase {
changes.notifications().clear();
new File(testDir, "test3.txt").delete();
new File(testDir, "test4.txt").delete();
Files.delete(new File(testDir, "test3.txt").toPath());
Files.delete(new File(testDir, "test4.txt").toPath());
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(
equalTo("onFileDeleted: test-dir/test3.txt"),
@ -197,7 +199,9 @@ public class FileWatcherTest extends ElasticsearchTestCase {
changes.notifications().clear();
deleteRecursively(testDir);
if (testDir.exists()) {
IOUtils.rm(testDir.toPath());
}
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(
@ -261,7 +265,10 @@ public class FileWatcherTest extends ElasticsearchTestCase {
assertThat(changes.notifications(), hasSize(0));
// Delete a directory, check notifications for
deleteRecursively(new File(testDir, "first-level"));
Path path = new File(testDir, "first-level").toPath();
if (Files.exists(path)) {
IOUtils.rm(path);
}
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(
equalTo("onFileDeleted: test-dir/first-level/file1.txt"),
@ -294,7 +301,9 @@ public class FileWatcherTest extends ElasticsearchTestCase {
changes.notifications().clear();
deleteRecursively(subDir);
if (subDir.exists()) {
IOUtils.rm(subDir.toPath());
}
touch(subDir);
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(
@ -306,7 +315,7 @@ public class FileWatcherTest extends ElasticsearchTestCase {
changes.notifications().clear();
subDir.delete();
Files.delete(subDir.toPath());
subDir.mkdir();
fileWatcher.checkAndNotify();
@ -330,8 +339,8 @@ public class FileWatcherTest extends ElasticsearchTestCase {
fileWatcher.init();
changes.notifications().clear();
new File(testDir, "test0.txt").delete();
new File(testDir, "test1.txt").delete();
Files.delete(new File(testDir, "test0.txt").toPath());
Files.delete(new File(testDir, "test1.txt").toPath());
fileWatcher.checkAndNotify();
assertThat(changes.notifications(), contains(
equalTo("onFileDeleted: test-dir/test0.txt"),