diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java index 8e1acd55c4c..951d5e132d8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java @@ -47,7 +47,7 @@ public class FsBlobStore extends AbstractComponent implements BlobStore { super(settings); this.path = path; if (!path.exists()) { - boolean b = path.mkdirs(); + boolean b = FileSystemUtils.mkdirs(path); if (!b) { throw new BlobStoreException("Failed to create directory at [" + path + "]"); } @@ -89,7 +89,7 @@ public class FsBlobStore extends AbstractComponent implements BlobStore { private synchronized File buildAndCreate(BlobPath path) { File f = buildPath(path); - f.mkdirs(); + FileSystemUtils.mkdirs(f); return f; } diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java b/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java index 312e2a05efc..50eeb0c4f6d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java @@ -19,6 +19,10 @@ package org.elasticsearch.common.io; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.unit.TimeValue; + import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -34,6 +38,34 @@ import java.util.List; */ public class FileSystemUtils { + private static ESLogger logger = ESLoggerFactory.getLogger(FileSystemUtils.class.getName()); + + private static final long mkdirsStallTimeout = TimeValue.timeValueMinutes(5).millis(); + private static final Object mkdirsMutex = new Object(); + private static volatile Thread mkdirsThread; + private static volatile long mkdirsStartTime; + + public static boolean mkdirs(File dir) { + synchronized (mkdirsMutex) { + try { + mkdirsThread = Thread.currentThread(); + mkdirsStartTime = System.currentTimeMillis(); + return dir.mkdirs(); + } finally { + mkdirsThread = null; + } + } + } + + public static void checkMkdirsStall(long currentTime) { + Thread mkdirsThread1 = mkdirsThread; + long stallTime = currentTime - mkdirsStartTime; + if (mkdirsThread1 != null && (stallTime > mkdirsStallTimeout)) { + logger.error("mkdirs stalled for {} on {}, trying to interrupt", new TimeValue(stallTime), mkdirsThread1.getName()); + mkdirsThread1.interrupt(); // try and interrupt it... + } + } + public static int maxOpenFiles(File testDir) { boolean dirCreated = false; if (!testDir.exists()) { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 6b7d4b66106..f7bf85f32c5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -25,6 +25,7 @@ import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; @@ -60,7 +61,7 @@ public class NodeEnvironment extends AbstractComponent { for (int i = 0; i < 50; i++) { dir = new File(new File(environment.dataWithClusterFile(), "nodes"), Integer.toString(i)); if (!dir.exists()) { - dir.mkdirs(); + FileSystemUtils.mkdirs(dir); } logger.trace("obtaining node lock on {} ...", dir.getAbsolutePath()); try { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java index 9a51289c9ed..3159bba5a72 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java @@ -344,7 +344,7 @@ public class LocalGateway extends AbstractLifecycleComponent implements } else { // create the location where the state will be stored this.location = new File(nodeEnv.nodeDataLocation(), "_state"); - this.location.mkdirs(); + FileSystemUtils.mkdirs(this.location); if (clusterService.localNode().masterNode()) { try { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java index ce6c14be8e5..2b56139ee0b 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java @@ -26,6 +26,7 @@ import org.apache.lucene.store.MMapDirectory; import org.elasticsearch.cache.memory.ByteBufferCache; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.lucene.store.SwitchDirectory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; @@ -50,7 +51,7 @@ public class MmapFsStore extends FsStore { super(shardId, indexSettings, indexStore); LockFactory lockFactory = buildLockFactory(); File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); - location.mkdirs(); + FileSystemUtils.mkdirs(location); this.fsDirectory = new MMapDirectory(location, lockFactory); boolean suggestUseCompoundFile; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java index 603a8669f02..420caa6286d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java @@ -26,6 +26,7 @@ import org.apache.lucene.store.NIOFSDirectory; import org.elasticsearch.cache.memory.ByteBufferCache; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.lucene.store.SwitchDirectory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; @@ -50,7 +51,7 @@ public class NioFsStore extends FsStore { super(shardId, indexSettings, indexStore); LockFactory lockFactory = buildLockFactory(); File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); - location.mkdirs(); + FileSystemUtils.mkdirs(location); this.fsDirectory = new NIOFSDirectory(location, lockFactory); boolean suggestUseCompoundFile; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java index 4e9eeec53f4..9a9511314b8 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java @@ -26,6 +26,7 @@ import org.apache.lucene.store.SimpleFSDirectory; import org.elasticsearch.cache.memory.ByteBufferCache; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.lucene.store.SwitchDirectory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.settings.IndexSettings; @@ -50,7 +51,7 @@ public class SimpleFsStore extends FsStore { super(shardId, indexSettings, indexStore); LockFactory lockFactory = buildLockFactory(); File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId); - location.mkdirs(); + FileSystemUtils.mkdirs(location); this.fsDirectory = new SimpleFSDirectory(location, lockFactory); boolean suggestUseCompoundFile; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java index 820affef593..a9617b18ebd 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.translog.fs; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.CachedStreamOutput; import org.elasticsearch.common.settings.Settings; @@ -50,13 +51,13 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog @Inject public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) { super(shardId, indexSettings); this.location = new File(nodeEnv.shardLocation(shardId), "translog"); - this.location.mkdirs(); + FileSystemUtils.mkdirs(this.location); } public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, File location) { super(shardId, indexSettings); this.location = location; - this.location.mkdirs(); + FileSystemUtils.mkdirs(this.location); } public File location() { diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/monitor/dump/SimpleDumpGenerator.java b/modules/elasticsearch/src/main/java/org/elasticsearch/monitor/dump/SimpleDumpGenerator.java index 42345add792..6d5b6038e07 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/monitor/dump/SimpleDumpGenerator.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/monitor/dump/SimpleDumpGenerator.java @@ -22,6 +22,7 @@ package org.elasticsearch.monitor.dump; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableMap; +import org.elasticsearch.common.io.FileSystemUtils; import java.io.File; import java.io.FileNotFoundException; @@ -57,7 +58,7 @@ public class SimpleDumpGenerator implements DumpGenerator { fileName += localNode.id() + "-"; } File file = new File(dumpLocation, fileName + cause + "-" + timestamp); - file.mkdirs(); + FileSystemUtils.mkdirs(file); SimpleDump dump; try { dump = new SimpleDump(System.currentTimeMillis(), cause, context, file); diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java b/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java index b4a132f0121..f89f324d52e 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java @@ -192,7 +192,7 @@ public class PluginManager { } } File target = new File(extractLocation, zipName); - target.getParentFile().mkdirs(); + FileSystemUtils.mkdirs(target.getParentFile()); Streams.copy(zipFile.getInputStream(zipEntry), new FileOutputStream(target)); } } catch (Exception e) { @@ -216,7 +216,7 @@ public class PluginManager { File site = new File(extractLocation, "_site"); File tmpLocation = new File(environment.pluginsFile(), name + ".tmp"); extractLocation.renameTo(tmpLocation); - extractLocation.mkdirs(); + FileSystemUtils.mkdirs(extractLocation); tmpLocation.renameTo(site); } } @@ -239,7 +239,7 @@ public class PluginManager { Tuple initialSettings = InternalSettingsPerparer.prepareSettings(EMPTY_SETTINGS, true); if (!initialSettings.v2().pluginsFile().exists()) { - initialSettings.v2().pluginsFile().mkdirs(); + FileSystemUtils.mkdirs(initialSettings.v2().pluginsFile()); } String url = null; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 2c2e92431d1..66b0394bdb7 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.collect.ImmutableMap; import org.elasticsearch.common.collect.Maps; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.ImmutableSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; @@ -279,6 +280,11 @@ public class ThreadPool extends AbstractComponent { running = false; return; } + try { + FileSystemUtils.checkMkdirsStall(estimatedTimeInMillis); + } catch (Exception e) { + // ignore + } } } }