File#mkdirs gets stuck, might be concurrency issue, closes #1147.

modules/elasticsearch/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java

@@ -47,7 +47,7 @@ public class FsBlobStore extends AbstractComponent implements BlobStore {
         super(settings);
         this.path = path;
         if (!path.exists()) {
-            boolean b = path.mkdirs();
+            boolean b = FileSystemUtils.mkdirs(path);
             if (!b) {
                 throw new BlobStoreException("Failed to create directory at [" + path + "]");
             }
@@ -89,7 +89,7 @@ public class FsBlobStore extends AbstractComponent implements BlobStore {
 
     private synchronized File buildAndCreate(BlobPath path) {
         File f = buildPath(path);
-        f.mkdirs();
+        FileSystemUtils.mkdirs(f);
         return f;
     }
 

modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java

@@ -19,6 +19,10 @@
 
 package org.elasticsearch.common.io;
 
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.unit.TimeValue;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
@@ -34,6 +38,34 @@ import java.util.List;
  */
 public class FileSystemUtils {
 
+    private static ESLogger logger = ESLoggerFactory.getLogger(FileSystemUtils.class.getName());
+
+    private static final long mkdirsStallTimeout = TimeValue.timeValueMinutes(5).millis();
+    private static final Object mkdirsMutex = new Object();
+    private static volatile Thread mkdirsThread;
+    private static volatile long mkdirsStartTime;
+
+    public static boolean mkdirs(File dir) {
+        synchronized (mkdirsMutex) {
+            try {
+                mkdirsThread = Thread.currentThread();
+                mkdirsStartTime = System.currentTimeMillis();
+                return dir.mkdirs();
+            } finally {
+                mkdirsThread = null;
+            }
+        }
+    }
+
+    public static void checkMkdirsStall(long currentTime) {
+        Thread mkdirsThread1 = mkdirsThread;
+        long stallTime = currentTime - mkdirsStartTime;
+        if (mkdirsThread1 != null && (stallTime > mkdirsStallTimeout)) {
+            logger.error("mkdirs stalled for {} on {}, trying to interrupt", new TimeValue(stallTime), mkdirsThread1.getName());
+            mkdirsThread1.interrupt(); // try and interrupt it...
+        }
+    }
+
     public static int maxOpenFiles(File testDir) {
         boolean dirCreated = false;
         if (!testDir.exists()) {

modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -25,6 +25,7 @@ import org.elasticsearch.ElasticSearchIllegalStateException;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.shard.ShardId;
@@ -60,7 +61,7 @@ public class NodeEnvironment extends AbstractComponent {
         for (int i = 0; i < 50; i++) {
             dir = new File(new File(environment.dataWithClusterFile(), "nodes"), Integer.toString(i));
             if (!dir.exists()) {
-                dir.mkdirs();
+                FileSystemUtils.mkdirs(dir);
             }
             logger.trace("obtaining node lock on {} ...", dir.getAbsolutePath());
             try {

modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java

@@ -344,7 +344,7 @@ public class LocalGateway extends AbstractLifecycleComponent<Gateway> implements
         } else {
             // create the location where the state will be stored
             this.location = new File(nodeEnv.nodeDataLocation(), "_state");
-            this.location.mkdirs();
+            FileSystemUtils.mkdirs(this.location);
 
             if (clusterService.localNode().masterNode()) {
                 try {

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java

@@ -26,6 +26,7 @@ import org.apache.lucene.store.MMapDirectory;
 import org.elasticsearch.cache.memory.ByteBufferCache;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -50,7 +51,7 @@ public class MmapFsStore extends FsStore {
         super(shardId, indexSettings, indexStore);
         LockFactory lockFactory = buildLockFactory();
         File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
-        location.mkdirs();
+        FileSystemUtils.mkdirs(location);
         this.fsDirectory = new MMapDirectory(location, lockFactory);
 
         boolean suggestUseCompoundFile;

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java

@@ -26,6 +26,7 @@ import org.apache.lucene.store.NIOFSDirectory;
 import org.elasticsearch.cache.memory.ByteBufferCache;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -50,7 +51,7 @@ public class NioFsStore extends FsStore {
         super(shardId, indexSettings, indexStore);
         LockFactory lockFactory = buildLockFactory();
         File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
-        location.mkdirs();
+        FileSystemUtils.mkdirs(location);
         this.fsDirectory = new NIOFSDirectory(location, lockFactory);
 
         boolean suggestUseCompoundFile;

modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java

@@ -26,6 +26,7 @@ import org.apache.lucene.store.SimpleFSDirectory;
 import org.elasticsearch.cache.memory.ByteBufferCache;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.lucene.store.SwitchDirectory;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.settings.IndexSettings;
@@ -50,7 +51,7 @@ public class SimpleFsStore extends FsStore {
         super(shardId, indexSettings, indexStore);
         LockFactory lockFactory = buildLockFactory();
         File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
-        location.mkdirs();
+        FileSystemUtils.mkdirs(location);
         this.fsDirectory = new SimpleFSDirectory(location, lockFactory);
 
         boolean suggestUseCompoundFile;

modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.translog.fs;
 
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.CachedStreamOutput;
 import org.elasticsearch.common.settings.Settings;
@@ -50,13 +51,13 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
     @Inject public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) {
         super(shardId, indexSettings);
         this.location = new File(nodeEnv.shardLocation(shardId), "translog");
-        this.location.mkdirs();
+        FileSystemUtils.mkdirs(this.location);
     }
 
     public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, File location) {
         super(shardId, indexSettings);
         this.location = location;
-        this.location.mkdirs();
+        FileSystemUtils.mkdirs(this.location);
     }
 
     public File location() {

modules/elasticsearch/src/main/java/org/elasticsearch/monitor/dump/SimpleDumpGenerator.java

@@ -22,6 +22,7 @@ package org.elasticsearch.monitor.dump;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.collect.ImmutableMap;
+import org.elasticsearch.common.io.FileSystemUtils;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -57,7 +58,7 @@ public class SimpleDumpGenerator implements DumpGenerator {
             fileName += localNode.id() + "-";
         }
         File file = new File(dumpLocation, fileName + cause + "-" + timestamp);
-        file.mkdirs();
+        FileSystemUtils.mkdirs(file);
         SimpleDump dump;
         try {
             dump = new SimpleDump(System.currentTimeMillis(), cause, context, file);

modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java

@@ -192,7 +192,7 @@ public class PluginManager {
                     }
                 }
                 File target = new File(extractLocation, zipName);
-                target.getParentFile().mkdirs();
+                FileSystemUtils.mkdirs(target.getParentFile());
                 Streams.copy(zipFile.getInputStream(zipEntry), new FileOutputStream(target));
             }
         } catch (Exception e) {
@@ -216,7 +216,7 @@ public class PluginManager {
                 File site = new File(extractLocation, "_site");
                 File tmpLocation = new File(environment.pluginsFile(), name + ".tmp");
                 extractLocation.renameTo(tmpLocation);
-                extractLocation.mkdirs();
+                FileSystemUtils.mkdirs(extractLocation);
                 tmpLocation.renameTo(site);
             }
         }
@@ -239,7 +239,7 @@ public class PluginManager {
         Tuple<Settings, Environment> initialSettings = InternalSettingsPerparer.prepareSettings(EMPTY_SETTINGS, true);
 
         if (!initialSettings.v2().pluginsFile().exists()) {
-            initialSettings.v2().pluginsFile().mkdirs();
+            FileSystemUtils.mkdirs(initialSettings.v2().pluginsFile());
         }
 
         String url = null;

modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java

@@ -25,6 +25,7 @@ import org.elasticsearch.common.collect.ImmutableMap;
 import org.elasticsearch.common.collect.Maps;
 import org.elasticsearch.common.component.AbstractComponent;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.io.FileSystemUtils;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.SizeValue;
@@ -279,6 +280,11 @@ public class ThreadPool extends AbstractComponent {
                     running = false;
                     return;
                 }
+                try {
+                    FileSystemUtils.checkMkdirsStall(estimatedTimeInMillis);
+                } catch (Exception e) {
+                    // ignore
+                }
             }
         }
     }