honeymoose/OpenSearch
1
0
Fork 0
mirror of https://github.com/honeymoose/OpenSearch.git synced 2025-02-24 22:09:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

File#mkdirs gets stuck, might be concurrency issue, closes #1147.

Browse Source
This commit is contained in:
kimchy 2011-07-22 02:08:11 +03:00
parent d4d227ba6b
commit a7190ea8a3
11 changed files with 57 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
modules/elasticsearch/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobStore.java
View File

@ -47,7 +47,7 @@ public class FsBlobStore extends AbstractComponent implements BlobStore {
super(settings);
this.path = path;
if (!path.exists()) {
boolean b = path.mkdirs();
boolean b = FileSystemUtils.mkdirs(path);
if (!b) {
throw new BlobStoreException("Failed to create directory at [" + path + "]");
}
@ -89,7 +89,7 @@ public class FsBlobStore extends AbstractComponent implements BlobStore {
private synchronized File buildAndCreate(BlobPath path) {
File f = buildPath(path);
f.mkdirs();
FileSystemUtils.mkdirs(f);
return f;
}

32
modules/elasticsearch/src/main/java/org/elasticsearch/common/io/FileSystemUtils.java
View File

@ -19,6 +19,10 @@
package org.elasticsearch.common.io;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.logging.ESLoggerFactory;
import org.elasticsearch.common.unit.TimeValue;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@ -34,6 +38,34 @@ import java.util.List;
*/
public class FileSystemUtils {
private static ESLogger logger = ESLoggerFactory.getLogger(FileSystemUtils.class.getName());
private static final long mkdirsStallTimeout = TimeValue.timeValueMinutes(5).millis();
private static final Object mkdirsMutex = new Object();
private static volatile Thread mkdirsThread;
private static volatile long mkdirsStartTime;
public static boolean mkdirs(File dir) {
synchronized (mkdirsMutex) {
try {
mkdirsThread = Thread.currentThread();
mkdirsStartTime = System.currentTimeMillis();
return dir.mkdirs();
} finally {
mkdirsThread = null;
}
}
}
public static void checkMkdirsStall(long currentTime) {
Thread mkdirsThread1 = mkdirsThread;
long stallTime = currentTime - mkdirsStartTime;
if (mkdirsThread1 != null && (stallTime > mkdirsStallTimeout)) {
logger.error("mkdirs stalled for {} on {}, trying to interrupt", new TimeValue(stallTime), mkdirsThread1.getName());
mkdirsThread1.interrupt(); // try and interrupt it...
}
}
public static int maxOpenFiles(File testDir) {
boolean dirCreated = false;
if (!testDir.exists()) {

3
modules/elasticsearch/src/main/java/org/elasticsearch/env/NodeEnvironment.java vendored
View File

@ -25,6 +25,7 @@ import org.elasticsearch.ElasticSearchIllegalStateException;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
@ -60,7 +61,7 @@ public class NodeEnvironment extends AbstractComponent {
for (int i = 0; i < 50; i++) {
dir = new File(new File(environment.dataWithClusterFile(), "nodes"), Integer.toString(i));
if (!dir.exists()) {
dir.mkdirs();
FileSystemUtils.mkdirs(dir);
}
logger.trace("obtaining node lock on {} ...", dir.getAbsolutePath());
try {

2
modules/elasticsearch/src/main/java/org/elasticsearch/gateway/local/LocalGateway.java
View File

@ -344,7 +344,7 @@ public class LocalGateway extends AbstractLifecycleComponent<Gateway> implements
} else {
// create the location where the state will be stored
this.location = new File(nodeEnv.nodeDataLocation(), "_state");
this.location.mkdirs();
FileSystemUtils.mkdirs(this.location);
if (clusterService.localNode().masterNode()) {
try {

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.store.MMapDirectory;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
@ -50,7 +51,7 @@ public class MmapFsStore extends FsStore {
super(shardId, indexSettings, indexStore);
LockFactory lockFactory = buildLockFactory();
File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
location.mkdirs();
FileSystemUtils.mkdirs(location);
this.fsDirectory = new MMapDirectory(location, lockFactory);
boolean suggestUseCompoundFile;

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.store.NIOFSDirectory;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
@ -50,7 +51,7 @@ public class NioFsStore extends FsStore {
super(shardId, indexSettings, indexStore);
LockFactory lockFactory = buildLockFactory();
File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
location.mkdirs();
FileSystemUtils.mkdirs(location);
this.fsDirectory = new NIOFSDirectory(location, lockFactory);
boolean suggestUseCompoundFile;

3
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java
View File

@ -26,6 +26,7 @@ import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
@ -50,7 +51,7 @@ public class SimpleFsStore extends FsStore {
super(shardId, indexSettings, indexStore);
LockFactory lockFactory = buildLockFactory();
File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
location.mkdirs();
FileSystemUtils.mkdirs(location);
this.fsDirectory = new SimpleFSDirectory(location, lockFactory);
boolean suggestUseCompoundFile;

5
modules/elasticsearch/src/main/java/org/elasticsearch/index/translog/fs/FsTranslog.java
View File

@ -20,6 +20,7 @@
package org.elasticsearch.index.translog.fs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.CachedStreamOutput;
import org.elasticsearch.common.settings.Settings;
@ -50,13 +51,13 @@ public class FsTranslog extends AbstractIndexShardComponent implements Translog
@Inject public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) {
super(shardId, indexSettings);
this.location = new File(nodeEnv.shardLocation(shardId), "translog");
this.location.mkdirs();
FileSystemUtils.mkdirs(this.location);
}
public FsTranslog(ShardId shardId, @IndexSettings Settings indexSettings, File location) {
super(shardId, indexSettings);
this.location = location;
this.location.mkdirs();
FileSystemUtils.mkdirs(this.location);
}
public File location() {

3
modules/elasticsearch/src/main/java/org/elasticsearch/monitor/dump/SimpleDumpGenerator.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.monitor.dump;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.io.FileSystemUtils;
import java.io.File;
import java.io.FileNotFoundException;
@ -57,7 +58,7 @@ public class SimpleDumpGenerator implements DumpGenerator {
fileName += localNode.id() + "-";
}
File file = new File(dumpLocation, fileName + cause + "-" + timestamp);
file.mkdirs();
FileSystemUtils.mkdirs(file);
SimpleDump dump;
try {
dump = new SimpleDump(System.currentTimeMillis(), cause, context, file);

6
modules/elasticsearch/src/main/java/org/elasticsearch/plugins/PluginManager.java
View File

@ -192,7 +192,7 @@ public class PluginManager {
}
}
File target = new File(extractLocation, zipName);
target.getParentFile().mkdirs();
FileSystemUtils.mkdirs(target.getParentFile());
Streams.copy(zipFile.getInputStream(zipEntry), new FileOutputStream(target));
}
} catch (Exception e) {
@ -216,7 +216,7 @@ public class PluginManager {
File site = new File(extractLocation, "_site");
File tmpLocation = new File(environment.pluginsFile(), name + ".tmp");
extractLocation.renameTo(tmpLocation);
extractLocation.mkdirs();
FileSystemUtils.mkdirs(extractLocation);
tmpLocation.renameTo(site);
}
}
@ -239,7 +239,7 @@ public class PluginManager {
Tuple<Settings, Environment> initialSettings = InternalSettingsPerparer.prepareSettings(EMPTY_SETTINGS, true);
if (!initialSettings.v2().pluginsFile().exists()) {
initialSettings.v2().pluginsFile().mkdirs();
FileSystemUtils.mkdirs(initialSettings.v2().pluginsFile());
}
String url = null;

6
modules/elasticsearch/src/main/java/org/elasticsearch/threadpool/ThreadPool.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.component.AbstractComponent;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.SizeValue;
@ -279,6 +280,11 @@ public class ThreadPool extends AbstractComponent {
running = false;
return;
}
try {
FileSystemUtils.checkMkdirsStall(estimatedTimeInMillis);
} catch (Exception e) {
// ignore
}
}
}
}