honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improve handling of memory caching with file system, only force compound file when really needed (when an extension that exists within the compound file is part of the memory cached extensions)

This commit is contained in:
kimchy 2010-07-06 18:21:36 +03:00
parent ac01bb86e8
commit 267016758d
5 changed files with 49 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsStore.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.*;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.ImmutableSet;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
@ -73,7 +74,7 @@ public abstract class FsStore extends AbstractStore {
return lockFactory;
}
protected SwitchDirectory buildSwitchDirectoryIfNeeded(Directory fsDirectory, ByteBufferCache byteBufferCache) {
protected Tuple<SwitchDirectory, Boolean> buildSwitchDirectoryIfNeeded(Directory fsDirectory, ByteBufferCache byteBufferCache) {
boolean cache = componentSettings.getAsBoolean("memory.enabled", false);
if (!cache) {
return null;
@ -81,6 +82,17 @@ public abstract class FsStore extends AbstractStore {
Directory memDir = new ByteBufferDirectory(byteBufferCache);
// see http://lucene.apache.org/java/3_0_1/fileformats.html
String[] primaryExtensions = componentSettings.getAsArray("memory.extensions", new String[]{"", "del", "gen"});
return new SwitchDirectory(ImmutableSet.copyOf(primaryExtensions), memDir, fsDirectory, true);
if (primaryExtensions == null || primaryExtensions.length == 0) {
return null;
}
Boolean forceUseCompound = null;
for (String extension : primaryExtensions) {
if (!("".equals(extension) || "del".equals(extension) || "gen".equals(extension))) {
// caching internal CFS extension, don't use compound file extension
forceUseCompound = false;
}
}
return new Tuple<SwitchDirectory, Boolean>(new SwitchDirectory(ImmutableSet.copyOf(primaryExtensions), memDir, fsDirectory, true), forceUseCompound);
}
}

14
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MMapDirectory;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
@ -52,16 +53,21 @@ public class MmapFsStore extends FsStore {
location.mkdirs();
this.fsDirectory = new MMapDirectory(location, lockFactory);
SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
boolean suggestUseCompoundFile;
Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
if (switchDirectory != null) {
suggestUseCompoundFile = false;
logger.debug("Using [mmap_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.primaryExtensions());
directory = wrapDirectory(switchDirectory);
suggestUseCompoundFile = true;
if (switchDirectory.v2() != null) {
suggestUseCompoundFile = switchDirectory.v2();
}
logger.debug("Using [mmap_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.v1().primaryExtensions());
directory = wrapDirectory(switchDirectory.v1());
} else {
suggestUseCompoundFile = true;
directory = wrapDirectory(fsDirectory);
logger.debug("Using [mmap_fs] Store with path [{}]", fsDirectory.getFile());
}
this.suggestUseCompoundFile = suggestUseCompoundFile;
}
@Override public FSDirectory fsDirectory() {

14
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.NIOFSDirectory;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
@ -52,16 +53,21 @@ public class NioFsStore extends FsStore {
location.mkdirs();
this.fsDirectory = new NIOFSDirectory(location, lockFactory);
SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
boolean suggestUseCompoundFile;
Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
if (switchDirectory != null) {
suggestUseCompoundFile = false;
logger.debug("Using [nio_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.primaryExtensions());
directory = wrapDirectory(switchDirectory);
suggestUseCompoundFile = true;
if (switchDirectory.v2() != null) {
suggestUseCompoundFile = switchDirectory.v2();
}
logger.debug("Using [nio_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.v1().primaryExtensions());
directory = wrapDirectory(switchDirectory.v1());
} else {
suggestUseCompoundFile = true;
directory = wrapDirectory(fsDirectory);
logger.debug("Using [nio_fs] Store with path [{}]", fsDirectory.getFile());
}
this.suggestUseCompoundFile = suggestUseCompoundFile;
}
@Override public FSDirectory fsDirectory() {

14
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java
View File

@ -24,6 +24,7 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.cache.memory.ByteBufferCache;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.lucene.store.SwitchDirectory;
import org.elasticsearch.common.settings.Settings;
@ -52,16 +53,21 @@ public class SimpleFsStore extends FsStore {
location.mkdirs();
this.fsDirectory = new SimpleFSDirectory(location, lockFactory);
SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
boolean suggestUseCompoundFile;
Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
if (switchDirectory != null) {
suggestUseCompoundFile = false;
logger.debug("Using [simple_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.primaryExtensions());
directory = wrapDirectory(switchDirectory);
suggestUseCompoundFile = true;
if (switchDirectory.v2() != null) {
suggestUseCompoundFile = switchDirectory.v2();
}
logger.debug("Using [simple_fs] Store with path [{}], cache [true] with extensions [{}]", fsDirectory.getFile(), switchDirectory.v1().primaryExtensions());
directory = wrapDirectory(switchDirectory.v1());
} else {
suggestUseCompoundFile = true;
directory = wrapDirectory(fsDirectory);
logger.debug("Using [simple_fs] Store with path [{}]", fsDirectory.getFile());
}
this.suggestUseCompoundFile = suggestUseCompoundFile;
}
@Override public FSDirectory fsDirectory() {

4
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java
View File

@ -167,7 +167,11 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
synchronized (mutex) {
MapBuilder<String, StoreFileMetaData> builder = MapBuilder.newMapBuilder();
for (String file : delegate.listAll()) {
try {
builder.put(file, new StoreFileMetaData(file, delegate.fileLength(file), delegate.fileModified(file), preComputedMd5(file)));
} catch (FileNotFoundException e) {
// ignore
}
}
filesMetadata = builder.immutableMap();
files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]);