honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Index FS Store: Allow to cache (in memory) specific files, closes #82

This commit is contained in:
kimchy 2010-03-22 17:09:03 +02:00
parent 267859c784
commit fa55c40c87
7 changed files with 280 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/AbstractFsStore.java
View File

@ -19,11 +19,18 @@
package org.elasticsearch.index.store.fs;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.memory.ByteBufferDirectory;
import org.elasticsearch.index.store.memory.HeapDirectory;
import org.elasticsearch.index.store.support.AbstractStore;
import org.elasticsearch.util.SizeUnit;
import org.elasticsearch.util.SizeValue;
import org.elasticsearch.util.io.FileSystemUtils;
import org.elasticsearch.util.lucene.store.SwitchDirectory;
import org.elasticsearch.util.settings.Settings;
import java.io.IOException;
@ -31,17 +38,40 @@ import java.io.IOException;
/**
* @author kimchy (Shay Banon)
*/
public abstract class AbstractFsStore<T extends FSDirectory> extends AbstractStore<T> {
public abstract class AbstractFsStore<T extends Directory> extends AbstractStore<T> {
public AbstractFsStore(ShardId shardId, @IndexSettings Settings indexSettings) {
super(shardId, indexSettings);
}
@Override public void fullDelete() throws IOException {
FileSystemUtils.deleteRecursively(directory().getFile());
FileSystemUtils.deleteRecursively(fsDirectory().getFile());
// if we are the last ones, delete also the actual index
if (directory().getFile().getParentFile().list().length == 0) {
FileSystemUtils.deleteRecursively(directory().getFile().getParentFile());
if (fsDirectory().getFile().getParentFile().list().length == 0) {
FileSystemUtils.deleteRecursively(fsDirectory().getFile().getParentFile());
}
}
public abstract FSDirectory fsDirectory();
protected SwitchDirectory buildSwitchDirectoryIfNeeded(Directory fsDirectory) {
boolean cache = componentSettings.getAsBoolean("cache.enabled", false);
if (!cache) {
return null;
}
SizeValue bufferSize = componentSettings.getAsSize("cache.bufferSize", new SizeValue(100, SizeUnit.KB));
SizeValue cacheSize = componentSettings.getAsSize("cache.cacheSize", new SizeValue(20, SizeUnit.MB));
boolean direct = componentSettings.getAsBoolean("cache.direct", true);
boolean warmCache = componentSettings.getAsBoolean("cache.warmCache", true);
Directory memDir;
if (direct) {
memDir = new ByteBufferDirectory((int) bufferSize.bytes(), (int) cacheSize.bytes(), true, warmCache);
} else {
memDir = new HeapDirectory(bufferSize, cacheSize, warmCache);
}
// see http://lucene.apache.org/java/3_0_1/fileformats.html
String[] primaryExtensions = componentSettings.getAsArray("cache.extensions", new String[]{"", "del", "gen"});
return new SwitchDirectory(ImmutableSet.of(primaryExtensions), memDir, fsDirectory, true);
}
}

35
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsStore.java
View File

@ -20,12 +20,15 @@
package org.elasticsearch.index.store.fs;
import com.google.inject.Inject;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.MMapDirectory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.LocalNodeId;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.IndexShardLifecycle;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.util.lucene.store.SwitchDirectory;
import org.elasticsearch.util.settings.Settings;
import java.io.File;
@ -37,24 +40,46 @@ import static org.elasticsearch.index.store.fs.FsStores.*;
* @author kimchy (Shay Banon)
*/
@IndexShardLifecycle
public class MmapFsStore extends AbstractFsStore<MMapDirectory> {
public class MmapFsStore extends AbstractFsStore<Directory> {
private final boolean syncToDisk;
private final MMapDirectory directory;
private final MMapDirectory fsDirectory;
private final Directory directory;
private final boolean suggestUseCompoundFile;
@Inject public MmapFsStore(ShardId shardId, @IndexSettings Settings indexSettings, Environment environment, @LocalNodeId String localNodeId) throws IOException {
super(shardId, indexSettings);
// by default, we don't need to sync to disk, since we use the gateway
this.syncToDisk = componentSettings.getAsBoolean("syncToDisk", false);
this.directory = new CustomMMapDirectory(createStoreFilePath(environment.workWithClusterFile(), localNodeId, shardId, MAIN_INDEX_SUFFIX), syncToDisk);
logger.debug("Using [MmapFs] Store with path [{}]", directory.getFile());
this.fsDirectory = new CustomMMapDirectory(createStoreFilePath(environment.workWithClusterFile(), localNodeId, shardId, MAIN_INDEX_SUFFIX), syncToDisk);
SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory);
if (switchDirectory != null) {
suggestUseCompoundFile = false;
logger.debug("Using [MmapFs] Store with path [{}], cache [true] with extensions [{}]", new Object[]{fsDirectory.getFile(), switchDirectory.primaryExtensions()});
directory = switchDirectory;
} else {
suggestUseCompoundFile = true;
directory = fsDirectory;
logger.debug("Using [MmapFs] Store with path [{}]", fsDirectory.getFile());
}
}
@Override public MMapDirectory directory() {
@Override public FSDirectory fsDirectory() {
return fsDirectory;
}
@Override public Directory directory() {
return directory;
}
@Override public boolean suggestUseCompoundFile() {
return suggestUseCompoundFile;
}
private static class CustomMMapDirectory extends MMapDirectory {
private final boolean syncToDisk;

35
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsStore.java
View File

@ -20,11 +20,14 @@
package org.elasticsearch.index.store.fs;
import com.google.inject.Inject;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.NIOFSDirectory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.LocalNodeId;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.util.lucene.store.SwitchDirectory;
import org.elasticsearch.util.settings.Settings;
import java.io.File;
@ -35,24 +38,46 @@ import static org.elasticsearch.index.store.fs.FsStores.*;
/**
* @author kimchy (Shay Banon)
*/
public class NioFsStore extends AbstractFsStore<NIOFSDirectory> {
public class NioFsStore extends AbstractFsStore<Directory> {
private final boolean syncToDisk;
private final NIOFSDirectory directory;
private final NIOFSDirectory fsDirectory;
private final Directory directory;
private final boolean suggestUseCompoundFile;
@Inject public NioFsStore(ShardId shardId, @IndexSettings Settings indexSettings, Environment environment, @LocalNodeId String localNodeId) throws IOException {
super(shardId, indexSettings);
// by default, we don't need to sync to disk, since we use the gateway
this.syncToDisk = componentSettings.getAsBoolean("syncToDisk", false);
this.directory = new CustomNioFSDirectory(createStoreFilePath(environment.workWithClusterFile(), localNodeId, shardId, MAIN_INDEX_SUFFIX), syncToDisk);
logger.debug("Using [NioFs] Store with path [{}], syncToDisk [{}]", directory.getFile(), syncToDisk);
this.fsDirectory = new CustomNioFSDirectory(createStoreFilePath(environment.workWithClusterFile(), localNodeId, shardId, MAIN_INDEX_SUFFIX), syncToDisk);
SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory);
if (switchDirectory != null) {
suggestUseCompoundFile = false;
logger.debug("Using [NioFs] Store with path [{}], cache [true] with extensions [{}]", new Object[]{fsDirectory.getFile(), switchDirectory.primaryExtensions()});
directory = switchDirectory;
} else {
suggestUseCompoundFile = true;
directory = fsDirectory;
logger.debug("Using [NioFs] Store with path [{}]", fsDirectory.getFile());
}
}
@Override public NIOFSDirectory directory() {
@Override public FSDirectory fsDirectory() {
return fsDirectory;
}
@Override public Directory directory() {
return directory;
}
@Override public boolean suggestUseCompoundFile() {
return suggestUseCompoundFile;
}
private static class CustomNioFSDirectory extends NIOFSDirectory {
private final boolean syncToDisk;

35
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsStore.java
View File

@ -20,11 +20,14 @@
package org.elasticsearch.index.store.fs;
import com.google.inject.Inject;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.env.Environment;
import org.elasticsearch.index.LocalNodeId;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.util.lucene.store.SwitchDirectory;
import org.elasticsearch.util.settings.Settings;
import java.io.File;
@ -35,24 +38,46 @@ import static org.elasticsearch.index.store.fs.FsStores.*;
/**
* @author kimchy (Shay Banon)
*/
public class SimpleFsStore extends AbstractFsStore<SimpleFSDirectory> {
public class SimpleFsStore extends AbstractFsStore<Directory> {
private final boolean syncToDisk;
private SimpleFSDirectory directory;
private SimpleFSDirectory fsDirectory;
private final Directory directory;
private final boolean suggestUseCompoundFile;
@Inject public SimpleFsStore(ShardId shardId, @IndexSettings Settings indexSettings, Environment environment, @LocalNodeId String localNodeId) throws IOException {
super(shardId, indexSettings);
// by default, we don't need to sync to disk, since we use the gateway
this.syncToDisk = componentSettings.getAsBoolean("syncToDisk", false);
this.directory = new CustomSimpleFSDirectory(createStoreFilePath(environment.workWithClusterFile(), localNodeId, shardId, MAIN_INDEX_SUFFIX), syncToDisk);
logger.debug("Using [SimpleFs] Store with path [{}], syncToDisk [{}]", directory.getFile(), syncToDisk);
this.fsDirectory = new CustomSimpleFSDirectory(createStoreFilePath(environment.workWithClusterFile(), localNodeId, shardId, MAIN_INDEX_SUFFIX), syncToDisk);
SwitchDirectory switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory);
if (switchDirectory != null) {
suggestUseCompoundFile = false;
logger.debug("Using [SimpleFs] Store with path [{}], cache [true] with extensions [{}]", new Object[]{fsDirectory.getFile(), switchDirectory.primaryExtensions()});
directory = switchDirectory;
} else {
suggestUseCompoundFile = true;
directory = fsDirectory;
logger.debug("Using [SimpleFs] Store with path [{}]", fsDirectory.getFile());
}
}
@Override public SimpleFSDirectory directory() {
@Override public FSDirectory fsDirectory() {
return fsDirectory;
}
@Override public Directory directory() {
return directory;
}
@Override public boolean suggestUseCompoundFile() {
return suggestUseCompoundFile;
}
private static class CustomSimpleFSDirectory extends SimpleFSDirectory {
private final boolean syncToDisk;

147
modules/elasticsearch/src/main/java/org/elasticsearch/util/lucene/store/SwitchDirectory.java Normal file
View File

@ -0,0 +1,147 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.util.lucene.store;
import com.google.common.collect.ImmutableSet;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import java.io.IOException;
import java.util.Set;
/**
* A Directory instance that switches files between
* two other Directory instances.
*
* <p>Files with the specified extensions are placed in the
* primary directory; others are placed in the secondary
* directory.
*
* @author kimchy (shay.banon)
*/
public class SwitchDirectory extends Directory {
private final Directory secondaryDir;
private final Directory primaryDir;
private final ImmutableSet<String> primaryExtensions;
private boolean doClose;
public SwitchDirectory(Set<String> primaryExtensions, Directory primaryDir, Directory secondaryDir, boolean doClose) {
this.primaryExtensions = ImmutableSet.copyOf(primaryExtensions);
this.primaryDir = primaryDir;
this.secondaryDir = secondaryDir;
this.doClose = doClose;
this.lockFactory = primaryDir.getLockFactory();
}
public ImmutableSet<String> primaryExtensions() {
return primaryExtensions;
}
/**
* Return the primary directory
*/
public Directory primaryDir() {
return primaryDir;
}
/**
* Return the secondary directory
*/
public Directory secondaryDir() {
return secondaryDir;
}
@Override public void close() throws IOException {
if (doClose) {
try {
secondaryDir.close();
} finally {
primaryDir.close();
}
doClose = false;
}
}
@Override public String[] listAll() throws IOException {
String[] primaryFiles = primaryDir.listAll();
String[] secondaryFiles = secondaryDir.listAll();
String[] files = new String[primaryFiles.length + secondaryFiles.length];
System.arraycopy(primaryFiles, 0, files, 0, primaryFiles.length);
System.arraycopy(secondaryFiles, 0, files, primaryFiles.length, secondaryFiles.length);
return files;
}
/**
* Utility method to return a file's extension.
*/
public static String getExtension(String name) {
int i = name.lastIndexOf('.');
if (i == -1) {
return "";
}
return name.substring(i + 1, name.length());
}
private Directory getDirectory(String name) {
String ext = getExtension(name);
if (primaryExtensions.contains(ext)) {
return primaryDir;
} else {
return secondaryDir;
}
}
@Override public boolean fileExists(String name) throws IOException {
return getDirectory(name).fileExists(name);
}
@Override public long fileModified(String name) throws IOException {
return getDirectory(name).fileModified(name);
}
@Override public void touchFile(String name) throws IOException {
getDirectory(name).touchFile(name);
}
@Override public void deleteFile(String name) throws IOException {
getDirectory(name).deleteFile(name);
}
@Override public long fileLength(String name) throws IOException {
return getDirectory(name).fileLength(name);
}
@Override public IndexOutput createOutput(String name) throws IOException {
return getDirectory(name).createOutput(name);
}
@Override public void sync(String name) throws IOException {
getDirectory(name).sync(name);
}
@Override public IndexInput openInput(String name) throws IOException {
return getDirectory(name).openInput(name);
}
}

7
modules/elasticsearch/src/main/java/org/elasticsearch/util/settings/ImmutableSettings.java
View File

@ -204,6 +204,10 @@ public class ImmutableSettings implements Settings {
}
@Override public String[] getAsArray(String settingPrefix) throws SettingsException {
return getAsArray(settingPrefix, Strings.EMPTY_ARRAY);
}
@Override public String[] getAsArray(String settingPrefix, String[] defaultArray) throws SettingsException {
List<String> result = newArrayList();
int counter = 0;
while (true) {
@ -213,6 +217,9 @@ public class ImmutableSettings implements Settings {
}
result.add(value);
}
if (result.isEmpty()) {
return defaultArray;
}
return result.toArray(new String[result.size()]);
}

2
modules/elasticsearch/src/main/java/org/elasticsearch/util/settings/Settings.java
View File

@ -201,6 +201,8 @@ public interface Settings {
* @return The setting array values
* @throws SettingsException
*/
String[] getAsArray(String settingPrefix, String[] defaultArray) throws SettingsException;
String[] getAsArray(String settingPrefix) throws SettingsException;
/**