honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[STORE] Make a hybrid directory default using `mmapfs` and `niofs` 

`mmapfs` is really good for random access but can have sideeffects if
memory maps are large depending on the operating system etc. A hybrid
solution where only selected files are actually memory mapped but others
mostly consumed sequentially brings the best of both worlds and
minimizes the memory map impact.
This commit mmaps only the `dvd` and `tim` file for fast random access
on docvalues and term dictionaries.

Closes #6636
This commit is contained in:
Simon Willnauer 2014-06-26 22:46:21 +02:00
parent b69fa52588
commit d82a434d10
17 changed files with 370 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
docs/reference/index-modules/store.asciidoc
View File

@ -57,8 +57,8 @@ using the index update settings API dynamically.
File system based storage is the default storage used. There are
different implementations or _storage types_. The best one for the
operating environment will be automatically chosen: `mmapfs` on
Solaris/Linux/Windows 64bit, `simplefs` on Windows 32bit, and
`niofs` for the rest.
Windows 64bit, `simplefs` on Windows 32bit, and `default`
(hybrid `niofs` and `mmapfs`) for the rest.
This can be overridden for all indices by adding this to the
`config/elasticsearch.yml` file:
@ -112,6 +112,17 @@ process equal to the size of the file being mapped. Before using this
class, be sure your have plenty of virtual address space.
See <<vm-max-map-count>>
[[default_fs]]
[float]
==== Hybrid MMap / NIO FS coming[1.3.0]
The `default` type stores the shard index on the file system depending on
the file type by mapping a file into memory (mmap) or using Java NIO. Currently
only the Lucene term dictionary and doc values files are memory mapped to reduce
the impact on the operating system. All other files are opened using Lucene `NIOFSDirectory`.
Address space settings (<<vm-map-map-count>>) might also apply if your term
dictionaries are large.
[float]
[[store-memory]]
=== Memory

2
docs/reference/setup/configuration.asciidoc
View File

@ -51,7 +51,7 @@ curl localhost:9200/_nodes/process?pretty
[[vm-max-map-count]]
==== Virtual memory
Elasticsearch uses <<mmapfs,`mmapfs`>> by default to store its indices. The default
Elasticsearch uses a <<default_fs,`hybrid mmapfs / niofs`>> directory by default to store its indices. The default
operating system limits on mmap counts is likely to be too low, which may
result in out of memory exceptions. On Linux, you can increase the limits by
running the following command as `root`:

2
src/main/java/org/apache/lucene/store/RateLimitedFSDirectory.java
View File

@ -28,7 +28,7 @@ public final class RateLimitedFSDirectory extends FilterDirectory {
private final StoreRateLimiting.Listener rateListener;
public RateLimitedFSDirectory(FSDirectory wrapped, StoreRateLimiting.Provider rateLimitingProvider,
public RateLimitedFSDirectory(Directory wrapped, StoreRateLimiting.Provider rateLimitingProvider,
StoreRateLimiting.Listener rateListener) {
super(wrapped);
this.rateLimitingProvider = rateLimitingProvider;

5
src/main/java/org/apache/lucene/store/StoreUtils.java
View File

@ -39,6 +39,11 @@ public final class StoreUtils {
SimpleFSDirectory simpleFSDirectory = (SimpleFSDirectory)directory;
return "simplefs(" + simpleFSDirectory.getDirectory() + ")";
}
if (directory instanceof FileSwitchDirectory) {
FileSwitchDirectory fileSwitchDirectory = (FileSwitchDirectory) directory;
return "default(" + toString(fileSwitchDirectory.getPrimaryDir()) + "," + toString(fileSwitchDirectory.getSecondaryDir()) + ")";
}
return directory.toString();
}
}

32
src/main/java/org/elasticsearch/index/store/DirectoryUtils.java
View File

@ -19,9 +19,8 @@
package org.elasticsearch.index.store;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.*;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
/**
@ -29,6 +28,10 @@ import org.elasticsearch.common.Nullable;
*/
public final class DirectoryUtils {
static {
assert Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_4_9 : "Remove the special case for NRTCachingDirectory - it implements FilterDirectory in 4.10";
}
private DirectoryUtils() {} // no instance
/**
@ -52,10 +55,16 @@ public final class DirectoryUtils {
}
}
private static final Directory getLeafDirectory(FilterDirectory dir) {
static final Directory getLeafDirectory(FilterDirectory dir) {
Directory current = dir.getDelegate();
while ((current instanceof FilterDirectory)) {
current = ((FilterDirectory) current).getDelegate();
while (true) {
if ((current instanceof FilterDirectory)) {
current = ((FilterDirectory) current).getDelegate();
} else if (current instanceof NRTCachingDirectory) { // remove this when we upgrade to Lucene 4.10
current = ((NRTCachingDirectory) current).getDelegate();
} else {
break;
}
}
return current;
}
@ -78,7 +87,16 @@ public final class DirectoryUtils {
if (dir instanceof FilterDirectory) {
d = getLeafDirectory((FilterDirectory) dir);
}
if (targetClass.isAssignableFrom(d.getClass())) {
if (d instanceof FileSwitchDirectory) {
T leaf = getLeaf(((FileSwitchDirectory) d).getPrimaryDir(), targetClass);
if (leaf == null) {
d = getLeaf(((FileSwitchDirectory) d).getSecondaryDir(), targetClass, defaultValue);
} else {
d = leaf;
}
}
if (d != null && targetClass.isAssignableFrom(d.getClass())) {
return targetClass.cast(d);
} else {
return defaultValue;

68
src/main/java/org/elasticsearch/index/store/IndexStoreModule.java
View File

@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.Modules;
import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.store.fs.DefaultFsIndexStoreModule;
import org.elasticsearch.index.store.fs.MmapFsIndexStoreModule;
import org.elasticsearch.index.store.fs.NioFsIndexStoreModule;
import org.elasticsearch.index.store.fs.SimpleFsIndexStoreModule;
@ -39,6 +40,51 @@ public class IndexStoreModule extends AbstractModule implements SpawnModules {
private final Settings settings;
public static enum Type {
NIOFS {
public boolean match(String setting) {
return super.match(setting) || "nio_fs".equalsIgnoreCase(setting);
}
},
MMAPFS {
public boolean match(String setting) {
return super.match(setting) || "mmap_fs".equalsIgnoreCase(setting);
}
},
SIMPLEFS {
public boolean match(String setting) {
return super.match(setting) || "simple_fs".equalsIgnoreCase(setting);
}
},
RAM {
public boolean fsStore() {
return false;
}
},
MEMORY {
public boolean fsStore() {
return false;
}
},
FS,
DEFAULT,;
/**
* Returns true iff this store type is a filesystem based store.
*/
public boolean fsStore() {
return true;
}
/**
* Returns true iff this settings matches the type.
*/
public boolean match(String setting) {
return this.name().equalsIgnoreCase(setting);
}
}
public IndexStoreModule(Settings settings) {
this.settings = settings;
}
@ -46,26 +92,32 @@ public class IndexStoreModule extends AbstractModule implements SpawnModules {
@Override
public Iterable<? extends Module> spawnModules() {
Class<? extends Module> indexStoreModule = NioFsIndexStoreModule.class;
// Same logic as FSDirectory#open ...
if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
&& Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
indexStoreModule = MmapFsIndexStoreModule.class;
if (Constants.WINDOWS) {
indexStoreModule = MmapFsIndexStoreModule.class;
} else {
// on linux and friends we only mmap dedicated files
indexStoreModule = DefaultFsIndexStoreModule.class;
}
} else if (Constants.WINDOWS) {
indexStoreModule = SimpleFsIndexStoreModule.class;
}
String storeType = settings.get("index.store.type");
if ("ram".equalsIgnoreCase(storeType)) {
if (Type.RAM.name().equalsIgnoreCase(storeType)) {
indexStoreModule = RamIndexStoreModule.class;
} else if ("memory".equalsIgnoreCase(storeType)) {
} else if (Type.MEMORY.match(storeType)) {
indexStoreModule = RamIndexStoreModule.class;
} else if ("fs".equalsIgnoreCase(storeType)) {
} else if (Type.FS.match(storeType)) {
// nothing to set here ... (we default to fs)
} else if ("simplefs".equalsIgnoreCase(storeType) || "simple_fs".equals(storeType)) {
} else if (Type.SIMPLEFS.match(storeType)) {
indexStoreModule = SimpleFsIndexStoreModule.class;
} else if ("niofs".equalsIgnoreCase(storeType) || "nio_fs".equalsIgnoreCase(storeType)) {
} else if (Type.NIOFS.match(storeType)) {
indexStoreModule = NioFsIndexStoreModule.class;
} else if ("mmapfs".equalsIgnoreCase(storeType) || "mmap_fs".equalsIgnoreCase(storeType)) {
} else if (Type.MMAPFS.match(storeType)) {
indexStoreModule = MmapFsIndexStoreModule.class;
} else if (Type.DEFAULT.match(storeType)) {
indexStoreModule = DefaultFsIndexStoreModule.class;
} else if (storeType != null) {
indexStoreModule = settings.getAsClass("index.store.type", indexStoreModule, "org.elasticsearch.index.store.", "IndexStoreModule");
}

54
src/main/java/org/elasticsearch/index/store/fs/DefaultFsDirectoryService.java Normal file
View File

@ -0,0 +1,54 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import com.google.common.collect.Sets;
import org.apache.lucene.store.*;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.IndexStore;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
/**
*/
public class DefaultFsDirectoryService extends FsDirectoryService {
/*
* We are mmapping docvalues as well as term dictionaries, all other files are served through NIOFS
* this provides good random access performance while not creating unnecessary mmaps for files like stored
* fields etc.
*/
private static final Set<String> PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("dvd", "tim"));
@Inject
public DefaultFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore) {
super(shardId, indexSettings, indexStore);
}
@Override
protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
return new FileSwitchDirectory(PRIMARY_EXTENSIONS, new MMapDirectory(location, lockFactory), new NIOFSDirectory(location, lockFactory), true);
}
}

45
src/main/java/org/elasticsearch/index/store/fs/DefaultFsIndexStore.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.indices.store.IndicesStore;
/**
*
*/
public final class DefaultFsIndexStore extends FsIndexStore {
@Inject
public DefaultFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, indicesStore, nodeEnv);
}
@Override
public Class<? extends DirectoryService> shardDirectory() {
return DefaultFsDirectoryService.class;
}
}

34
src/main/java/org/elasticsearch/index/store/fs/DefaultFsIndexStoreModule.java Normal file
View File

@ -0,0 +1,34 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.store.IndexStore;
/**
*
*/
public class DefaultFsIndexStoreModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexStore.class).to(DefaultFsIndexStore.class).asEagerSingleton();
}
}

4
src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java
View File

@ -126,13 +126,13 @@ public abstract class FsDirectoryService extends AbstractIndexShardComponent imp
Directory[] dirs = new Directory[locations.length];
for (int i = 0; i < dirs.length; i++) {
FileSystemUtils.mkdirs(locations[i]);
FSDirectory wrapped = newFSDirectory(locations[i], buildLockFactory());
Directory wrapped = newFSDirectory(locations[i], buildLockFactory());
dirs[i] = new RateLimitedFSDirectory(wrapped, this, this) ;
}
return dirs;
}
protected abstract FSDirectory newFSDirectory(File location, LockFactory lockFactory) throws IOException;
protected abstract Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException;
@Override
public final void onPause(long nanos) {

4
src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MMapDirectory;
import org.elasticsearch.common.inject.Inject;
@ -41,7 +41,7 @@ public class MmapFsDirectoryService extends FsDirectoryService {
}
@Override
protected FSDirectory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
return new MMapDirectory(location, buildLockFactory());
}
}

4
src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.NIOFSDirectory;
import org.elasticsearch.common.inject.Inject;
@ -41,7 +41,7 @@ public class NioFsDirectoryService extends FsDirectoryService {
}
@Override
protected FSDirectory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
return new NIOFSDirectory(location, lockFactory);
}
}

4
src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java
View File

@ -19,7 +19,7 @@
package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.common.inject.Inject;
@ -41,7 +41,7 @@ public class SimpleFsDirectoryService extends FsDirectoryService {
}
@Override
protected FSDirectory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
protected Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
return new SimpleFSDirectory(location, lockFactory);
}
}

87
src/test/java/org/elasticsearch/index/store/DirectoryUtilsTest.java Normal file
View File

@ -0,0 +1,87 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.carrotsearch.randomizedtesting.annotations.Repeat;
import com.carrotsearch.randomizedtesting.annotations.Seed;
import org.apache.lucene.store.*;
import org.elasticsearch.test.ElasticsearchLuceneTestCase;
import org.elasticsearch.test.ElasticsearchTestCase;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import static org.hamcrest.CoreMatchers.*;
public class DirectoryUtilsTest extends ElasticsearchLuceneTestCase {
@Test
public void testGetLeave() throws IOException {
File file = ElasticsearchTestCase.newTempDir(LifecycleScope.TEST);
final int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
{
BaseDirectoryWrapper dir = newFSDirectory(file);
FSDirectory directory = DirectoryUtils.getLeaf(new FilterDirectory(dir) {}, FSDirectory.class, null);
assertThat(directory, notNullValue());
assertThat(directory, sameInstance(DirectoryUtils.getLeafDirectory(dir)));
dir.close();
}
{
BaseDirectoryWrapper dir = newFSDirectory(file);
FSDirectory directory = DirectoryUtils.getLeaf(dir, FSDirectory.class, null);
assertThat(directory, notNullValue());
assertThat(directory, sameInstance(DirectoryUtils.getLeafDirectory(dir)));
dir.close();
}
{
Set<String> stringSet = Collections.emptySet();
BaseDirectoryWrapper dir = newFSDirectory(file);
FSDirectory directory = DirectoryUtils.getLeaf(new FileSwitchDirectory(stringSet, dir, dir, random().nextBoolean()), FSDirectory.class, null);
assertThat(directory, notNullValue());
assertThat(directory, sameInstance(DirectoryUtils.getLeafDirectory(dir)));
dir.close();
}
{
Set<String> stringSet = Collections.emptySet();
BaseDirectoryWrapper dir = newFSDirectory(file);
FSDirectory directory = DirectoryUtils.getLeaf(new FilterDirectory(new FileSwitchDirectory(stringSet, dir, dir, random().nextBoolean())) {}, FSDirectory.class, null);
assertThat(directory, notNullValue());
assertThat(directory, sameInstance(DirectoryUtils.getLeafDirectory(dir)));
dir.close();
}
{
Set<String> stringSet = Collections.emptySet();
BaseDirectoryWrapper dir = newFSDirectory(file);
RAMDirectory directory = DirectoryUtils.getLeaf(new FilterDirectory(new FileSwitchDirectory(stringSet, dir, dir, random().nextBoolean())) {}, RAMDirectory.class, null);
assertThat(directory, nullValue());
dir.close();
}
}
}
}

39
src/test/java/org/elasticsearch/indices/store/SimpleDistributorTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.indices.store;
import org.apache.lucene.store.Directory;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
@ -39,19 +40,19 @@ import static org.hamcrest.Matchers.*;
*/
public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
public final static String[] STORE_TYPES = {"fs", "simplefs", "niofs", "mmapfs"};
@Test
public void testAvailableSpaceDetection() {
for (String store : STORE_TYPES) {
createIndexWithStoreType("test", store, StrictDistributor.class.getCanonicalName());
for (IndexStoreModule.Type store : IndexStoreModule.Type.values()) {
if (store.fsStore()) {
createIndexWithStoreType("test", store, StrictDistributor.class.getCanonicalName());
}
}
}
@Test
public void testDirectoryToString() throws IOException {
internalCluster().wipeTemplates(); // no random settings please
createIndexWithStoreType("test", "niofs", "least_used");
createIndexWithStoreType("test", IndexStoreModule.Type.NIOFS, "least_used");
String storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
File[] dataPaths = dataPaths();
@ -61,7 +62,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
createIndexWithStoreType("test", "niofs", "random");
createIndexWithStoreType("test", IndexStoreModule.Type.NIOFS, "random");
storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
dataPaths = dataPaths();
@ -71,7 +72,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
createIndexWithStoreType("test", "mmapfs", "least_used");
createIndexWithStoreType("test", IndexStoreModule.Type.MMAPFS, "least_used");
storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
dataPaths = dataPaths();
@ -81,7 +82,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
createIndexWithStoreType("test", "simplefs", "least_used");
createIndexWithStoreType("test", IndexStoreModule.Type.SIMPLEFS, "least_used");
storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
dataPaths = dataPaths();
@ -91,13 +92,25 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
createIndexWithStoreType("test", "memory", "least_used");
createIndexWithStoreType("test", IndexStoreModule.Type.DEFAULT, "least_used");
storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
dataPaths = dataPaths();
assertThat(storeString.toLowerCase(Locale.ROOT), startsWith("store(least_used[rate_limited(default(mmapfs(" + dataPaths[0].getAbsolutePath().toLowerCase(Locale.ROOT)));
assertThat(storeString.toLowerCase(Locale.ROOT), containsString("),niofs(" + dataPaths[0].getAbsolutePath().toLowerCase(Locale.ROOT)));
if (dataPaths.length > 1) {
assertThat(storeString.toLowerCase(Locale.ROOT), containsString("), rate_limited(default(mmapfs(" + dataPaths[1].getAbsolutePath().toLowerCase(Locale.ROOT)));
}
assertThat(storeString, endsWith(", type=MERGE, rate=20.0)])"));
createIndexWithStoreType("test", IndexStoreModule.Type.MEMORY, "least_used");
storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
dataPaths = dataPaths();
assertThat(storeString, equalTo("store(least_used[ram])"));
createIndexWithoutRateLimitingStoreType("test", "niofs", "least_used");
createIndexWithoutRateLimitingStoreType("test", IndexStoreModule.Type.NIOFS, "least_used");
storeString = getStoreDirectory("test", 0).toString();
logger.info(storeString);
dataPaths = dataPaths();
@ -108,12 +121,12 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
assertThat(storeString, endsWith(")])"));
}
private void createIndexWithStoreType(String index, String storeType, String distributor) {
private void createIndexWithStoreType(String index, IndexStoreModule.Type storeType, String distributor) {
cluster().wipeIndices(index);
client().admin().indices().prepareCreate(index)
.setSettings(settingsBuilder()
.put("index.store.distributor", distributor)
.put("index.store.type", storeType)
.put("index.store.type", storeType.name())
.put("index.number_of_replicas", 0)
.put("index.number_of_shards", 1)
)
@ -121,7 +134,7 @@ public class SimpleDistributorTests extends ElasticsearchIntegrationTest {
assertThat(client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet().isTimedOut(), equalTo(false));
}
private void createIndexWithoutRateLimitingStoreType(String index, String storeType, String distributor) {
private void createIndexWithoutRateLimitingStoreType(String index, IndexStoreModule.Type storeType, String distributor) {
cluster().wipeIndices(index);
client().admin().indices().prepareCreate(index)
.setSettings(settingsBuilder()

21
src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java
View File

@ -31,10 +31,7 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.fs.FsDirectoryService;
import org.elasticsearch.index.store.fs.MmapFsDirectoryService;
import org.elasticsearch.index.store.fs.NioFsDirectoryService;
import org.elasticsearch.index.store.fs.SimpleFsDirectoryService;
import org.elasticsearch.index.store.fs.*;
import org.elasticsearch.index.store.ram.RamDirectoryService;
import java.io.IOException;
@ -107,13 +104,15 @@ public class MockDirectoryHelper {
} else if (Constants.WINDOWS) {
return new SimpleFsDirectoryService(shardId, indexSettings, indexStore);
}
switch (random.nextInt(3)) {
case 1:
return new MmapFsDirectoryService(shardId, indexSettings, indexStore);
case 0:
return new SimpleFsDirectoryService(shardId, indexSettings, indexStore);
default:
return new NioFsDirectoryService(shardId, indexSettings, indexStore);
switch (random.nextInt(4)) {
case 2:
return new DefaultFsDirectoryService(shardId, indexSettings, indexStore);
case 1:
return new MmapFsDirectoryService(shardId, indexSettings, indexStore);
case 0:
return new SimpleFsDirectoryService(shardId, indexSettings, indexStore);
default:
return new NioFsDirectoryService(shardId, indexSettings, indexStore);
}
}

3
src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java
View File

@ -22,7 +22,6 @@ package org.elasticsearch.test.store;
import com.google.common.base.Charsets;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.LockFactory;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
@ -82,7 +81,7 @@ public class MockFSDirectoryService extends FsDirectoryService {
}
@Override
protected synchronized FSDirectory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
protected synchronized Directory newFSDirectory(File location, LockFactory lockFactory) throws IOException {
throw new UnsupportedOperationException();
}