honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[CORE] Simplify IndexStore and friends 

Today we have a lot of bloat in the IndexStore and related classes. THe IndexStore interface
is unneeded as we always subclass AbstractIndexStore and it hides circular dependencies
that are problematic when added. Guice proxies them if you have an interface which is bad in
general. This commit removes most of the bloat classes and unifies all the classes we have
into a single one since they all just structural and don't encode any functionality.
This commit is contained in:
Simon Willnauer 2015-04-24 11:43:23 +02:00
parent 5bdfdc42d9
commit c25ca1ebfb
30 changed files with 273 additions and 775 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/org/elasticsearch/index/IndexService.java
View File

@ -115,8 +115,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
private final BitsetFilterCache bitsetFilterCache;
private final IndexStore indexStore;
private final IndexSettingsService settingsService;
private final NodeEnvironment nodeEnv;
@ -131,7 +129,7 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
public IndexService(Injector injector, Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv,
AnalysisService analysisService, MapperService mapperService, IndexQueryParserService queryParserService,
SimilarityService similarityService, IndexAliasesService aliasesService, IndexCache indexCache,
IndexStore indexStore, IndexSettingsService settingsService,
IndexSettingsService settingsService,
IndexFieldDataService indexFieldData, BitsetFilterCache bitSetFilterCache, IndicesService indicesServices) {
super(index, indexSettings);
this.injector = injector;
@ -143,7 +141,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
this.aliasesService = aliasesService;
this.indexCache = indexCache;
this.indexFieldData = indexFieldData;
this.indexStore = indexStore;
this.settingsService = settingsService;
this.bitsetFilterCache = bitSetFilterCache;
@ -214,10 +211,6 @@ public class IndexService extends AbstractIndexComponent implements IndexCompone
return this.settingsService;
}
public IndexStore store() {
return indexStore;
}
public IndexCache cache() {
return indexCache;
}

6
src/main/java/org/elasticsearch/index/settings/IndexDynamicSettingsModule.java
View File

@ -36,7 +36,7 @@ import org.elasticsearch.index.merge.policy.TieredMergePolicyProvider;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.translog.TranslogService;
import org.elasticsearch.index.translog.fs.FsTranslog;
import org.elasticsearch.indices.IndicesWarmer;
@ -51,8 +51,8 @@ public class IndexDynamicSettingsModule extends AbstractModule {
public IndexDynamicSettingsModule() {
indexDynamicSettings = new DynamicSettings();
indexDynamicSettings.addDynamicSetting(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE);
indexDynamicSettings.addDynamicSetting(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE);
indexDynamicSettings.addDynamicSetting(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, Validator.BYTES_SIZE);
indexDynamicSettings.addDynamicSetting(IndexStore.INDEX_STORE_THROTTLE_TYPE);
indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.MAX_THREAD_COUNT);
indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.MAX_MERGE_COUNT);
indexDynamicSettings.addDynamicSetting(ConcurrentMergeSchedulerProvider.AUTO_THROTTLE);

52
src/main/java/org/elasticsearch/index/store/fs/FsDirectoryService.java → src/main/java/org/elasticsearch/index/store/FsDirectoryService.java
View File

@ -17,13 +17,18 @@
* under the License.
*/
package org.elasticsearch.index.store.fs;
package org.elasticsearch.index.store;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Set;
import com.google.common.collect.Sets;
import org.apache.lucene.store.*;
import org.apache.lucene.util.Constants;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.metrics.CounterMetric;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
@ -31,19 +36,21 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.index.store.StoreException;
/**
*/
public abstract class FsDirectoryService extends DirectoryService implements StoreRateLimiting.Listener, StoreRateLimiting.Provider {
public class FsDirectoryService extends DirectoryService implements StoreRateLimiting.Listener, StoreRateLimiting.Provider {
protected final IndexStore indexStore;
private final CounterMetric rateLimitingTimeInNanos = new CounterMetric();
private final ShardPath path;
public FsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath path) {
super(shardId, indexSettings);
@Inject
public FsDirectoryService(@IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath path) {
super(path.getShardId(), indexSettings);
this.path = path;
this.indexStore = indexStore;
}
@ -79,10 +86,45 @@ public abstract class FsDirectoryService extends DirectoryService implements Sto
return new RateLimitedFSDirectory(wrapped, this, this) ;
}
protected abstract Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException;
@Override
public void onPause(long nanos) {
rateLimitingTimeInNanos.inc(nanos);
}
/*
* We are mmapping docvalues as well as term dictionaries, all other files are served through NIOFS
* this provides good random access performance while not creating unnecessary mmaps for files like stored
* fields etc.
*/
private static final Set<String> PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("dvd", "tim"));
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
final String storeType = indexSettings.get(IndexStoreModule.STORE_TYPE, IndexStoreModule.Type.DEFAULT.name());
if (IndexStoreModule.Type.FS.match(storeType) || IndexStoreModule.Type.DEFAULT.match(storeType)) {
final FSDirectory open = FSDirectory.open(location, lockFactory); // use lucene defaults
if (open instanceof MMapDirectory && Constants.WINDOWS == false) {
return newDefaultDir(location, (MMapDirectory) open, lockFactory);
}
return open;
} else if (IndexStoreModule.Type.SIMPLEFS.match(storeType)) {
return new SimpleFSDirectory(location, lockFactory);
} else if (IndexStoreModule.Type.NIOFS.match(storeType)) {
return new NIOFSDirectory(location, lockFactory);
} else if (IndexStoreModule.Type.MMAPFS.match(storeType)) {
return new MMapDirectory(location, lockFactory);
}
throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
}
private Directory newDefaultDir(Path location, final MMapDirectory mmapDir, LockFactory lockFactory) throws IOException {
return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) {
@Override
public String[] listAll() throws IOException {
// Avoid doing listAll twice:
return mmapDir.listAll();
}
};
}
}

89
src/main/java/org/elasticsearch/index/store/IndexStore.java
View File

@ -20,28 +20,101 @@
package org.elasticsearch.index.store;
import org.apache.lucene.store.StoreRateLimiting;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.indices.store.IndicesStore;
import java.io.Closeable;
import java.io.IOException;
import java.nio.file.Path;
/**
* Index store is an index level information of the {@link Store} each shard will use.
*
*/
public interface IndexStore extends Closeable {
public class IndexStore extends AbstractIndexComponent implements Closeable {
public static final String INDEX_STORE_THROTTLE_TYPE = "index.store.throttle.type";
public static final String INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC = "index.store.throttle.max_bytes_per_sec";
private final IndexSettingsService settingsService;
class ApplySettings implements IndexSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, IndexStore.this.rateLimitingType);
if (!rateLimitingType.equals(IndexStore.this.rateLimitingType)) {
logger.info("updating index.store.throttle.type from [{}] to [{}]", IndexStore.this.rateLimitingType, rateLimitingType);
if (rateLimitingType.equalsIgnoreCase("node")) {
IndexStore.this.rateLimitingType = rateLimitingType;
IndexStore.this.nodeRateLimiting = true;
} else {
StoreRateLimiting.Type.fromString(rateLimitingType);
IndexStore.this.rateLimitingType = rateLimitingType;
IndexStore.this.nodeRateLimiting = false;
IndexStore.this.rateLimiting.setType(rateLimitingType);
}
}
ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndexStore.this.rateLimitingThrottle);
if (!rateLimitingThrottle.equals(IndexStore.this.rateLimitingThrottle)) {
logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndexStore.this.rateLimitingThrottle, rateLimitingThrottle, IndexStore.this.rateLimitingType);
IndexStore.this.rateLimitingThrottle = rateLimitingThrottle;
IndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle);
}
}
}
protected final IndicesStore indicesStore;
private volatile String rateLimitingType;
private volatile ByteSizeValue rateLimitingThrottle;
private volatile boolean nodeRateLimiting;
private final StoreRateLimiting rateLimiting = new StoreRateLimiting();
private final ApplySettings applySettings = new ApplySettings();
@Inject
public IndexStore(Index index, @IndexSettings Settings indexSettings, IndexSettingsService settingsService, IndicesStore indicesStore) {
super(index, indexSettings);
this.indicesStore = indicesStore;
this.rateLimitingType = indexSettings.get(INDEX_STORE_THROTTLE_TYPE, "none");
if (rateLimitingType.equalsIgnoreCase("node")) {
nodeRateLimiting = true;
} else {
nodeRateLimiting = false;
rateLimiting.setType(rateLimitingType);
}
this.rateLimitingThrottle = indexSettings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0));
rateLimiting.setMaxRate(rateLimitingThrottle);
logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle);
this.settingsService = settingsService;
this.settingsService.addListener(applySettings);
}
@Override
public void close() throws ElasticsearchException {
settingsService.removeListener(applySettings);
}
/**
* Returns the rate limiting, either of the index is explicitly configured, or
* the node level one (defaults to the node level one).
*/
StoreRateLimiting rateLimiting();
public StoreRateLimiting rateLimiting() {
return nodeRateLimiting ? indicesStore.rateLimiting() : this.rateLimiting;
}
/**
* The shard store class that should be used for each shard.
*/
Class<? extends DirectoryService> shardDirectory();
public Class<? extends DirectoryService> shardDirectory() {
return FsDirectoryService.class;
}
}

50
src/main/java/org/elasticsearch/index/store/IndexStoreModule.java
View File

@ -20,23 +20,16 @@
package org.elasticsearch.index.store;
import com.google.common.collect.ImmutableList;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.Constants;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.inject.Modules;
import org.elasticsearch.common.inject.SpawnModules;
import org.elasticsearch.common.inject.*;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.store.fs.DefaultFsIndexStoreModule;
import org.elasticsearch.index.store.fs.MmapFsIndexStoreModule;
import org.elasticsearch.index.store.fs.NioFsIndexStoreModule;
import org.elasticsearch.index.store.fs.SimpleFsIndexStoreModule;
/**
*
*/
public class IndexStoreModule extends AbstractModule implements SpawnModules {
public static final String STORE_TYPE = "index.store.type";
private final Settings settings;
public static enum Type {
@ -75,36 +68,23 @@ public class IndexStoreModule extends AbstractModule implements SpawnModules {
@Override
public Iterable<? extends Module> spawnModules() {
Class<? extends Module> indexStoreModule = NioFsIndexStoreModule.class;
if ((Constants.WINDOWS || Constants.SUN_OS || Constants.LINUX)
&& Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
if (Constants.WINDOWS) {
indexStoreModule = MmapFsIndexStoreModule.class;
} else {
// on linux and friends we only mmap dedicated files
indexStoreModule = DefaultFsIndexStoreModule.class;
final String storeType = settings.get(STORE_TYPE, Type.DEFAULT.name());
for (Type type : Type.values()) {
if (type.match(storeType)) {
return ImmutableList.of(new DefaultStoreModule());
}
} else if (Constants.WINDOWS) {
indexStoreModule = SimpleFsIndexStoreModule.class;
}
String storeType = settings.get("index.store.type");
if (Type.FS.match(storeType)) {
// nothing to set here ... (we default to fs)
} else if (Type.SIMPLEFS.match(storeType)) {
indexStoreModule = SimpleFsIndexStoreModule.class;
} else if (Type.NIOFS.match(storeType)) {
indexStoreModule = NioFsIndexStoreModule.class;
} else if (Type.MMAPFS.match(storeType)) {
indexStoreModule = MmapFsIndexStoreModule.class;
} else if (Type.DEFAULT.match(storeType)) {
indexStoreModule = DefaultFsIndexStoreModule.class;
} else if (storeType != null) {
indexStoreModule = settings.getAsClass("index.store.type", indexStoreModule, "org.elasticsearch.index.store.", "IndexStoreModule");
}
final Class<? extends Module> indexStoreModule = settings.getAsClass(STORE_TYPE, null, "org.elasticsearch.index.store.", "IndexStoreModule");
return ImmutableList.of(Modules.createModule(indexStoreModule, settings));
}
@Override
protected void configure() {
protected void configure() {}
private static class DefaultStoreModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexStore.class).asEagerSingleton();
}
}
}

63
src/main/java/org/elasticsearch/index/store/fs/DefaultFsDirectoryService.java
View File

@ -1,63 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import com.google.common.collect.Sets;
import org.apache.lucene.store.*;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.IndexStore;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Set;
/**
*/
public class DefaultFsDirectoryService extends FsDirectoryService {
/*
* We are mmapping docvalues as well as term dictionaries, all other files are served through NIOFS
* this provides good random access performance while not creating unnecessary mmaps for files like stored
* fields etc.
*/
private static final Set<String> PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("dvd", "tim"));
@Inject
public DefaultFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath shardPath) {
super(shardId, indexSettings, indexStore, shardPath);
}
@Override
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
final MMapDirectory mmapDir = new MMapDirectory(location, lockFactory);
return new FileSwitchDirectory(PRIMARY_EXTENSIONS, mmapDir, new NIOFSDirectory(location, lockFactory), true) {
@Override
public String[] listAll() throws IOException {
// Avoid doing listAll twice:
return mmapDir.listAll();
}
};
}
}

46
src/main/java/org/elasticsearch/index/store/fs/DefaultFsIndexStore.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.indices.store.IndicesStore;
/**
*
*/
public final class DefaultFsIndexStore extends AbstractIndexStore {
@Inject
public DefaultFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, indicesStore, nodeEnv);
}
@Override
public Class<? extends DirectoryService> shardDirectory() {
return DefaultFsDirectoryService.class;
}
}

34
src/main/java/org/elasticsearch/index/store/fs/DefaultFsIndexStoreModule.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.store.IndexStore;
/**
*
*/
public class DefaultFsIndexStoreModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexStore.class).to(DefaultFsIndexStore.class).asEagerSingleton();
}
}

49
src/main/java/org/elasticsearch/index/store/fs/MmapFsDirectoryService.java
View File

@ -1,49 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.MMapDirectory;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.IndexStore;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
/**
*/
public class MmapFsDirectoryService extends FsDirectoryService {
@Inject
public MmapFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath shardPath) {
super(shardId, indexSettings, indexStore, shardPath);
}
@Override
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
return new MMapDirectory(location, buildLockFactory());
}
}

46
src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.indices.store.IndicesStore;
/**
*
*/
public final class MmapFsIndexStore extends AbstractIndexStore {
@Inject
public MmapFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, indicesStore, nodeEnv);
}
@Override
public Class<? extends DirectoryService> shardDirectory() {
return MmapFsDirectoryService.class;
}
}

34
src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStoreModule.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.store.IndexStore;
/**
*
*/
public class MmapFsIndexStoreModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexStore.class).to(MmapFsIndexStore.class).asEagerSingleton();
}
}

49
src/main/java/org/elasticsearch/index/store/fs/NioFsDirectoryService.java
View File

@ -1,49 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.NIOFSDirectory;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.IndexStore;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
/**
*/
public class NioFsDirectoryService extends FsDirectoryService {
@Inject
public NioFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath shardPath) {
super(shardId, indexSettings, indexStore, shardPath);
}
@Override
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
return new NIOFSDirectory(location, lockFactory);
}
}

46
src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.indices.store.IndicesStore;
/**
*
*/
public final class NioFsIndexStore extends AbstractIndexStore {
@Inject
public NioFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, indicesStore, nodeEnv);
}
@Override
public Class<? extends DirectoryService> shardDirectory() {
return NioFsDirectoryService.class;
}
}

34
src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStoreModule.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.store.IndexStore;
/**
*
*/
public class NioFsIndexStoreModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexStore.class).to(NioFsIndexStore.class).asEagerSingleton();
}
}

49
src/main/java/org/elasticsearch/index/store/fs/SimpleFsDirectoryService.java
View File

@ -1,49 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockFactory;
import org.apache.lucene.store.SimpleFSDirectory;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.IndexStore;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
/**
*/
public class SimpleFsDirectoryService extends FsDirectoryService {
@Inject
public SimpleFsDirectoryService(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ShardPath shardPath) {
super(shardId, indexSettings, indexStore, shardPath);
}
@Override
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
return new SimpleFSDirectory(location, lockFactory);
}
}

46
src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java
View File

@ -1,46 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.indices.store.IndicesStore;
/**
*
*/
public final class SimpleFsIndexStore extends AbstractIndexStore {
@Inject
public SimpleFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, indicesStore, nodeEnv);
}
@Override
public Class<? extends DirectoryService> shardDirectory() {
return SimpleFsDirectoryService.class;
}
}

34
src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStoreModule.java
View File

@ -1,34 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.index.store.IndexStore;
/**
*
*/
public class SimpleFsIndexStoreModule extends AbstractModule {
@Override
protected void configure() {
bind(IndexStore.class).to(SimpleFsIndexStore.class).asEagerSingleton();
}
}

126
src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java
View File

@ -1,126 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.support;
import org.apache.lucene.store.StoreRateLimiting;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchIllegalStateException;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.indices.store.IndicesStore;
import java.io.IOException;
import java.nio.file.Path;
/**
*
*/
public abstract class AbstractIndexStore extends AbstractIndexComponent implements IndexStore {
public static final String INDEX_STORE_THROTTLE_TYPE = "index.store.throttle.type";
public static final String INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC = "index.store.throttle.max_bytes_per_sec";
class ApplySettings implements IndexSettingsService.Listener {
@Override
public void onRefreshSettings(Settings settings) {
String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, AbstractIndexStore.this.rateLimitingType);
if (!rateLimitingType.equals(AbstractIndexStore.this.rateLimitingType)) {
logger.info("updating index.store.throttle.type from [{}] to [{}]", AbstractIndexStore.this.rateLimitingType, rateLimitingType);
if (rateLimitingType.equalsIgnoreCase("node")) {
AbstractIndexStore.this.rateLimitingType = rateLimitingType;
AbstractIndexStore.this.nodeRateLimiting = true;
} else {
StoreRateLimiting.Type.fromString(rateLimitingType);
AbstractIndexStore.this.rateLimitingType = rateLimitingType;
AbstractIndexStore.this.nodeRateLimiting = false;
AbstractIndexStore.this.rateLimiting.setType(rateLimitingType);
}
}
ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, AbstractIndexStore.this.rateLimitingThrottle);
if (!rateLimitingThrottle.equals(AbstractIndexStore.this.rateLimitingThrottle)) {
logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", AbstractIndexStore.this.rateLimitingThrottle, rateLimitingThrottle, AbstractIndexStore.this.rateLimitingType);
AbstractIndexStore.this.rateLimitingThrottle = rateLimitingThrottle;
AbstractIndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle);
}
}
}
private final NodeEnvironment nodeEnv;
private final Path[] locations;
protected final IndexService indexService;
protected final IndicesStore indicesStore;
private volatile String rateLimitingType;
private volatile ByteSizeValue rateLimitingThrottle;
private volatile boolean nodeRateLimiting;
private final StoreRateLimiting rateLimiting = new StoreRateLimiting();
private final ApplySettings applySettings = new ApplySettings();
protected AbstractIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings);
this.indexService = indexService;
this.indicesStore = indicesStore;
this.rateLimitingType = indexSettings.get(INDEX_STORE_THROTTLE_TYPE, "none");
if (rateLimitingType.equalsIgnoreCase("node")) {
nodeRateLimiting = true;
} else {
nodeRateLimiting = false;
rateLimiting.setType(rateLimitingType);
}
this.rateLimitingThrottle = indexSettings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0));
rateLimiting.setMaxRate(rateLimitingThrottle);
logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle);
indexService.settingsService().addListener(applySettings);
this.nodeEnv = nodeEnv;
if (nodeEnv.hasNodeFile()) {
this.locations = nodeEnv.indexPaths(index);
} else {
this.locations = null;
}
}
@Override
public void close() throws ElasticsearchException {
indexService.settingsService().removeListener(applySettings);
}
@Override
public StoreRateLimiting rateLimiting() {
return nodeRateLimiting ? indicesStore.rateLimiting() : this.rateLimiting;
}
}

6
src/main/java/org/elasticsearch/indices/store/IndicesStore.java
View File

@ -89,8 +89,6 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
}
}
private final NodeEnvironment nodeEnv;
private final NodeSettingsService nodeSettingsService;
private final IndicesService indicesService;
@ -107,10 +105,9 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
private TimeValue deleteShardTimeout;
@Inject
public IndicesStore(Settings settings, NodeEnvironment nodeEnv, NodeSettingsService nodeSettingsService, IndicesService indicesService,
public IndicesStore(Settings settings, NodeSettingsService nodeSettingsService, IndicesService indicesService,
ClusterService clusterService, TransportService transportService) {
super(settings);
this.nodeEnv = nodeEnv;
this.nodeSettingsService = nodeSettingsService;
this.indicesService = indicesService;
this.clusterService = clusterService;
@ -133,7 +130,6 @@ public class IndicesStore extends AbstractComponent implements ClusterStateListe
IndicesStore() {
super(ImmutableSettings.EMPTY);
nodeEnv = null;
nodeSettingsService = null;
indicesService = null;
this.clusterService = null;

3
src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetaData.java
View File

@ -43,6 +43,7 @@ import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.StoreFileMetaData;
import org.elasticsearch.indices.IndicesService;
@ -168,7 +169,7 @@ public class TransportNodesListShardStoreMetaData extends TransportNodesOperatio
if (metaData == null) {
return new StoreFilesMetaData(false, shardId, ImmutableMap.<String, StoreFileMetaData>of());
}
String storeType = metaData.settings().get("index.store.type", "fs");
String storeType = metaData.settings().get(IndexStoreModule.STORE_TYPE, "fs");
if (!storeType.contains("fs")) {
return new StoreFilesMetaData(false, shardId, ImmutableMap.<String, StoreFileMetaData>of());
}

2
src/test/java/org/elasticsearch/client/transport/TransportClientTests.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.node.Node;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope;
@ -57,7 +58,6 @@ public class TransportClientTests extends ElasticsearchIntegrationTest {
.put("path.home", createTempDir())
.put("node.name", "testNodeVersionIsUpdated")
.put("http.enabled", false)
.put("index.store.type", "ram")
.put("config.ignore_system_properties", true) // make sure we get what we set :)
.build()).clusterName("foobar").build();
node.start();

86
src/test/java/org/elasticsearch/index/store/IndexStoreTests.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.store.*;
import org.apache.lucene.util.Constants;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.test.ElasticsearchTestCase;
import java.io.IOException;
import java.nio.file.Path;
/**
*/
public class IndexStoreTests extends ElasticsearchTestCase {
public void testStoreDirectory() throws IOException {
final Path tempDir = createTempDir();
final IndexStoreModule.Type[] values = IndexStoreModule.Type.values();
final IndexStoreModule.Type type = RandomPicks.randomFrom(random(), values);
Settings settings = ImmutableSettings.settingsBuilder().put(IndexStoreModule.STORE_TYPE, type.name()).build();
FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
switch (type) {
case NIOFS:
assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
break;
case MMAPFS:
assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
break;
case SIMPLEFS:
assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
break;
case FS:
case DEFAULT:
if (Constants.WINDOWS) {
if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
} else {
assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
}
} else if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
assertTrue(type + " " + directory.toString(), directory instanceof FileSwitchDirectory);
} else {
assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
}
break;
}
}
}
public void testStoreDirectoryDefault() throws IOException {
final Path tempDir = createTempDir();
Settings settings = ImmutableSettings.EMPTY;
FsDirectoryService service = new FsDirectoryService(settings, null, new ShardPath(tempDir, tempDir, "foo", new ShardId("foo", 0)));
try (final Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
if (Constants.WINDOWS) {
assertTrue(directory.toString(), directory instanceof MMapDirectory || directory instanceof SimpleFSDirectory);
} else {
assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
}
}
}
}

13
src/test/java/org/elasticsearch/indices/settings/UpdateSettingsTests.java
View File

@ -23,7 +23,6 @@ import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.node.stats.NodeStats;
@ -37,8 +36,8 @@ import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.merge.policy.TieredMergePolicyProvider;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.merge.scheduler.MergeSchedulerModule;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;
@ -137,7 +136,7 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest {
// No throttling at first, only 1 non-replicated shard, force lots of merging:
assertAcked(prepareCreate("test")
.setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "none")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "none")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
@ -175,13 +174,13 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest {
.indices()
.prepareUpdateSettings("test")
.setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "1mb"))
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, "1mb"))
.get();
// Make sure setting says it is in fact changed:
GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test").get();
assertThat(getSettingsResponse.getSetting("test", AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE), equalTo("merge"));
assertThat(getSettingsResponse.getSetting("test", IndexStore.INDEX_STORE_THROTTLE_TYPE), equalTo("merge"));
// Also make sure we see throttling kicking in:
boolean done = false;
@ -215,7 +214,7 @@ public class UpdateSettingsTests extends ElasticsearchIntegrationTest {
.indices()
.prepareUpdateSettings("test")
.setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "none"))
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "none"))
.get();
// Optimize does a waitForMerges, which we must do to make sure all in-flight (throttled) merges finish:

6
src/test/java/org/elasticsearch/indices/stats/IndexStatsTests.java
View File

@ -44,7 +44,7 @@ import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
import org.elasticsearch.index.merge.policy.TieredMergePolicyProvider;
import org.elasticsearch.index.merge.scheduler.ConcurrentMergeSchedulerProvider;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.indices.cache.query.IndicesQueryCache;
import org.elasticsearch.search.sort.SortOrder;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
@ -308,7 +308,7 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest {
public void nonThrottleStats() throws Exception {
assertAcked(prepareCreate("test")
.setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")
@ -341,7 +341,7 @@ public class IndexStatsTests extends ElasticsearchIntegrationTest {
public void throttleStats() throws Exception {
assertAcked(prepareCreate("test")
.setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "merge")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0")
.put(TieredMergePolicyProvider.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE, "2")

10
src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreTests.java
View File

@ -21,11 +21,9 @@ package org.elasticsearch.snapshots;
import com.carrotsearch.hppc.IntOpenHashSet;
import com.carrotsearch.hppc.IntSet;
import com.carrotsearch.randomizedtesting.LifecycleScope;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.ListenableActionFuture;
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
@ -52,7 +50,7 @@ import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.discovery.zen.elect.ElectMasterService;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.indices.ttl.IndicesTTLService;
import org.elasticsearch.repositories.RepositoryMissingException;
import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
@ -699,7 +697,7 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests
logger.info("--> update index settings to back to normal");
assertAcked(client().admin().indices().prepareUpdateSettings("test-*").setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "node")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "node")
));
// Make sure that snapshot finished - doesn't matter if it failed or succeeded
@ -745,8 +743,8 @@ public class DedicatedClusterSnapshotRestoreTests extends AbstractSnapshotTests
}
assertAcked(client().admin().indices().prepareUpdateSettings(name).setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "all")
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, between(100, 50000))
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "all")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, between(100, 50000))
));
}

9
src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreTests.java
View File

@ -52,11 +52,10 @@ import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.indices.InvalidIndexNameException;
import org.elasticsearch.repositories.RepositoriesService;
import org.elasticsearch.snapshots.mockstore.MockRepositoryModule;
import org.elasticsearch.test.junit.annotations.TestLogging;
import org.junit.Test;
import java.nio.channels.SeekableByteChannel;
@ -1486,8 +1485,8 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
// Update settings to make sure that relocation is slow so we can start snapshot before relocation is finished
assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "all")
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, 100)
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "all")
.put(IndexStore.INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, 100)
));
logger.info("--> start relocations");
@ -1502,7 +1501,7 @@ public class SharedClusterSnapshotRestoreTests extends AbstractSnapshotTests {
// Update settings to back to normal
assertAcked(client.admin().indices().prepareUpdateSettings("test-idx").setSettings(ImmutableSettings.builder()
.put(AbstractIndexStore.INDEX_STORE_THROTTLE_TYPE, "node")
.put(IndexStore.INDEX_STORE_THROTTLE_TYPE, "node")
));
logger.info("--> wait for snapshot to complete");

3
src/test/java/org/elasticsearch/test/InternalTestCluster.java
View File

@ -83,6 +83,7 @@ import org.elasticsearch.index.cache.filter.none.NoneFilterCache;
import org.elasticsearch.index.cache.filter.weighted.WeightedFilterCache;
import org.elasticsearch.index.shard.IndexShardModule;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.IndexStoreModule;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.indices.breaker.CircuitBreakerService;
import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService;
@ -383,7 +384,7 @@ public final class InternalTestCluster extends TestCluster {
.put("cluster.routing.schedule", (30 + random.nextInt(50)) + "ms")
.put(SETTING_CLUSTER_NODE_SEED, seed);
if (ENABLE_MOCK_MODULES && usually(random)) {
builder.put("index.store.type", MockFSIndexStoreModule.class.getName()); // no RAM dir for now!
builder.put(IndexStoreModule.STORE_TYPE, MockFSIndexStoreModule.class.getName()); // no RAM dir for now!
builder.put(IndexShardModule.ENGINE_FACTORY, MockEngineFactory.class);
builder.put(PageCacheRecyclerModule.CACHE_IMPL, MockPageCacheRecyclerModule.class.getName());
builder.put(BigArraysModule.IMPL, MockBigArraysModule.class.getName());

31
src/test/java/org/elasticsearch/test/store/MockDirectoryHelper.java
View File

@ -19,22 +19,21 @@
package org.elasticsearch.test.store;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FilterDirectory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase;
import org.elasticsearch.common.logging.ESLogger;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ConcurrentCollections;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.ShardPath;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.FsDirectoryService;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.fs.*;
import org.elasticsearch.index.store.IndexStoreModule;
import com.carrotsearch.randomizedtesting.SeedUtils;
import java.io.IOException;
@ -98,24 +97,10 @@ public class MockDirectoryHelper {
}
public FsDirectoryService randomDirectorService(IndexStore indexStore, ShardPath path) {
if ((Constants.WINDOWS || Constants.SUN_OS) && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
return new MmapFsDirectoryService(shardId, indexSettings, indexStore, path);
} else if (Constants.WINDOWS) {
return new SimpleFsDirectoryService(shardId, indexSettings, indexStore, path);
}
switch (random.nextInt(4)) {
case 2:
return new DefaultFsDirectoryService(shardId, indexSettings, indexStore, path);
case 1:
return new MmapFsDirectoryService(shardId, indexSettings, indexStore, path);
case 0:
if (random.nextInt(10) == 0) {
// use simplefs less, it synchronizes all threads reads
return new SimpleFsDirectoryService(shardId, indexSettings, indexStore, path);
}
default:
return new NioFsDirectoryService(shardId, indexSettings, indexStore, path);
}
ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder();
builder.put(indexSettings);
builder.put(IndexStoreModule.STORE_TYPE, RandomPicks.randomFrom(random, IndexStoreModule.Type.values()));
return new FsDirectoryService(builder.build(), indexStore, path);
}
public static final class ElasticsearchMockDirectoryWrapper extends MockDirectoryWrapper {

6
src/test/java/org/elasticsearch/test/store/MockFSDirectoryService.java
View File

@ -37,7 +37,7 @@ import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.*;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.fs.FsDirectoryService;
import org.elasticsearch.index.store.FsDirectoryService;
import org.elasticsearch.indices.IndicesLifecycle;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.test.ElasticsearchTestCase;
@ -62,8 +62,8 @@ public class MockFSDirectoryService extends FsDirectoryService {
private final boolean checkIndexOnClose;
@Inject
public MockFSDirectoryService(final ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, final IndicesService service, final ShardPath path) {
super(shardId, indexSettings, indexStore, path);
public MockFSDirectoryService(@IndexSettings Settings indexSettings, IndexStore indexStore, final IndicesService service, final ShardPath path) {
super(indexSettings, indexStore, path);
final long seed = indexSettings.getAsLong(ElasticsearchIntegrationTest.SETTING_INDEX_SEED, 0l);
Random random = new Random(seed);
helper = new MockDirectoryHelper(shardId, indexSettings, logger, random, seed);

11
src/test/java/org/elasticsearch/test/store/MockFSIndexStore.java
View File

@ -25,16 +25,17 @@ import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.settings.IndexSettingsService;
import org.elasticsearch.index.store.DirectoryService;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.indices.store.IndicesStore;
public class MockFSIndexStore extends AbstractIndexStore {
public class MockFSIndexStore extends IndexStore {
@Inject
public MockFSIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService,
IndicesStore indicesStore, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, indicesStore, nodeEnv);
public MockFSIndexStore(Index index, @IndexSettings Settings indexSettings, IndexSettingsService indexSettingsService,
IndicesStore indicesStore) {
super(index, indexSettings, indexSettingsService, indicesStore);
}
@Override