honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more apis on the index level store to help clean unused work shards when needed

This commit is contained in:
kimchy 2010-06-25 03:39:42 +03:00
parent 811856e1f8
commit 83010f7ee1
11 changed files with 164 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
modules/benchmark/micro/src/main/java/org/elasticsearch/benchmark/index/store/SimpleStoreBenchmark.java
View File

@ -271,11 +271,11 @@ public class SimpleStoreBenchmark {
if (type.equalsIgnoreCase("ram")) {
store = new RamStore(shardId, settings);
} else if (type.equalsIgnoreCase("simple-fs")) {
store = new SimpleFsStore(shardId, settings, new SimpleFsIndexStore(shardId.index(), settings, nodeEnvironment));
store = new SimpleFsStore(shardId, settings, new SimpleFsIndexStore(shardId.index(), settings, null, nodeEnvironment));
} else if (type.equalsIgnoreCase("mmap-fs")) {
store = new NioFsStore(shardId, settings, new NioFsIndexStore(shardId.index(), settings, nodeEnvironment));
store = new NioFsStore(shardId, settings, new NioFsIndexStore(shardId.index(), settings, null, nodeEnvironment));
} else if (type.equalsIgnoreCase("nio-fs")) {
store = new MmapFsStore(shardId, settings, new MmapFsIndexStore(shardId.index(), settings, nodeEnvironment));
store = new MmapFsStore(shardId, settings, new MmapFsIndexStore(shardId.index(), settings, null, nodeEnvironment));
} else if (type.equalsIgnoreCase("memory-direct")) {
Settings byteBufferSettings = settingsBuilder()
.put(settings)

2
modules/elasticsearch/src/main/java/org/elasticsearch/action/search/ShardSearchFailure.java
View File

@ -96,7 +96,7 @@ public class ShardSearchFailure implements ShardOperationFailedException {
}
@Override public String toString() {
return "Search Failure Shard " + shardTarget + ", reason [" + reason + "]";
return "shard [" + (shardTarget == null ? "_na" : shardTarget) + "], reason [" + reason + "]";
}
public static ShardSearchFailure readShardSearchFailure(StreamInput in) throws IOException {

51
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/IndexStore.java
View File

@ -21,6 +21,11 @@ package org.elasticsearch.index.store;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.IndexComponent;
import org.elasticsearch.index.shard.ShardId;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
/**
* Index store is an index level information of the {@link Store} each shard will use.
@ -48,4 +53,50 @@ public interface IndexStore extends IndexComponent {
* Returns the backing store free space. Return <tt>-1</tt> if not available.
*/
ByteSizeValue backingStoreFreeSpace();
/**
* Lists the store files metadata for a shard. Note, this should be able to list also
* metadata for shards that are no allocated as well.
*/
StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException;
static class StoreFilesMetaData implements Iterable<StoreFileMetaData> {
private final boolean allocated;
private final Map<String, StoreFileMetaData> files;
public StoreFilesMetaData(boolean allocated, Map<String, StoreFileMetaData> files) {
this.allocated = allocated;
this.files = files;
}
public boolean allocated() {
return allocated;
}
@Override public Iterator<StoreFileMetaData> iterator() {
return files.values().iterator();
}
public StoreFileMetaData file(String name) {
return files.get(name);
}
}
static class StoreFileMetaData {
private final String name;
private final long sizeInBytes;
public StoreFileMetaData(String name, long sizeInBytes) {
this.name = name;
this.sizeInBytes = sizeInBytes;
}
public String name() {
return name;
}
public long sizeInBytes() {
return sizeInBytes;
}
}
}

26
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/FsIndexStore.java
View File

@ -19,26 +19,30 @@
package org.elasticsearch.index.store.fs;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import java.io.File;
import java.io.IOException;
import java.util.Map;
/**
* @author kimchy (shay.banon)
*/
public abstract class FsIndexStore extends AbstractIndexComponent implements IndexStore {
public abstract class FsIndexStore extends AbstractIndexStore {
private final File location;
public FsIndexStore(Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) {
super(index, indexSettings);
public FsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService);
this.location = new File(new File(nodeEnv.nodeFile(), "indices"), index.name());
if (!location.exists()) {
@ -70,6 +74,18 @@ public abstract class FsIndexStore extends AbstractIndexComponent implements Ind
return new ByteSizeValue(usableSpace);
}
@Override protected StoreFilesMetaData listUnallocatedStoreMetaData(ShardId shardId) throws IOException {
File shardIndexLocation = shardIndexLocation(shardId);
if (!shardIndexLocation.exists()) {
return new StoreFilesMetaData(false, ImmutableMap.<String, StoreFileMetaData>of());
}
Map<String, StoreFileMetaData> files = Maps.newHashMap();
for (File file : shardIndexLocation.listFiles()) {
files.put(file.getName(), new StoreFileMetaData(file.getName(), file.length()));
}
return new StoreFilesMetaData(false, files);
}
public File location() {
return location;
}

5
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/MmapFsIndexStore.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.Store;
@ -31,8 +32,8 @@ import org.elasticsearch.index.store.Store;
*/
public class MmapFsIndexStore extends FsIndexStore {
@Inject public MmapFsIndexStore(Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) {
super(index, indexSettings, nodeEnv);
@Inject public MmapFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, nodeEnv);
}
@Override public Class<? extends Store> shardStoreClass() {

5
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/NioFsIndexStore.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.Store;
@ -31,8 +32,8 @@ import org.elasticsearch.index.store.Store;
*/
public class NioFsIndexStore extends FsIndexStore {
@Inject public NioFsIndexStore(Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) {
super(index, indexSettings, nodeEnv);
@Inject public NioFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, nodeEnv);
}
@Override public Class<? extends Store> shardStoreClass() {

5
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/fs/SimpleFsIndexStore.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.Store;
@ -31,8 +32,8 @@ import org.elasticsearch.index.store.Store;
*/
public class SimpleFsIndexStore extends FsIndexStore {
@Inject public SimpleFsIndexStore(Index index, @IndexSettings Settings indexSettings, NodeEnvironment nodeEnv) {
super(index, indexSettings, nodeEnv);
@Inject public SimpleFsIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService, NodeEnvironment nodeEnv) {
super(index, indexSettings, indexService, nodeEnv);
}
@Override public Class<? extends Store> shardStoreClass() {

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/ByteBufferIndexStore.java
View File

@ -23,23 +23,23 @@ import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.jvm.JvmStats;
/**
* @author kimchy (shay.banon)
*/
public class ByteBufferIndexStore extends AbstractIndexComponent implements IndexStore {
public class ByteBufferIndexStore extends AbstractIndexStore {
private final boolean direct;
@Inject public ByteBufferIndexStore(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
@Inject public ByteBufferIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService) {
super(index, indexSettings, indexService);
this.direct = componentSettings.getAsBoolean("direct", true);
}

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/memory/HeapIndexStore.java
View File

@ -22,21 +22,21 @@ package org.elasticsearch.index.store.memory;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.jvm.JvmStats;
/**
* @author kimchy (shay.banon)
*/
public class HeapIndexStore extends AbstractIndexComponent implements IndexStore {
public class HeapIndexStore extends AbstractIndexStore {
@Inject public HeapIndexStore(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
@Inject public HeapIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService) {
super(index, indexSettings, indexService);
}
@Override public boolean persistent() {

10
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/ram/RamIndexStore.java
View File

@ -22,21 +22,21 @@ package org.elasticsearch.index.store.ram;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.store.IndexStore;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.support.AbstractIndexStore;
import org.elasticsearch.monitor.jvm.JvmInfo;
import org.elasticsearch.monitor.jvm.JvmStats;
/**
* @author kimchy (shay.banon)
*/
public class RamIndexStore extends AbstractIndexComponent implements IndexStore {
public class RamIndexStore extends AbstractIndexStore {
@Inject public RamIndexStore(Index index, @IndexSettings Settings indexSettings) {
super(index, indexSettings);
@Inject public RamIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService) {
super(index, indexSettings, indexService);
}
@Override public boolean persistent() {

64
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractIndexStore.java Normal file
View File

@ -0,0 +1,64 @@
/*
* Licensed to Elastic Search and Shay Banon under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. Elastic Search licenses this
* file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.store.support;
import org.elasticsearch.common.collect.ImmutableMap;
import org.elasticsearch.common.collect.Maps;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.AbstractIndexComponent;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.service.IndexService;
import org.elasticsearch.index.settings.IndexSettings;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.IndexStore;
import java.io.IOException;
import java.util.Map;
/**
* @author kimchy (shay.banon)
*/
public abstract class AbstractIndexStore extends AbstractIndexComponent implements IndexStore {
private final IndexService indexService;
protected AbstractIndexStore(Index index, @IndexSettings Settings indexSettings, IndexService indexService) {
super(index, indexSettings);
this.indexService = indexService;
}
@Override public StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException {
InternalIndexShard indexShard = (InternalIndexShard) indexService.shard(shardId.id());
if (indexShard == null) {
return listUnallocatedStoreMetaData(shardId);
} else {
Map<String, StoreFileMetaData> files = Maps.newHashMap();
for (String file : indexShard.store().directory().listAll()) {
files.put(file, new StoreFileMetaData(file, indexShard.store().directory().fileLength(file)));
}
return new StoreFilesMetaData(true, files);
}
}
protected StoreFilesMetaData listUnallocatedStoreMetaData(ShardId shardId) throws IOException {
return new StoreFilesMetaData(false, ImmutableMap.<String, StoreFileMetaData>of());
}
}