honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use directly the directory and not the metaData API

This commit is contained in:
kimchy 2010-09-24 11:36:13 +02:00
parent 9bb86ea865
commit 2f43af0bd7
4 changed files with 37 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java
View File

@ -43,7 +43,6 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.index.shard.service.InternalIndexShard; import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.Store; import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.StoreFileMetaData;
import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.index.translog.TranslogStreams; import org.elasticsearch.index.translog.TranslogStreams;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
@ -182,9 +181,9 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
int indexNumberOfFiles = 0; int indexNumberOfFiles = 0;
long indexTotalFilesSize = 0; long indexTotalFilesSize = 0;
for (final String fileName : snapshotIndexCommit.getFiles()) { for (final String fileName : snapshotIndexCommit.getFiles()) {
StoreFileMetaData storeMetaData; long fileLength = 0;
try { try {
storeMetaData = store.metaData(fileName); fileLength = store.directory().fileLength(fileName);
} catch (IOException e) { } catch (IOException e) {
throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to get store file metadata", e); throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to get store file metadata", e);
} }
@ -195,17 +194,17 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
} }
CommitPoint.FileInfo fileInfo = commitPoints.findPhysicalIndexFile(fileName); CommitPoint.FileInfo fileInfo = commitPoints.findPhysicalIndexFile(fileName);
if (fileInfo == null || fileInfo.length() != storeMetaData.length() || !commitPointFileExistsInBlobs(fileInfo, blobs)) { if (fileInfo == null || fileInfo.length() != fileLength || !commitPointFileExistsInBlobs(fileInfo, blobs)) {
// commit point file does not exists in any commit point, or has different length, or does not fully exists in the listed blobs // commit point file does not exists in any commit point, or has different length, or does not fully exists in the listed blobs
snapshotRequired = true; snapshotRequired = true;
} }
if (snapshotRequired) { if (snapshotRequired) {
indexNumberOfFiles++; indexNumberOfFiles++;
indexTotalFilesSize += storeMetaData.length(); indexTotalFilesSize += fileLength;
// create a new FileInfo // create a new FileInfo
try { try {
CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo(fileNameFromGeneration(++generation), storeMetaData.name(), storeMetaData.length()); CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo(fileNameFromGeneration(++generation), fileName, fileLength);
indexCommitPointFiles.add(snapshotFileInfo); indexCommitPointFiles.add(snapshotFileInfo);
snapshotFile(snapshotIndexCommit.getDirectory(), snapshotFileInfo, indexLatch, failures); snapshotFile(snapshotIndexCommit.getDirectory(), snapshotFileInfo, indexLatch, failures);
} catch (IOException e) { } catch (IOException e) {
@ -520,21 +519,27 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
List<CommitPoint.FileInfo> filesToRecover = Lists.newArrayList(); List<CommitPoint.FileInfo> filesToRecover = Lists.newArrayList();
for (CommitPoint.FileInfo fileInfo : commitPoint.indexFiles()) { for (CommitPoint.FileInfo fileInfo : commitPoint.indexFiles()) {
StoreFileMetaData storeFile = store.metaData(fileInfo.physicalName()); String fileName = fileInfo.physicalName();
if (storeFile != null && !storeFile.name().contains("segment") && storeFile.length() == fileInfo.length()) { long fileLength = -1;
try {
fileLength = store.directory().fileLength(fileName);
} catch (Exception e) {
// no file
}
if (!fileName.contains("segment") && fileLength == fileInfo.length()) {
numberOfFiles++; numberOfFiles++;
totalSize += storeFile.length(); totalSize += fileLength;
numberOfReusedFiles++; numberOfReusedFiles++;
reusedTotalSize += storeFile.length(); reusedTotalSize += fileLength;
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("not_recovering [{}], exists in local store and has same length [{}]", fileInfo.physicalName(), fileInfo.length()); logger.trace("not_recovering [{}], exists in local store and has same length [{}]", fileInfo.physicalName(), fileInfo.length());
} }
} else { } else {
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
if (storeFile == null) { if (fileLength == -1) {
logger.trace("recovering [{}], does not exists in local store", fileInfo.physicalName()); logger.trace("recovering [{}], does not exists in local store", fileInfo.physicalName());
} else { } else {
logger.trace("recovering [{}], exists in local store but has different length: gateway [{}], local [{}]", fileInfo.physicalName(), fileInfo.length(), storeFile.length()); logger.trace("recovering [{}], exists in local store but has different length: gateway [{}], local [{}]", fileInfo.physicalName(), fileInfo.length(), fileLength);
} }
} }
numberOfFiles++; numberOfFiles++;

17
modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/recovery/RecoverySource.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException;
import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.IndexShardClosedException;
import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.IndexShardState;
import org.elasticsearch.index.shard.service.InternalIndexShard; import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.StoreFileMetaData;
import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
@ -97,28 +96,28 @@ public class RecoverySource extends AbstractComponent {
StopWatch stopWatch = new StopWatch().start(); StopWatch stopWatch = new StopWatch().start();
for (String name : snapshot.getFiles()) { for (String name : snapshot.getFiles()) {
StoreFileMetaData md = shard.store().metaData(name); long length = shard.store().directory().fileLength(name);
boolean useExisting = false; boolean useExisting = false;
if (request.existingFiles().containsKey(name)) { if (request.existingFiles().containsKey(name)) {
if (!md.name().contains("segment") && md.length() == request.existingFiles().get(name).length()) { if (!name.contains("segment") && length == request.existingFiles().get(name).length()) {
response.phase1ExistingFileNames.add(name); response.phase1ExistingFileNames.add(name);
response.phase1ExistingFileSizes.add(md.length()); response.phase1ExistingFileSizes.add(length);
existingTotalSize += md.length(); existingTotalSize += length;
useExisting = true; useExisting = true;
if (logger.isTraceEnabled()) { if (logger.isTraceEnabled()) {
logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has size [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, md.length()); logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has size [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, length);
} }
} }
} }
if (!useExisting) { if (!useExisting) {
if (request.existingFiles().containsKey(name)) { if (request.existingFiles().containsKey(name)) {
logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but has different length: remote [{}], local [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, request.existingFiles().get(name).length(), md.length()); logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but has different length: remote [{}], local [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, request.existingFiles().get(name).length(), length);
} else { } else {
logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], does not exists in remote", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name); logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], does not exists in remote", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name);
} }
response.phase1FileNames.add(name); response.phase1FileNames.add(name);
response.phase1FileSizes.add(md.length()); response.phase1FileSizes.add(length);
totalSize += md.length(); totalSize += length;
} }
} }
response.phase1TotalSize = totalSize; response.phase1TotalSize = totalSize;

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -36,8 +36,6 @@ public interface Store extends IndexShardComponent {
*/ */
Directory directory(); Directory directory();
StoreFileMetaData metaData(String name) throws IOException;
ImmutableMap<String, StoreFileMetaData> list() throws IOException; ImmutableMap<String, StoreFileMetaData> list() throws IOException;
/** /**

24
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java
View File

@ -62,18 +62,6 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
return new StoreDirectory(dir); return new StoreDirectory(dir);
} }
@Override public StoreFileMetaData metaData(String name) throws IOException {
StoreFileMetaData md = filesMetadata.get(name);
if (md == null) {
return null;
}
// IndexOutput not closed, does not exists
if (md.lastModified() == -1 || md.length() == -1) {
return null;
}
return md;
}
@Override public ImmutableMap<String, StoreFileMetaData> list() throws IOException { @Override public ImmutableMap<String, StoreFileMetaData> list() throws IOException {
ImmutableMap.Builder<String, StoreFileMetaData> builder = ImmutableMap.builder(); ImmutableMap.Builder<String, StoreFileMetaData> builder = ImmutableMap.builder();
for (String name : files) { for (String name : files) {
@ -85,6 +73,18 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
return builder.build(); return builder.build();
} }
private StoreFileMetaData metaData(String name) throws IOException {
StoreFileMetaData md = filesMetadata.get(name);
if (md == null) {
return null;
}
// IndexOutput not closed, does not exists
if (md.lastModified() == -1 || md.length() == -1) {
return null;
}
return md;
}
@Override public void deleteContent() throws IOException { @Override public void deleteContent() throws IOException {
Directories.deleteFiles(directory()); Directories.deleteFiles(directory());
} }