honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use directly the directory and not the metaData API

This commit is contained in:
kimchy 2010-09-24 11:36:13 +02:00
parent 9bb86ea865
commit 2f43af0bd7
4 changed files with 37 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java
View File

@ -43,7 +43,6 @@ import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.index.shard.service.IndexShard;
import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.Store;
import org.elasticsearch.index.store.StoreFileMetaData;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.index.translog.TranslogStreams;
import org.elasticsearch.threadpool.ThreadPool;
@ -182,9 +181,9 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
int indexNumberOfFiles = 0;
long indexTotalFilesSize = 0;
for (final String fileName : snapshotIndexCommit.getFiles()) {
StoreFileMetaData storeMetaData;
long fileLength = 0;
try {
storeMetaData = store.metaData(fileName);
fileLength = store.directory().fileLength(fileName);
} catch (IOException e) {
throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to get store file metadata", e);
}
@ -195,17 +194,17 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
}
CommitPoint.FileInfo fileInfo = commitPoints.findPhysicalIndexFile(fileName);
if (fileInfo == null || fileInfo.length() != storeMetaData.length() || !commitPointFileExistsInBlobs(fileInfo, blobs)) {
if (fileInfo == null || fileInfo.length() != fileLength || !commitPointFileExistsInBlobs(fileInfo, blobs)) {
// commit point file does not exists in any commit point, or has different length, or does not fully exists in the listed blobs
snapshotRequired = true;
}
if (snapshotRequired) {
indexNumberOfFiles++;
indexTotalFilesSize += storeMetaData.length();
indexTotalFilesSize += fileLength;
// create a new FileInfo
try {
CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo(fileNameFromGeneration(++generation), storeMetaData.name(), storeMetaData.length());
CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo(fileNameFromGeneration(++generation), fileName, fileLength);
indexCommitPointFiles.add(snapshotFileInfo);
snapshotFile(snapshotIndexCommit.getDirectory(), snapshotFileInfo, indexLatch, failures);
} catch (IOException e) {
@ -520,21 +519,27 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo
List<CommitPoint.FileInfo> filesToRecover = Lists.newArrayList();
for (CommitPoint.FileInfo fileInfo : commitPoint.indexFiles()) {
StoreFileMetaData storeFile = store.metaData(fileInfo.physicalName());
if (storeFile != null && !storeFile.name().contains("segment") && storeFile.length() == fileInfo.length()) {
String fileName = fileInfo.physicalName();
long fileLength = -1;
try {
fileLength = store.directory().fileLength(fileName);
} catch (Exception e) {
// no file
}
if (!fileName.contains("segment") && fileLength == fileInfo.length()) {
numberOfFiles++;
totalSize += storeFile.length();
totalSize += fileLength;
numberOfReusedFiles++;
reusedTotalSize += storeFile.length();
reusedTotalSize += fileLength;
if (logger.isTraceEnabled()) {
logger.trace("not_recovering [{}], exists in local store and has same length [{}]", fileInfo.physicalName(), fileInfo.length());
}
} else {
if (logger.isTraceEnabled()) {
if (storeFile == null) {
if (fileLength == -1) {
logger.trace("recovering [{}], does not exists in local store", fileInfo.physicalName());
} else {
logger.trace("recovering [{}], exists in local store but has different length: gateway [{}], local [{}]", fileInfo.physicalName(), fileInfo.length(), storeFile.length());
logger.trace("recovering [{}], exists in local store but has different length: gateway [{}], local [{}]", fileInfo.physicalName(), fileInfo.length(), fileLength);
}
}
numberOfFiles++;

17
modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/recovery/RecoverySource.java
View File

@ -35,7 +35,6 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException;
import org.elasticsearch.index.shard.IndexShardClosedException;
import org.elasticsearch.index.shard.IndexShardState;
import org.elasticsearch.index.shard.service.InternalIndexShard;
import org.elasticsearch.index.store.StoreFileMetaData;
import org.elasticsearch.index.translog.Translog;
import org.elasticsearch.indices.IndicesService;
import org.elasticsearch.threadpool.ThreadPool;
@ -97,28 +96,28 @@ public class RecoverySource extends AbstractComponent {
StopWatch stopWatch = new StopWatch().start();
for (String name : snapshot.getFiles()) {
StoreFileMetaData md = shard.store().metaData(name);
long length = shard.store().directory().fileLength(name);
boolean useExisting = false;
if (request.existingFiles().containsKey(name)) {
if (!md.name().contains("segment") && md.length() == request.existingFiles().get(name).length()) {
if (!name.contains("segment") && length == request.existingFiles().get(name).length()) {
response.phase1ExistingFileNames.add(name);
response.phase1ExistingFileSizes.add(md.length());
existingTotalSize += md.length();
response.phase1ExistingFileSizes.add(length);
existingTotalSize += length;
useExisting = true;
if (logger.isTraceEnabled()) {
logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has size [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, md.length());
logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has size [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, length);
}
}
}
if (!useExisting) {
if (request.existingFiles().containsKey(name)) {
logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but has different length: remote [{}], local [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, request.existingFiles().get(name).length(), md.length());
logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but has different length: remote [{}], local [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, request.existingFiles().get(name).length(), length);
} else {
logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], does not exists in remote", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name);
}
response.phase1FileNames.add(name);
response.phase1FileSizes.add(md.length());
totalSize += md.length();
response.phase1FileSizes.add(length);
totalSize += length;
}
}
response.phase1TotalSize = totalSize;

2
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java
View File

@ -36,8 +36,6 @@ public interface Store extends IndexShardComponent {
*/
Directory directory();
StoreFileMetaData metaData(String name) throws IOException;
ImmutableMap<String, StoreFileMetaData> list() throws IOException;
/**

24
modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java
View File

@ -62,18 +62,6 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
return new StoreDirectory(dir);
}
@Override public StoreFileMetaData metaData(String name) throws IOException {
StoreFileMetaData md = filesMetadata.get(name);
if (md == null) {
return null;
}
// IndexOutput not closed, does not exists
if (md.lastModified() == -1 || md.length() == -1) {
return null;
}
return md;
}
@Override public ImmutableMap<String, StoreFileMetaData> list() throws IOException {
ImmutableMap.Builder<String, StoreFileMetaData> builder = ImmutableMap.builder();
for (String name : files) {
@ -85,6 +73,18 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen
return builder.build();
}
private StoreFileMetaData metaData(String name) throws IOException {
StoreFileMetaData md = filesMetadata.get(name);
if (md == null) {
return null;
}
// IndexOutput not closed, does not exists
if (md.lastModified() == -1 || md.length() == -1) {
return null;
}
return md;
}
@Override public void deleteContent() throws IOException {
Directories.deleteFiles(directory());
}