From 2f43af0bd7c442853c19f83d6cd15a5303923ff9 Mon Sep 17 00:00:00 2001 From: kimchy Date: Fri, 24 Sep 2010 11:36:13 +0200 Subject: [PATCH] use directly the directory and not the metaData API --- .../blobstore/BlobStoreIndexShardGateway.java | 29 +++++++++++-------- .../index/shard/recovery/RecoverySource.java | 17 +++++------ .../org/elasticsearch/index/store/Store.java | 2 -- .../index/store/support/AbstractStore.java | 24 +++++++-------- 4 files changed, 37 insertions(+), 35 deletions(-) diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java index 3b0d1487a86..d95049cb3f5 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/gateway/blobstore/BlobStoreIndexShardGateway.java @@ -43,7 +43,6 @@ import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.service.IndexShard; import org.elasticsearch.index.shard.service.InternalIndexShard; import org.elasticsearch.index.store.Store; -import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogStreams; import org.elasticsearch.threadpool.ThreadPool; @@ -182,9 +181,9 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo int indexNumberOfFiles = 0; long indexTotalFilesSize = 0; for (final String fileName : snapshotIndexCommit.getFiles()) { - StoreFileMetaData storeMetaData; + long fileLength = 0; try { - storeMetaData = store.metaData(fileName); + fileLength = store.directory().fileLength(fileName); } catch (IOException e) { throw new IndexShardGatewaySnapshotFailedException(shardId, "Failed to get store file metadata", e); } @@ -195,17 +194,17 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo } CommitPoint.FileInfo fileInfo = commitPoints.findPhysicalIndexFile(fileName); - if (fileInfo == null || fileInfo.length() != storeMetaData.length() || !commitPointFileExistsInBlobs(fileInfo, blobs)) { + if (fileInfo == null || fileInfo.length() != fileLength || !commitPointFileExistsInBlobs(fileInfo, blobs)) { // commit point file does not exists in any commit point, or has different length, or does not fully exists in the listed blobs snapshotRequired = true; } if (snapshotRequired) { indexNumberOfFiles++; - indexTotalFilesSize += storeMetaData.length(); + indexTotalFilesSize += fileLength; // create a new FileInfo try { - CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo(fileNameFromGeneration(++generation), storeMetaData.name(), storeMetaData.length()); + CommitPoint.FileInfo snapshotFileInfo = new CommitPoint.FileInfo(fileNameFromGeneration(++generation), fileName, fileLength); indexCommitPointFiles.add(snapshotFileInfo); snapshotFile(snapshotIndexCommit.getDirectory(), snapshotFileInfo, indexLatch, failures); } catch (IOException e) { @@ -520,21 +519,27 @@ public abstract class BlobStoreIndexShardGateway extends AbstractIndexShardCompo List filesToRecover = Lists.newArrayList(); for (CommitPoint.FileInfo fileInfo : commitPoint.indexFiles()) { - StoreFileMetaData storeFile = store.metaData(fileInfo.physicalName()); - if (storeFile != null && !storeFile.name().contains("segment") && storeFile.length() == fileInfo.length()) { + String fileName = fileInfo.physicalName(); + long fileLength = -1; + try { + fileLength = store.directory().fileLength(fileName); + } catch (Exception e) { + // no file + } + if (!fileName.contains("segment") && fileLength == fileInfo.length()) { numberOfFiles++; - totalSize += storeFile.length(); + totalSize += fileLength; numberOfReusedFiles++; - reusedTotalSize += storeFile.length(); + reusedTotalSize += fileLength; if (logger.isTraceEnabled()) { logger.trace("not_recovering [{}], exists in local store and has same length [{}]", fileInfo.physicalName(), fileInfo.length()); } } else { if (logger.isTraceEnabled()) { - if (storeFile == null) { + if (fileLength == -1) { logger.trace("recovering [{}], does not exists in local store", fileInfo.physicalName()); } else { - logger.trace("recovering [{}], exists in local store but has different length: gateway [{}], local [{}]", fileInfo.physicalName(), fileInfo.length(), storeFile.length()); + logger.trace("recovering [{}], exists in local store but has different length: gateway [{}], local [{}]", fileInfo.physicalName(), fileInfo.length(), fileLength); } } numberOfFiles++; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/recovery/RecoverySource.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/recovery/RecoverySource.java index 3b541c8cd9b..93082a184be 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/recovery/RecoverySource.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/shard/recovery/RecoverySource.java @@ -35,7 +35,6 @@ import org.elasticsearch.index.shard.IllegalIndexShardStateException; import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.IndexShardState; import org.elasticsearch.index.shard.service.InternalIndexShard; -import org.elasticsearch.index.store.StoreFileMetaData; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.threadpool.ThreadPool; @@ -97,28 +96,28 @@ public class RecoverySource extends AbstractComponent { StopWatch stopWatch = new StopWatch().start(); for (String name : snapshot.getFiles()) { - StoreFileMetaData md = shard.store().metaData(name); + long length = shard.store().directory().fileLength(name); boolean useExisting = false; if (request.existingFiles().containsKey(name)) { - if (!md.name().contains("segment") && md.length() == request.existingFiles().get(name).length()) { + if (!name.contains("segment") && length == request.existingFiles().get(name).length()) { response.phase1ExistingFileNames.add(name); - response.phase1ExistingFileSizes.add(md.length()); - existingTotalSize += md.length(); + response.phase1ExistingFileSizes.add(length); + existingTotalSize += length; useExisting = true; if (logger.isTraceEnabled()) { - logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has size [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, md.length()); + logger.trace("[{}][{}] recovery [phase1] to {}: not recovering [{}], exists in local store and has size [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, length); } } } if (!useExisting) { if (request.existingFiles().containsKey(name)) { - logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but has different length: remote [{}], local [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, request.existingFiles().get(name).length(), md.length()); + logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], exists in local store, but has different length: remote [{}], local [{}]", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name, request.existingFiles().get(name).length(), length); } else { logger.trace("[{}][{}] recovery [phase1] to {}: recovering [{}], does not exists in remote", request.shardId().index().name(), request.shardId().id(), request.targetNode(), name); } response.phase1FileNames.add(name); - response.phase1FileSizes.add(md.length()); - totalSize += md.length(); + response.phase1FileSizes.add(length); + totalSize += length; } } response.phase1TotalSize = totalSize; diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java index 3619ff22f78..8d916ae4a4d 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/Store.java @@ -36,8 +36,6 @@ public interface Store extends IndexShardComponent { */ Directory directory(); - StoreFileMetaData metaData(String name) throws IOException; - ImmutableMap list() throws IOException; /** diff --git a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java index 1eb4472ca5e..ac4b5fd4502 100644 --- a/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java +++ b/modules/elasticsearch/src/main/java/org/elasticsearch/index/store/support/AbstractStore.java @@ -62,18 +62,6 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen return new StoreDirectory(dir); } - @Override public StoreFileMetaData metaData(String name) throws IOException { - StoreFileMetaData md = filesMetadata.get(name); - if (md == null) { - return null; - } - // IndexOutput not closed, does not exists - if (md.lastModified() == -1 || md.length() == -1) { - return null; - } - return md; - } - @Override public ImmutableMap list() throws IOException { ImmutableMap.Builder builder = ImmutableMap.builder(); for (String name : files) { @@ -85,6 +73,18 @@ public abstract class AbstractStore extends AbstractIndexShardComponent implemen return builder.build(); } + private StoreFileMetaData metaData(String name) throws IOException { + StoreFileMetaData md = filesMetadata.get(name); + if (md == null) { + return null; + } + // IndexOutput not closed, does not exists + if (md.lastModified() == -1 || md.length() == -1) { + return null; + } + return md; + } + @Override public void deleteContent() throws IOException { Directories.deleteFiles(directory()); }