honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove index-N Rebuild in Shard Snapshot Updates (#45740) (#45778) 

* There is no point in listing out every shard over and over when the `index-N` blob in the shard contains a list of all the files
   * Rebuilding the `index-N` from the `snap-${uuid}.dat` blobs does not provide any material benefit. It only would in the corner case of a corrupted `index-N` but otherwise uncorrupted blobs since we neither check the correctness of the content of all segment blobs nor do we do a similar recovery at the root of the repository.
   * Also, at least in version `6.x` we only mark a shard snapshot as successful after writing out the updated `index-N` blob so all snapshots that would work with `7.x` and newer must have correct `index-N` blobs

=> Removed the rebuilding of the `index-N` content from `snap-${uuid}.dat` files and moved to only listing `index-N` when taking a snapshot instead of listing all files
=> Removed check of file existence against physical blob listing
=> Kept full listing on the delete side to retain full cleanup of blobs that aren't referenced by the `index-N`
This commit is contained in:
Armin Braun 2019-08-22 11:32:45 +02:00 committed by GitHub
parent 7512337922
commit 88acae48ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/main/java/org/elasticsearch/index/snapshots/blobstore/BlobStoreIndexShardSnapshots.java
View File

@ -44,6 +44,8 @@ import static java.util.Collections.unmodifiableMap;
*/ */
public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, ToXContentFragment { public class BlobStoreIndexShardSnapshots implements Iterable<SnapshotFiles>, ToXContentFragment {
public static final BlobStoreIndexShardSnapshots EMPTY = new BlobStoreIndexShardSnapshots(Collections.emptyList());
private final List<SnapshotFiles> shardSnapshots; private final List<SnapshotFiles> shardSnapshots;
private final Map<String, FileInfo> files; private final Map<String, FileInfo> files;
private final Map<String, List<FileInfo>> physicalFiles; private final Map<String, List<FileInfo>> physicalFiles;

22
server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -920,7 +920,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
final BlobContainer shardContainer = shardContainer(indexId, shardId); final BlobContainer shardContainer = shardContainer(indexId, shardId);
final Map<String, BlobMetaData> blobs; final Map<String, BlobMetaData> blobs;
try { try {
blobs = shardContainer.listBlobs(); blobs = shardContainer.listBlobsByPrefix(INDEX_FILE_PREFIX);
} catch (IOException e) { } catch (IOException e) {
throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e); throw new IndexShardSnapshotFailedException(shardId, "failed to list blobs", e);
} }
@ -965,7 +965,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName); List<BlobStoreIndexShardSnapshot.FileInfo> filesInfo = snapshots.findPhysicalIndexFiles(fileName);
if (filesInfo != null) { if (filesInfo != null) {
for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) { for (BlobStoreIndexShardSnapshot.FileInfo fileInfo : filesInfo) {
if (fileInfo.isSame(md) && snapshotFileExistsInBlobs(fileInfo, blobs)) { if (fileInfo.isSame(md)) {
// a commit point file with the same name, size and checksum was already copied to repository // a commit point file with the same name, size and checksum was already copied to repository
// we will reuse it for this snapshot // we will reuse it for this snapshot
existingFileInfo = fileInfo; existingFileInfo = fileInfo;
@ -1236,23 +1236,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
} else if (blobKeys.isEmpty() == false) { } else if (blobKeys.isEmpty() == false) {
logger.warn("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", shardContainer.path()); logger.warn("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", shardContainer.path());
} }
return new Tuple<>(BlobStoreIndexShardSnapshots.EMPTY, latest);
// We couldn't load the index file - falling back to loading individual snapshots
List<SnapshotFiles> snapshots = new ArrayList<>();
for (String name : blobKeys) {
try {
BlobStoreIndexShardSnapshot snapshot = null;
if (name.startsWith(SNAPSHOT_PREFIX)) {
snapshot = indexShardSnapshotFormat.readBlob(shardContainer, name);
}
if (snapshot != null) {
snapshots.add(new SnapshotFiles(snapshot.snapshot(), snapshot.indexFiles()));
}
} catch (IOException e) {
logger.warn(() -> new ParameterizedMessage("Failed to read blob [{}]", name), e);
}
}
return new Tuple<>(new BlobStoreIndexShardSnapshots(snapshots), latest);
} }
/** /**