honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Log Warning on Failed Blob Deletes in BlobStoreRepository (#40188) (#40340) 

* Log Warning on Failed Blob Deletes in BlobStoreRepository
* We should not just debug log these spots, they all can and will lead to leaked files when snapshot deletion fails
This commit is contained in:
Armin Braun 2019-03-25 08:52:09 +01:00 committed by GitHub
parent b9f96a8e1f
commit dc5ff0fffc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java
View File

@ -466,17 +466,17 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
final BlobContainer indicesBlobContainer = blobStore().blobContainer(basePath().add("indices")); final BlobContainer indicesBlobContainer = blobStore().blobContainer(basePath().add("indices"));
for (final IndexId indexId : indicesToCleanUp) { for (final IndexId indexId : indicesToCleanUp) {
try { try {
indicesBlobContainer.deleteBlob(indexId.getId()); indicesBlobContainer.deleteBlobIgnoringIfNotExists(indexId.getId());
} catch (DirectoryNotEmptyException dnee) { } catch (DirectoryNotEmptyException dnee) {
// if the directory isn't empty for some reason, it will fail to clean up; // if the directory isn't empty for some reason, it will fail to clean up;
// we'll ignore that and accept that cleanup didn't fully succeed. // we'll ignore that and accept that cleanup didn't fully succeed.
// since we are using UUIDs for path names, this won't be an issue for // since we are using UUIDs for path names, this won't be an issue for
// snapshotting indices of the same name // snapshotting indices of the same name
logger.debug(() -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, " + logger.warn(() -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, " +
"but failed to clean up its index folder due to the directory not being empty.", metadata.name(), indexId), dnee); "but failed to clean up its index folder due to the directory not being empty.", metadata.name(), indexId), dnee);
} catch (IOException ioe) { } catch (IOException ioe) {
// a different IOException occurred while trying to delete - will just log the issue for now // a different IOException occurred while trying to delete - will just log the issue for now
logger.debug(() -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, " + logger.warn(() -> new ParameterizedMessage("[{}] index [{}] no longer part of any snapshots in the repository, " +
"but failed to clean up its index folder.", metadata.name(), indexId), ioe); "but failed to clean up its index folder.", metadata.name(), indexId), ioe);
} }
} }
@ -826,7 +826,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
} catch (NumberFormatException nfe) { } catch (NumberFormatException nfe) {
// the index- blob wasn't of the format index-N where N is a number, // the index- blob wasn't of the format index-N where N is a number,
// no idea what this blob is but it doesn't belong in the repository! // no idea what this blob is but it doesn't belong in the repository!
logger.debug("[{}] Unknown blob in the repository: {}", metadata.name(), blobName); logger.warn("[{}] Unknown blob in the repository: {}", metadata.name(), blobName);
} }
} }
return latest; return latest;
@ -969,7 +969,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
try { try {
indexShardSnapshotFormat.delete(blobContainer, snapshotId.getUUID()); indexShardSnapshotFormat.delete(blobContainer, snapshotId.getUUID());
} catch (IOException e) { } catch (IOException e) {
logger.debug("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId); logger.warn(new ParameterizedMessage("[{}] [{}] failed to delete shard snapshot file", shardId, snapshotId), e);
} }
// Build a list of snapshots that should be preserved // Build a list of snapshots that should be preserved
@ -1129,7 +1129,7 @@ public abstract class BlobStoreRepository extends AbstractLifecycleComponent imp
logger.warn(() -> new ParameterizedMessage("failed to read index file [{}]", file), e); logger.warn(() -> new ParameterizedMessage("failed to read index file [{}]", file), e);
} }
} else if (blobKeys.isEmpty() == false) { } else if (blobKeys.isEmpty() == false) {
logger.debug("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", blobContainer.path()); logger.warn("Could not find a readable index-N file in a non-empty shard snapshot directory [{}]", blobContainer.path());
} }
// We couldn't load the index file - falling back to loading individual snapshots // We couldn't load the index file - falling back to loading individual snapshots