Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add logs for deleting files using storage connector (#14350) 

* Add logs for deleting files using storage connector

* Address review comments

* Update log message format
This commit is contained in:
Adarsh Sanjeev 2023-06-11 21:24:30 +05:30 committed by GitHub
parent 6e158704cb
commit 267cbac6ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
View File

@ -54,6 +54,7 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
/**
*
@ -260,6 +261,8 @@ public class S3Utils
)
throws Exception
{
log.debug("Deleting directory at bucket: [%s], path: [%s]", bucket, prefix);
final List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>(maxListingLength);
final ObjectSummaryIterator iterator = new ObjectSummaryIterator(
s3Client,
@ -291,6 +294,10 @@ public class S3Utils
)
throws Exception
{
if (keysToDelete != null && log.isDebugEnabled()) {
List<String> keys = keysToDelete.stream().map(DeleteObjectsRequest.KeyVersion::getKey).collect(Collectors.toList());
log.debug("Deleting keys from bucket: [%s], keys: [%s]", bucket, keys);
}
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete);
S3Utils.retryS3Operation(() -> {
s3Client.deleteObjects(deleteRequest);

6
extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/output/S3StorageConnector.java
View File

@ -280,8 +280,11 @@ public class S3StorageConnector implements StorageConnector
public void deleteFile(String path) throws IOException
{
try {
final String fullPath = objectPath(path);
log.debug("Deleting file at bucket: [%s], path: [%s]", config.getBucket(), fullPath);
S3Utils.retryS3Operation(() -> {
s3Client.deleteObject(config.getBucket(), objectPath(path));
s3Client.deleteObject(config.getBucket(), fullPath);
return null;
}, config.getMaxRetry());
}
@ -320,6 +323,7 @@ public class S3StorageConnector implements StorageConnector
S3Utils.deleteBucketKeys(s3Client, config.getBucket(), versions, config.getMaxRetry());
}
catch (Exception e) {
log.error("Error occurred while deleting files from S3. Error: [%s]", e.getMessage());
throw new IOException(e);
}
}

5
processing/src/main/java/org/apache/druid/storage/local/LocalFileStorageConnector.java
View File

@ -24,6 +24,7 @@ import org.apache.druid.java.util.common.FileUtils;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.storage.StorageConnector;
import java.io.File;
@ -44,6 +45,7 @@ import java.util.Iterator;
*/
public class LocalFileStorageConnector implements StorageConnector
{
private static final Logger log = new Logger(LocalFileStorageConnector.class);
private final File basePath;
@ -111,6 +113,7 @@ public class LocalFileStorageConnector implements StorageConnector
@Override
public void deleteFile(String path) throws IOException
{
log.debug("Deleting file at path: [%s]", path);
File toDelete = fileWithBasePath(path);
if (toDelete.isDirectory()) {
throw new IAE(StringUtils.format(
@ -129,6 +132,7 @@ public class LocalFileStorageConnector implements StorageConnector
public void deleteFiles(Iterable<String> paths) throws IOException
{
for (String path : paths) {
log.debug("Deleting file at path: [%s]", path);
deleteFile(path);
}
}
@ -142,6 +146,7 @@ public class LocalFileStorageConnector implements StorageConnector
@Override
public void deleteRecursively(String dirName) throws IOException
{
log.debug("Deleting directory at path: [%s]", dirName);
FileUtils.deleteDirectory(fileWithBasePath(dirName));
}