Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support to read task logs from some S3 compatible cloud storage (#13195) 

* follow RFC7232

* Only unquoted strings are processed according to RFC7232.

* Add help method and test cases.
This commit is contained in:
hnakamor 2022-10-15 11:44:23 +09:00 committed by GitHub
parent 42384d85e7
commit 6332c571bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 30 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3TaskLogs.java
View File

@ -95,7 +95,7 @@ public class S3TaskLogs implements TaskLogs
} }
final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey) final GetObjectRequest request = new GetObjectRequest(config.getS3Bucket(), taskKey)
.withMatchingETagConstraint(objectMetadata.getETag()) .withMatchingETagConstraint(ensureQuotated(objectMetadata.getETag()))
.withRange(start, end); .withRange(start, end);
return Optional.of(service.getObject(request).getObjectContent()); return Optional.of(service.getObject(request).getObjectContent());
@ -115,6 +115,16 @@ public class S3TaskLogs implements TaskLogs
} }
} }
static String ensureQuotated(String eTag)
{
if (eTag != null) {
if (!eTag.startsWith("\"") && !eTag.endsWith("\"")) {
return "\"" + eTag + "\"";
}
}
return eTag;
}
@Override @Override
public void pushTaskLog(final String taskid, final File logFile) throws IOException public void pushTaskLog(final String taskid, final File logFile) throws IOException
{ {

19
extensions-core/s3-extensions/src/test/java/org/apache/druid/storage/s3/S3TaskLogsTest.java
View File

@ -480,4 +480,23 @@ public class S3TaskLogsTest extends EasyMockSupport
return aclExpected.getGrantsAsList(); return aclExpected.getGrantsAsList();
} }
@Test
public void testEnsureQuotated()
{
Assert.assertEquals("\"etag\"", S3TaskLogs.ensureQuotated("etag"));
Assert.assertNull(S3TaskLogs.ensureQuotated(null));
Assert.assertEquals("\"etag", S3TaskLogs.ensureQuotated("\"etag"));
Assert.assertEquals("etag\"", S3TaskLogs.ensureQuotated("etag\""));
}
@Test
public void testMatchingEtagConstraintWithEnsureQuotated()
{
String eTag = "etag";
final GetObjectRequest request = new GetObjectRequest(null, null)
.withMatchingETagConstraint(S3TaskLogs.ensureQuotated(eTag))
.withRange(0, 1);
Assert.assertEquals("\"" + eTag + "\"", request.getMatchingETagConstraints().get(0));
}
} }