Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update GET range referrer header logic to V2

This commit is contained in:
Ahmar Suhail 2023-01-18 13:21:17 +00:00
parent 644a32f9d2
commit ce02671de4
3 changed files with 63 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java
View File

@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import software.amazon.awssdk.awscore.AwsExecutionAttribute;
import software.amazon.awssdk.core.SdkRequest;
import software.amazon.awssdk.core.interceptor.Context;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
@ -261,21 +262,18 @@ public class LoggingAuditor
* Attach Range of data for GetObject Request.
* @param request given get object request
*/
// private void attachRangeFromRequest(AmazonWebServiceRequest request) {
// if (request instanceof GetObjectRequest) {
// long[] rangeValue = ((GetObjectRequest) request).getRange();
// if (rangeValue == null || rangeValue.length == 0) {
// return;
// }
// if (rangeValue.length != 2) {
// WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements."
// + " Got {} elements. Ignoring.", rangeValue.length);
// return;
// }
// String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]);
// referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue);
// }
// }
private void attachRangeFromRequest(SdkHttpRequest request,
ExecutionAttributes executionAttributes) {
if (executionAttributes.getAttribute(AwsExecutionAttribute.OPERATION_NAME).equals("GetObject")) {
if (request.headers() != null
&& request.headers().get("Range") != null) {
String rangeValue = request.headers().get("Range").get(0);
String rangeHeader = rangeValue.split("=")[1];
referrer.set(AuditConstants.PARAM_RANGE, rangeHeader);
}
}
}
private final String description;

41
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java
View File

@ -22,10 +22,14 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import software.amazon.awssdk.awscore.AwsExecutionAttribute;
import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
import software.amazon.awssdk.core.interceptor.InterceptorContext;
import software.amazon.awssdk.http.SdkHttpMethod;
@ -162,14 +166,37 @@ public abstract class AbstractAuditingTest extends AbstractHadoopTestBase {
}
/**
* Create a GetObject request and modify it before passing it through auditor.
* @param modifyRequest Consumer Interface for changing the request before passing to the auditor
* @return the request
* Create a get request and pass it through the manager's beforeExecution()
* callback.
*
* @return a processed request.
*/
protected GetObjectRequest get(Consumer<GetObjectRequest> modifyRequest) {
GetObjectRequest req = requestFactory.newGetObjectRequest("/");
modifyRequest.accept(req);
return manager.beforeExecution(req);
protected SdkHttpRequest get(String range) {
GetObjectRequest.Builder getObjectRequestBuilder =
requestFactory.newGetObjectRequestBuilder("/");
SdkHttpRequest.Builder httpRequestBuilder =
SdkHttpRequest.builder().uri(URI.create("https://test")).method(SdkHttpMethod.GET);
if (!range.isEmpty()) {
getObjectRequestBuilder.range(range);
List<String> rangeHeader = new ArrayList<>();
rangeHeader.add(range);
Map<String, List<String>> headers = new HashMap<>();
headers.put("Range", rangeHeader);
httpRequestBuilder.headers(headers);
}
manager.requestCreated(getObjectRequestBuilder);
GetObjectRequest getObjectRequest = getObjectRequestBuilder.build();
ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build().putAttribute(
AwsExecutionAttribute.OPERATION_NAME, "GetObject");
InterceptorContext context = InterceptorContext.builder()
.request(getObjectRequest)
.httpRequest(httpRequestBuilder.build())
.build();
manager.beforeExecution(context, executionAttributes);
return manager.modifyHttpRequest(context, executionAttributes);
}
/**

26
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java
View File

@ -308,13 +308,16 @@ public class TestHttpReferrerAuditHeader extends AbstractAuditingTest {
@Test
public void testGetObjectRange() throws Throwable {
AuditSpan span = span();
GetObjectRequest request = get(getObjectRequest -> getObjectRequest.setRange(100, 200));
Map<String, String> headers
= request.getCustomRequestHeaders();
SdkHttpRequest request = get("bytes=100-200");
Map<String, List<String>> headers = request.headers();
assertThat(headers)
.describedAs("Custom headers")
.containsKey(HEADER_REFERRER);
String header = headers.get(HEADER_REFERRER);
.describedAs("Custom headers")
.containsKey(HEADER_REFERRER);
List<String> headerValues = headers.get(HEADER_REFERRER);
assertThat(headerValues)
.describedAs("Multiple referrer headers")
.hasSize(1);
String header = headerValues.get(0);
LOG.info("Header is {}", header);
Map<String, String> params
= HttpReferrerAuditHeader.extractQueryParameters(header);
@ -327,13 +330,16 @@ public class TestHttpReferrerAuditHeader extends AbstractAuditingTest {
@Test
public void testGetObjectWithoutRange() throws Throwable {
AuditSpan span = span();
GetObjectRequest request = get(getObjectRequest -> {});
Map<String, String> headers
= request.getCustomRequestHeaders();
SdkHttpRequest request = get("");
Map<String, List<String>> headers = request.headers();
assertThat(headers)
.describedAs("Custom headers")
.containsKey(HEADER_REFERRER);
String header = headers.get(HEADER_REFERRER);
List<String> headerValues = headers.get(HEADER_REFERRER);
assertThat(headerValues)
.describedAs("Multiple referrer headers")
.hasSize(1);
String header = headerValues.get(0);
LOG.info("Header is {}", header);
Map<String, String> params
= HttpReferrerAuditHeader.extractQueryParameters(header);