From ce02671de4ab354ec019fa8c335bad55e46ac906 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 18 Jan 2023 13:21:17 +0000 Subject: [PATCH] update GET range referrer header logic to V2 --- .../fs/s3a/audit/impl/LoggingAuditor.java | 28 ++++++------- .../fs/s3a/audit/AbstractAuditingTest.java | 41 +++++++++++++++---- .../audit/TestHttpReferrerAuditHeader.java | 26 +++++++----- 3 files changed, 63 insertions(+), 32 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 0dd0976acc5..4005179d201 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import software.amazon.awssdk.awscore.AwsExecutionAttribute; import software.amazon.awssdk.core.SdkRequest; import software.amazon.awssdk.core.interceptor.Context; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; @@ -261,21 +262,18 @@ public class LoggingAuditor * Attach Range of data for GetObject Request. * @param request given get object request */ -// private void attachRangeFromRequest(AmazonWebServiceRequest request) { -// if (request instanceof GetObjectRequest) { -// long[] rangeValue = ((GetObjectRequest) request).getRange(); -// if (rangeValue == null || rangeValue.length == 0) { -// return; -// } -// if (rangeValue.length != 2) { -// WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements." -// + " Got {} elements. Ignoring.", rangeValue.length); -// return; -// } -// String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]); -// referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); -// } -// } + private void attachRangeFromRequest(SdkHttpRequest request, + ExecutionAttributes executionAttributes) { + + if (executionAttributes.getAttribute(AwsExecutionAttribute.OPERATION_NAME).equals("GetObject")) { + if (request.headers() != null + && request.headers().get("Range") != null) { + String rangeValue = request.headers().get("Range").get(0); + String rangeHeader = rangeValue.split("=")[1]; + referrer.set(AuditConstants.PARAM_RANGE, rangeHeader); + } + } + } private final String description; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index 3f3b3149d42..0f6421d1bc4 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -22,10 +22,14 @@ import java.io.IOException; import java.util.Arrays; import java.util.List; import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; import java.util.stream.Collectors; + +import software.amazon.awssdk.awscore.AwsExecutionAttribute; import software.amazon.awssdk.core.interceptor.ExecutionAttributes; import software.amazon.awssdk.core.interceptor.InterceptorContext; import software.amazon.awssdk.http.SdkHttpMethod; @@ -162,14 +166,37 @@ public abstract class AbstractAuditingTest extends AbstractHadoopTestBase { } /** - * Create a GetObject request and modify it before passing it through auditor. - * @param modifyRequest Consumer Interface for changing the request before passing to the auditor - * @return the request + * Create a get request and pass it through the manager's beforeExecution() + * callback. + * + * @return a processed request. */ - protected GetObjectRequest get(Consumer modifyRequest) { - GetObjectRequest req = requestFactory.newGetObjectRequest("/"); - modifyRequest.accept(req); - return manager.beforeExecution(req); + protected SdkHttpRequest get(String range) { + GetObjectRequest.Builder getObjectRequestBuilder = + requestFactory.newGetObjectRequestBuilder("/"); + + SdkHttpRequest.Builder httpRequestBuilder = + SdkHttpRequest.builder().uri(URI.create("https://test")).method(SdkHttpMethod.GET); + + if (!range.isEmpty()) { + getObjectRequestBuilder.range(range); + List rangeHeader = new ArrayList<>(); + rangeHeader.add(range); + Map> headers = new HashMap<>(); + headers.put("Range", rangeHeader); + httpRequestBuilder.headers(headers); + } + + manager.requestCreated(getObjectRequestBuilder); + GetObjectRequest getObjectRequest = getObjectRequestBuilder.build(); + ExecutionAttributes executionAttributes = ExecutionAttributes.builder().build().putAttribute( + AwsExecutionAttribute.OPERATION_NAME, "GetObject"); + InterceptorContext context = InterceptorContext.builder() + .request(getObjectRequest) + .httpRequest(httpRequestBuilder.build()) + .build(); + manager.beforeExecution(context, executionAttributes); + return manager.modifyHttpRequest(context, executionAttributes); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index 991379e435d..430e1370c03 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -308,13 +308,16 @@ public class TestHttpReferrerAuditHeader extends AbstractAuditingTest { @Test public void testGetObjectRange() throws Throwable { AuditSpan span = span(); - GetObjectRequest request = get(getObjectRequest -> getObjectRequest.setRange(100, 200)); - Map headers - = request.getCustomRequestHeaders(); + SdkHttpRequest request = get("bytes=100-200"); + Map> headers = request.headers(); assertThat(headers) - .describedAs("Custom headers") - .containsKey(HEADER_REFERRER); - String header = headers.get(HEADER_REFERRER); + .describedAs("Custom headers") + .containsKey(HEADER_REFERRER); + List headerValues = headers.get(HEADER_REFERRER); + assertThat(headerValues) + .describedAs("Multiple referrer headers") + .hasSize(1); + String header = headerValues.get(0); LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); @@ -327,13 +330,16 @@ public class TestHttpReferrerAuditHeader extends AbstractAuditingTest { @Test public void testGetObjectWithoutRange() throws Throwable { AuditSpan span = span(); - GetObjectRequest request = get(getObjectRequest -> {}); - Map headers - = request.getCustomRequestHeaders(); + SdkHttpRequest request = get(""); + Map> headers = request.headers(); assertThat(headers) .describedAs("Custom headers") .containsKey(HEADER_REFERRER); - String header = headers.get(HEADER_REFERRER); + List headerValues = headers.get(HEADER_REFERRER); + assertThat(headerValues) + .describedAs("Multiple referrer headers") + .hasSize(1); + String header = headerValues.get(0); LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header);