From 63d236c019909d321f7824bb0e043c7bddd60bf0 Mon Sep 17 00:00:00 2001 From: Thomas Marquardt Date: Wed, 17 Jun 2020 23:12:22 +0000 Subject: [PATCH] HADOOP-17076: ABFS: Delegation SAS Generator Updates Contributed by Thomas Marquardt. DETAILS: 1) The authentication version in the service has been updated from Dec19 to Feb20, so need to update the client. 2) Add support and test cases for getXattr and setXAttr. 3) Update DelegationSASGenerator and related to use Duration instead of int for time periods. 4) Cleanup DelegationSASGenerator switch/case statement that maps operations to permissions. 5) Cleanup SASGenerator classes to use String.equals instead of ==. TESTS: Added tests for getXAttr and setXAttr. All tests are passing against my account in eastus2euap: $mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=8 clean verify Tests run: 76, Failures: 0, Errors: 0, Skipped: 0 Tests run: 441, Failures: 0, Errors: 0, Skipped: 33 Tests run: 206, Failures: 0, Errors: 0, Skipped: 24 --- .../azurebfs/extensions/SASTokenProvider.java | 2 +- ...ITestAzureBlobFileSystemDelegationSAS.java | 16 ++++++++++ .../MockDelegationSASTokenProvider.java | 4 +-- .../utils/DelegationSASGenerator.java | 32 +++++++++---------- .../fs/azurebfs/utils/SASGenerator.java | 8 +++-- .../azurebfs/utils/ServiceSASGenerator.java | 6 ++-- 6 files changed, 42 insertions(+), 26 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java index 2cd44f1b907..a2cd292b0b2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java @@ -33,6 +33,7 @@ import org.apache.hadoop.security.AccessControlException; public interface SASTokenProvider { String CHECK_ACCESS_OPERATION = "check-access"; + String CREATE_DIRECTORY_OPERATION = "create-directory"; String CREATE_FILE_OPERATION = "create-file"; String DELETE_OPERATION = "delete"; String DELETE_RECURSIVE_OPERATION = "delete-recursive"; @@ -40,7 +41,6 @@ public interface SASTokenProvider { String GET_STATUS_OPERATION = "get-status"; String GET_PROPERTIES_OPERATION = "get-properties"; String LIST_OPERATION = "list"; - String CREATE_DIRECTORY_OPERATION = "create-directory"; String READ_OPERATION = "read"; String RENAME_SOURCE_OPERATION = "rename-source"; String RENAME_DESTINATION_OPERATION = "rename-destination"; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java index 07b5804d111..c2c691e2f62 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.List; import java.util.UUID; +import org.junit.Assert; import org.junit.Assume; import org.junit.Test; import org.slf4j.Logger; @@ -365,4 +366,19 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati } assertEquals(0, count); } + + @Test + // Test filesystem operations getXAttr and setXAttr + public void testProperties() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + Path reqPath = new Path(UUID.randomUUID().toString()); + + fs.create(reqPath).close(); + + final String propertyName = "user.mime_type"; + final byte[] propertyValue = "text/plain".getBytes("utf-8"); + fs.setXAttr(reqPath, propertyName, propertyValue); + + assertArrayEquals(propertyValue, fs.getXAttr(reqPath, propertyName)); + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java index fa50bef454b..121256c4dbc 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java @@ -55,8 +55,8 @@ public class MockDelegationSASTokenProvider implements SASTokenProvider { String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET); String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID); String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID); - String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minusSeconds(SASGenerator.FIVE_MINUTES)); - String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plusSeconds(SASGenerator.ONE_DAY)); + String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minus(SASGenerator.FIVE_MINUTES)); + String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plus(SASGenerator.ONE_DAY)); String skv = SASGenerator.AuthenticationVersion.Dec19.toString(); byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java index f84ed6ab4c7..6f2209a6e8c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java @@ -29,12 +29,12 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsUriQueryBuilder; * Test Delegation SAS generator. */ public class DelegationSASGenerator extends SASGenerator { - private String skoid; - private String sktid; - private String skt; - private String ske; + private final String skoid; + private final String sktid; + private final String skt; + private final String ske; private final String sks = "b"; - private String skv; + private final String skv; public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) { super(userDelegationKey); @@ -48,20 +48,18 @@ public class DelegationSASGenerator extends SASGenerator { public String getDelegationSAS(String accountName, String containerName, String path, String operation, String saoid, String suoid, String scid) { - final String sv = AuthenticationVersion.Dec19.toString(); - final String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES)); - final String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY)); + final String sv = AuthenticationVersion.Feb20.toString(); + final String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES)); + final String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY)); String sr = "b"; String sdd = null; - String sp = null; + String sp; switch (operation) { - case SASTokenProvider.CHECK_ACCESS_OPERATION: - sp = "e"; - break; - case SASTokenProvider.WRITE_OPERATION: case SASTokenProvider.CREATE_FILE_OPERATION: case SASTokenProvider.CREATE_DIRECTORY_OPERATION: + case SASTokenProvider.WRITE_OPERATION: + case SASTokenProvider.SET_PROPERTIES_OPERATION: sp = "w"; break; case SASTokenProvider.DELETE_OPERATION: @@ -72,6 +70,7 @@ public class DelegationSASGenerator extends SASGenerator { sr = "d"; sdd = Integer.toString(StringUtils.countMatches(path, "/")); break; + case SASTokenProvider.CHECK_ACCESS_OPERATION: case SASTokenProvider.GET_ACL_OPERATION: case SASTokenProvider.GET_STATUS_OPERATION: sp = "e"; @@ -79,6 +78,7 @@ public class DelegationSASGenerator extends SASGenerator { case SASTokenProvider.LIST_OPERATION: sp = "l"; break; + case SASTokenProvider.GET_PROPERTIES_OPERATION: case SASTokenProvider.READ_OPERATION: sp = "r"; break; @@ -87,14 +87,12 @@ public class DelegationSASGenerator extends SASGenerator { sp = "m"; break; case SASTokenProvider.SET_ACL_OPERATION: + case SASTokenProvider.SET_PERMISSION_OPERATION: sp = "p"; break; case SASTokenProvider.SET_OWNER_OPERATION: sp = "o"; break; - case SASTokenProvider.SET_PERMISSION_OPERATION: - sp = "p"; - break; default: throw new IllegalArgumentException(operation); } @@ -146,7 +144,7 @@ public class DelegationSASGenerator extends SASGenerator { sb.append(accountName); sb.append("/"); sb.append(containerName); - if (path != null && sr != "c") { + if (path != null && !sr.equals("c")) { sb.append(path); } sb.append("\n"); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java index 34d504a9a07..2e9289d8d44 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.azurebfs.utils; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.time.format.DateTimeFormatter; import java.time.ZoneId; import java.util.Locale; @@ -35,7 +36,8 @@ public abstract class SASGenerator { public enum AuthenticationVersion { Nov18("2018-11-09"), - Dec19("2019-12-12"); + Dec19("2019-12-12"), + Feb20("2020-02-10"); private final String ver; @@ -50,8 +52,8 @@ public abstract class SASGenerator { } protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class); - public static final int FIVE_MINUTES = 5 * 60; - public static final int ONE_DAY = 24 * 60 * 60; + public static final Duration FIVE_MINUTES = Duration.ofMinutes(5); + public static final Duration ONE_DAY = Duration.ofDays(1); public static final DateTimeFormatter ISO_8601_FORMATTER = DateTimeFormatter .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java index a76681b599b..3d8496eff81 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java @@ -40,8 +40,8 @@ public class ServiceSASGenerator extends SASGenerator { String sp = "rcwdl"; String sv = AuthenticationVersion.Nov18.toString(); String sr = "c"; - String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES)); - String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY)); + String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES)); + String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY)); String signature = computeSignatureForSAS(sp, st, se, sv, "c", accountName, containerName, null); @@ -71,7 +71,7 @@ public class ServiceSASGenerator extends SASGenerator { sb.append(accountName); sb.append("/"); sb.append(containerName); - if (path != null && sr != "c") { + if (path != null && !sr.equals("c")) { //sb.append("/"); sb.append(path); }