Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-17076: ABFS: Delegation SAS Generator Updates 

Contributed by Thomas Marquardt.

DETAILS:
1) The authentication version in the service has been updated from Dec19 to Feb20, so need to update the client.
2) Add support and test cases for getXattr and setXAttr.
3) Update DelegationSASGenerator and related to use Duration instead of int for time periods.
4) Cleanup DelegationSASGenerator switch/case statement that maps operations to permissions.
5) Cleanup SASGenerator classes to use String.equals instead of ==.

TESTS:
Added tests for getXAttr and setXAttr.

All tests are passing against my account in eastus2euap:

 $mvn -T 1C -Dparallel-tests=abfs -Dscale -DtestsThreadCount=8 clean verify
 Tests run: 76, Failures: 0, Errors: 0, Skipped: 0
 Tests run: 441, Failures: 0, Errors: 0, Skipped: 33
 Tests run: 206, Failures: 0, Errors: 0, Skipped: 24
This commit is contained in:
Thomas Marquardt 2020-06-17 23:12:22 +00:00
parent 89689c52c3
commit caf3995ac2
No known key found for this signature in database
GPG Key ID: AEB30C9E78868287
6 changed files with 42 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/extensions/SASTokenProvider.java
View File

@ -33,6 +33,7 @@ import org.apache.hadoop.security.AccessControlException;
public interface SASTokenProvider { public interface SASTokenProvider {
String CHECK_ACCESS_OPERATION = "check-access"; String CHECK_ACCESS_OPERATION = "check-access";
String CREATE_DIRECTORY_OPERATION = "create-directory";
String CREATE_FILE_OPERATION = "create-file"; String CREATE_FILE_OPERATION = "create-file";
String DELETE_OPERATION = "delete"; String DELETE_OPERATION = "delete";
String DELETE_RECURSIVE_OPERATION = "delete-recursive"; String DELETE_RECURSIVE_OPERATION = "delete-recursive";
@ -40,7 +41,6 @@ public interface SASTokenProvider {
String GET_STATUS_OPERATION = "get-status"; String GET_STATUS_OPERATION = "get-status";
String GET_PROPERTIES_OPERATION = "get-properties"; String GET_PROPERTIES_OPERATION = "get-properties";
String LIST_OPERATION = "list"; String LIST_OPERATION = "list";
String CREATE_DIRECTORY_OPERATION = "create-directory";
String READ_OPERATION = "read"; String READ_OPERATION = "read";
String RENAME_SOURCE_OPERATION = "rename-source"; String RENAME_SOURCE_OPERATION = "rename-source";
String RENAME_DESTINATION_OPERATION = "rename-destination"; String RENAME_DESTINATION_OPERATION = "rename-destination";

16
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelegationSAS.java
View File

@ -25,6 +25,7 @@ import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import org.junit.Assert;
import org.junit.Assume; import org.junit.Assume;
import org.junit.Test; import org.junit.Test;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -365,4 +366,19 @@ public class ITestAzureBlobFileSystemDelegationSAS extends AbstractAbfsIntegrati
} }
assertEquals(0, count); assertEquals(0, count);
} }
@Test
// Test filesystem operations getXAttr and setXAttr
public void testProperties() throws Exception {
final AzureBlobFileSystem fs = getFileSystem();
Path reqPath = new Path(UUID.randomUUID().toString());
fs.create(reqPath).close();
final String propertyName = "user.mime_type";
final byte[] propertyValue = "text/plain".getBytes("utf-8");
fs.setXAttr(reqPath, propertyName, propertyValue);
assertArrayEquals(propertyValue, fs.getXAttr(reqPath, propertyName));
}
} }

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/extensions/MockDelegationSASTokenProvider.java
View File

@ -55,8 +55,8 @@ public class MockDelegationSASTokenProvider implements SASTokenProvider {
String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET); String appSecret = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SECRET);
String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID); String sktid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_TENANT_ID);
String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID); String skoid = configuration.get(TestConfigurationKeys.FS_AZURE_TEST_APP_SERVICE_PRINCIPAL_OBJECT_ID);
String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minusSeconds(SASGenerator.FIVE_MINUTES)); String skt = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().minus(SASGenerator.FIVE_MINUTES));
String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plusSeconds(SASGenerator.ONE_DAY)); String ske = SASGenerator.ISO_8601_FORMATTER.format(Instant.now().plus(SASGenerator.ONE_DAY));
String skv = SASGenerator.AuthenticationVersion.Dec19.toString(); String skv = SASGenerator.AuthenticationVersion.Dec19.toString();
byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv); byte[] key = getUserDelegationKey(accountName, appID, appSecret, sktid, skt, ske, skv);

32
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/DelegationSASGenerator.java
View File

@ -29,12 +29,12 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsUriQueryBuilder;
* Test Delegation SAS generator. * Test Delegation SAS generator.
*/ */
public class DelegationSASGenerator extends SASGenerator { public class DelegationSASGenerator extends SASGenerator {
private String skoid; private final String skoid;
private String sktid; private final String sktid;
private String skt; private final String skt;
private String ske; private final String ske;
private final String sks = "b"; private final String sks = "b";
private String skv; private final String skv;
public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) { public DelegationSASGenerator(byte[] userDelegationKey, String skoid, String sktid, String skt, String ske, String skv) {
super(userDelegationKey); super(userDelegationKey);
@ -48,20 +48,18 @@ public class DelegationSASGenerator extends SASGenerator {
public String getDelegationSAS(String accountName, String containerName, String path, String operation, public String getDelegationSAS(String accountName, String containerName, String path, String operation,
String saoid, String suoid, String scid) { String saoid, String suoid, String scid) {
final String sv = AuthenticationVersion.Dec19.toString(); final String sv = AuthenticationVersion.Feb20.toString();
final String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES)); final String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
final String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY)); final String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
String sr = "b"; String sr = "b";
String sdd = null; String sdd = null;
String sp = null; String sp;
switch (operation) { switch (operation) {
case SASTokenProvider.CHECK_ACCESS_OPERATION:
sp = "e";
break;
case SASTokenProvider.WRITE_OPERATION:
case SASTokenProvider.CREATE_FILE_OPERATION: case SASTokenProvider.CREATE_FILE_OPERATION:
case SASTokenProvider.CREATE_DIRECTORY_OPERATION: case SASTokenProvider.CREATE_DIRECTORY_OPERATION:
case SASTokenProvider.WRITE_OPERATION:
case SASTokenProvider.SET_PROPERTIES_OPERATION:
sp = "w"; sp = "w";
break; break;
case SASTokenProvider.DELETE_OPERATION: case SASTokenProvider.DELETE_OPERATION:
@ -72,6 +70,7 @@ public class DelegationSASGenerator extends SASGenerator {
sr = "d"; sr = "d";
sdd = Integer.toString(StringUtils.countMatches(path, "/")); sdd = Integer.toString(StringUtils.countMatches(path, "/"));
break; break;
case SASTokenProvider.CHECK_ACCESS_OPERATION:
case SASTokenProvider.GET_ACL_OPERATION: case SASTokenProvider.GET_ACL_OPERATION:
case SASTokenProvider.GET_STATUS_OPERATION: case SASTokenProvider.GET_STATUS_OPERATION:
sp = "e"; sp = "e";
@ -79,6 +78,7 @@ public class DelegationSASGenerator extends SASGenerator {
case SASTokenProvider.LIST_OPERATION: case SASTokenProvider.LIST_OPERATION:
sp = "l"; sp = "l";
break; break;
case SASTokenProvider.GET_PROPERTIES_OPERATION:
case SASTokenProvider.READ_OPERATION: case SASTokenProvider.READ_OPERATION:
sp = "r"; sp = "r";
break; break;
@ -87,14 +87,12 @@ public class DelegationSASGenerator extends SASGenerator {
sp = "m"; sp = "m";
break; break;
case SASTokenProvider.SET_ACL_OPERATION: case SASTokenProvider.SET_ACL_OPERATION:
case SASTokenProvider.SET_PERMISSION_OPERATION:
sp = "p"; sp = "p";
break; break;
case SASTokenProvider.SET_OWNER_OPERATION: case SASTokenProvider.SET_OWNER_OPERATION:
sp = "o"; sp = "o";
break; break;
case SASTokenProvider.SET_PERMISSION_OPERATION:
sp = "p";
break;
default: default:
throw new IllegalArgumentException(operation); throw new IllegalArgumentException(operation);
} }
@ -146,7 +144,7 @@ public class DelegationSASGenerator extends SASGenerator {
sb.append(accountName); sb.append(accountName);
sb.append("/"); sb.append("/");
sb.append(containerName); sb.append(containerName);
if (path != null && sr != "c") { if (path != null && !sr.equals("c")) {
sb.append(path); sb.append(path);
} }
sb.append("\n"); sb.append("\n");

8
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/SASGenerator.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.fs.azurebfs.utils;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.time.ZoneId; import java.time.ZoneId;
import java.util.Locale; import java.util.Locale;
@ -35,7 +36,8 @@ public abstract class SASGenerator {
public enum AuthenticationVersion { public enum AuthenticationVersion {
Nov18("2018-11-09"), Nov18("2018-11-09"),
Dec19("2019-12-12"); Dec19("2019-12-12"),
Feb20("2020-02-10");
private final String ver; private final String ver;
@ -50,8 +52,8 @@ public abstract class SASGenerator {
} }
protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class); protected static final Logger LOG = LoggerFactory.getLogger(SASGenerator.class);
public static final int FIVE_MINUTES = 5 * 60; public static final Duration FIVE_MINUTES = Duration.ofMinutes(5);
public static final int ONE_DAY = 24 * 60 * 60; public static final Duration ONE_DAY = Duration.ofDays(1);
public static final DateTimeFormatter ISO_8601_FORMATTER = public static final DateTimeFormatter ISO_8601_FORMATTER =
DateTimeFormatter DateTimeFormatter
.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT)

6
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/ServiceSASGenerator.java
View File

@ -40,8 +40,8 @@ public class ServiceSASGenerator extends SASGenerator {
String sp = "rcwdl"; String sp = "rcwdl";
String sv = AuthenticationVersion.Nov18.toString(); String sv = AuthenticationVersion.Nov18.toString();
String sr = "c"; String sr = "c";
String st = ISO_8601_FORMATTER.format(Instant.now().minusSeconds(FIVE_MINUTES)); String st = ISO_8601_FORMATTER.format(Instant.now().minus(FIVE_MINUTES));
String se = ISO_8601_FORMATTER.format(Instant.now().plusSeconds(ONE_DAY)); String se = ISO_8601_FORMATTER.format(Instant.now().plus(ONE_DAY));
String signature = computeSignatureForSAS(sp, st, se, sv, "c", String signature = computeSignatureForSAS(sp, st, se, sv, "c",
accountName, containerName, null); accountName, containerName, null);
@ -71,7 +71,7 @@ public class ServiceSASGenerator extends SASGenerator {
sb.append(accountName); sb.append(accountName);
sb.append("/"); sb.append("/");
sb.append(containerName); sb.append(containerName);
if (path != null && sr != "c") { if (path != null && !sr.equals("c")) {
//sb.append("/"); //sb.append("/");
sb.append(path); sb.append(path);
} }