diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 3b9193ae7da..61c9df7da3e 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2512,6 +2512,81 @@ + + + + fs.adl.impl + org.apache.hadoop.fs.adl.AdlFileSystem + + + + fs.AbstractFileSystem.adl.impl + org.apache.hadoop.fs.adl.Adl + + + + adl.feature.ownerandgroup.enableupn + false + + When true : User and Group in FileStatus/AclStatus response is + represented as user friendly name as per Azure AD profile. + + When false (default) : User and Group in FileStatus/AclStatus + response is represented by the unique identifier from Azure AD + profile (Object ID as GUID). + + For optimal performance, false is recommended. + + + + + fs.adl.oauth2.access.token.provider.type + ClientCredential + + Defines Azure Active Directory OAuth2 access token provider type. + Supported types are ClientCredential, RefreshToken, and Custom. + The ClientCredential type requires property fs.adl.oauth2.client.id, + fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. + The RefreshToken type requires property fs.adl.oauth2.client.id and + fs.adl.oauth2.refresh.token. + The Custom type requires property fs.adl.oauth2.access.token.provider. + + + + + fs.adl.oauth2.client.id + + The OAuth2 client id. + + + + fs.adl.oauth2.credential + + The OAuth2 access key. + + + + fs.adl.oauth2.refresh.url + + The OAuth2 token endpoint. + + + + fs.adl.oauth2.refresh.token + + The OAuth2 refresh token. + + + + fs.adl.oauth2.access.token.provider + + + The class name of the OAuth2 access token provider. + + + + + hadoop.caller.context.enabled false diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java index dd0ebab474e..30e08d58892 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java @@ -115,7 +115,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase { // ADL properties are in a different subtree // - org.apache.hadoop.hdfs.web.ADLConfKeys xmlPrefixToSkipCompare.add("adl."); - xmlPropsToSkipCompare.add("fs.adl.impl"); + xmlPrefixToSkipCompare.add("fs.adl."); xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl"); // Deprecated properties. These should eventually be removed from the diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index d3a5565e6d5..31df22254c2 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -40,6 +40,8 @@ public final class AdlConfKeys { "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = "fs.adl.oauth2.access.token.provider.type"; + public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT = + TokenProviderType.ClientCredential; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 0b860b36592..e63f1152569 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -243,7 +243,8 @@ public class AdlFileSystem extends FileSystem { Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( config, AdlFileSystem.class); TokenProviderType type = conf.getEnum( - AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT); switch (type) { case RefreshToken: diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java index 55c8f812c25..d843d5556b3 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/AdlMockWebServer.java @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider; import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import com.squareup.okhttp.mockwebserver.MockWebServer; @@ -84,6 +86,7 @@ public class AdlMockWebServer { // Responses are returned in the same order that they are enqueued. fs = new TestableAdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 3867e74fa77..36498c6696c 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -101,6 +101,7 @@ public class TestAzureADTokenProvider { public void testCustomCredTokenProvider() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); @@ -115,6 +116,7 @@ public class TestAzureADTokenProvider { public void testInvalidProviderConfigurationForType() throws URISyntaxException, IOException { Configuration conf = new Configuration(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); try { @@ -136,6 +138,7 @@ public class TestAzureADTokenProvider { Configuration conf = new Configuration(); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem(); + conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom); conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "wrong.classpath.CustomMockTokenProvider"); try { diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java index c594c6540b4..737534cf010 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestCustomTokenProvider.java @@ -38,6 +38,8 @@ import java.util.Collection; import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * Test access token provider behaviour with custom token provider and for token @@ -89,6 +91,8 @@ public class TestCustomTokenProvider extends AdlMockWebServer { */ public void init() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, typeOfTokenProviderClass.getName()); fileSystems = new TestableAdlFileSystem[fsObjectCount]; diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java index 908f8b8919e..334c372e7b1 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestRelativePathFormation.java @@ -29,6 +29,8 @@ import java.net.URISyntaxException; import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_CLASS_KEY; +import static org.apache.hadoop.fs.adl.AdlConfKeys + .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; /** * This class verifies path conversion to SDK. @@ -39,6 +41,8 @@ public class TestRelativePathFormation { public void testToRelativePath() throws URISyntaxException, IOException { AdlFileSystem fs = new AdlFileSystem(); Configuration configuration = new Configuration(); + configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, + TokenProviderType.Custom); configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, "org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");