HADOOP-14174. Set default ADLS access token provider type to ClientCredential. Contributed by John Zhuge.
Signed-off-by: John Zhuge <jzhuge@apache.org>
(cherry picked from commit 56e81f2a20)
Conflicts:
hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestCommonConfigurationFields.java
This commit is contained in:
John Zhuge
parent
cbb1c76195
commit
ff2870310e
|
|
@ -2512,6 +2512,81 @@
|
|||
</description>
|
||||
</property>
|
||||
|
||||
<!-- Azure Data Lake File System Configurations -->
|
||||
|
||||
<property>
|
||||
<name>fs.adl.impl</name>
|
||||
<value>org.apache.hadoop.fs.adl.AdlFileSystem</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.AbstractFileSystem.adl.impl</name>
|
||||
<value>org.apache.hadoop.fs.adl.Adl</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>adl.feature.ownerandgroup.enableupn</name>
|
||||
<value>false</value>
|
||||
<description>
|
||||
When true : User and Group in FileStatus/AclStatus response is
|
||||
represented as user friendly name as per Azure AD profile.
|
||||
|
||||
When false (default) : User and Group in FileStatus/AclStatus
|
||||
response is represented by the unique identifier from Azure AD
|
||||
profile (Object ID as GUID).
|
||||
|
||||
For optimal performance, false is recommended.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.adl.oauth2.access.token.provider.type</name>
|
||||
<value>ClientCredential</value>
|
||||
<description>
|
||||
Defines Azure Active Directory OAuth2 access token provider type.
|
||||
Supported types are ClientCredential, RefreshToken, and Custom.
|
||||
The ClientCredential type requires property fs.adl.oauth2.client.id,
|
||||
fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url.
|
||||
The RefreshToken type requires property fs.adl.oauth2.client.id and
|
||||
fs.adl.oauth2.refresh.token.
|
||||
The Custom type requires property fs.adl.oauth2.access.token.provider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.adl.oauth2.client.id</name>
|
||||
<value></value>
|
||||
<description>The OAuth2 client id.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.adl.oauth2.credential</name>
|
||||
<value></value>
|
||||
<description>The OAuth2 access key.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.adl.oauth2.refresh.url</name>
|
||||
<value></value>
|
||||
<description>The OAuth2 token endpoint.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.adl.oauth2.refresh.token</name>
|
||||
<value></value>
|
||||
<description>The OAuth2 refresh token.</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.adl.oauth2.access.token.provider</name>
|
||||
<value></value>
|
||||
<description>
|
||||
The class name of the OAuth2 access token provider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<!-- Azure Data Lake File System Configurations Ends Here-->
|
||||
|
||||
<property>
|
||||
<name>hadoop.caller.context.enabled</name>
|
||||
<value>false</value>
|
||||
|
|
|
|||
|
|
@ -115,7 +115,7 @@ public class TestCommonConfigurationFields extends TestConfigurationFieldsBase {
|
|||
// ADL properties are in a different subtree
|
||||
// - org.apache.hadoop.hdfs.web.ADLConfKeys
|
||||
xmlPrefixToSkipCompare.add("adl.");
|
||||
xmlPropsToSkipCompare.add("fs.adl.impl");
|
||||
xmlPrefixToSkipCompare.add("fs.adl.");
|
||||
xmlPropsToSkipCompare.add("fs.AbstractFileSystem.adl.impl");
|
||||
|
||||
// Deprecated properties. These should eventually be removed from the
|
||||
|
|
|
|||
|
|
@ -40,6 +40,8 @@ public final class AdlConfKeys {
|
|||
"fs.adl.oauth2.client.id";
|
||||
public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
|
||||
"fs.adl.oauth2.access.token.provider.type";
|
||||
public static final TokenProviderType AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT =
|
||||
TokenProviderType.ClientCredential;
|
||||
|
||||
// OAuth Refresh Token Configuration
|
||||
public static final String AZURE_AD_REFRESH_TOKEN_KEY =
|
||||
|
|
|
|||
|
|
@ -243,7 +243,8 @@ public class AdlFileSystem extends FileSystem {
|
|||
Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders(
|
||||
config, AdlFileSystem.class);
|
||||
TokenProviderType type = conf.getEnum(
|
||||
AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
|
||||
AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
|
||||
AdlConfKeys.AZURE_AD_TOKEN_PROVIDER_TYPE_DEFAULT);
|
||||
|
||||
switch (type) {
|
||||
case RefreshToken:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ import org.apache.hadoop.fs.adl.common.CustomMockTokenProvider;
|
|||
import org.apache.hadoop.fs.adl.oauth2.AzureADTokenProvider;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
|
||||
|
||||
import com.squareup.okhttp.mockwebserver.MockWebServer;
|
||||
|
||||
|
|
@ -84,6 +86,7 @@ public class AdlMockWebServer {
|
|||
// Responses are returned in the same order that they are enqueued.
|
||||
fs = new TestableAdlFileSystem();
|
||||
|
||||
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
|
||||
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
CustomMockTokenProvider.class, AzureADTokenProvider.class);
|
||||
|
||||
|
|
|
|||
|
|
@ -101,6 +101,7 @@ public class TestAzureADTokenProvider {
|
|||
public void testCustomCredTokenProvider()
|
||||
throws URISyntaxException, IOException {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
|
||||
conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
CustomMockTokenProvider.class, AzureADTokenProvider.class);
|
||||
|
||||
|
|
@ -115,6 +116,7 @@ public class TestAzureADTokenProvider {
|
|||
public void testInvalidProviderConfigurationForType()
|
||||
throws URISyntaxException, IOException {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
|
||||
URI uri = new URI("adl://localhost:8080");
|
||||
AdlFileSystem fileSystem = new AdlFileSystem();
|
||||
try {
|
||||
|
|
@ -136,6 +138,7 @@ public class TestAzureADTokenProvider {
|
|||
Configuration conf = new Configuration();
|
||||
URI uri = new URI("adl://localhost:8080");
|
||||
AdlFileSystem fileSystem = new AdlFileSystem();
|
||||
conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, TokenProviderType.Custom);
|
||||
conf.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
"wrong.classpath.CustomMockTokenProvider");
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -38,6 +38,8 @@ import java.util.Collection;
|
|||
import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
|
||||
|
||||
/**
|
||||
* Test access token provider behaviour with custom token provider and for token
|
||||
|
|
@ -89,6 +91,8 @@ public class TestCustomTokenProvider extends AdlMockWebServer {
|
|||
*/
|
||||
public void init() throws IOException, URISyntaxException {
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
|
||||
TokenProviderType.Custom);
|
||||
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
typeOfTokenProviderClass.getName());
|
||||
fileSystems = new TestableAdlFileSystem[fsObjectCount];
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ import java.net.URISyntaxException;
|
|||
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_CLASS_KEY;
|
||||
import static org.apache.hadoop.fs.adl.AdlConfKeys
|
||||
.AZURE_AD_TOKEN_PROVIDER_TYPE_KEY;
|
||||
|
||||
/**
|
||||
* This class verifies path conversion to SDK.
|
||||
|
|
@ -39,6 +41,8 @@ public class TestRelativePathFormation {
|
|||
public void testToRelativePath() throws URISyntaxException, IOException {
|
||||
AdlFileSystem fs = new AdlFileSystem();
|
||||
Configuration configuration = new Configuration();
|
||||
configuration.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY,
|
||||
TokenProviderType.Custom);
|
||||
configuration.set(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
|
||||
"org.apache.hadoop.fs.adl.common.CustomMockTokenProvider");
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue