diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 23739b070d1..269a13ca273 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -2717,8 +2717,7 @@ fs.adl.oauth2.credential, and fs.adl.oauth2.refresh.url. The RefreshToken type requires property fs.adl.oauth2.client.id and fs.adl.oauth2.refresh.token. - The MSI type requires properties fs.adl.oauth2.msi.port and - fs.adl.oauth2.msi.tenantguid. + The MSI type reads optional property fs.adl.oauth2.msi.port, if specified. The DeviceCode type requires property fs.adl.oauth2.devicecode.clientapp.id. The Custom type requires property fs.adl.oauth2.access.token.provider. @@ -2762,17 +2761,8 @@ The localhost port for the MSI token service. This is the port specified - when creating the Azure VM. - Used by MSI token provider. - - - - - fs.adl.oauth2.msi.tenantguid - - - The tenant guid for the Azure AAD tenant under which the azure data lake - store account is created. + when creating the Azure VM. The default, if this setting is not specified, + is 50342. Used by MSI token provider. diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml index f699464c8c9..53b7b5e5192 100644 --- a/hadoop-tools/hadoop-azure-datalake/pom.xml +++ b/hadoop-tools/hadoop-azure-datalake/pom.xml @@ -110,7 +110,7 @@ com.microsoft.azure azure-data-lake-store-sdk - 2.2.2 + 2.2.3 diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index f77d98100cd..790902c6229 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -56,7 +56,6 @@ public final class AdlConfKeys { // MSI Auth Configuration public static final String MSI_PORT = "fs.adl.oauth2.msi.port"; - public static final String MSI_TENANT_GUID = "fs.adl.oauth2.msi.tenantguid"; // DeviceCode Auth configuration public static final String DEVICE_CODE_CLIENT_APP_ID = diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index a5e31e153c9..a4965959dee 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -290,9 +290,7 @@ public class AdlFileSystem extends FileSystem { private AccessTokenProvider getMsiBasedTokenProvider( Configuration conf) throws IOException { - int port = Integer.parseInt(getNonEmptyVal(conf, MSI_PORT)); - String tenantGuid = getPasswordString(conf, MSI_TENANT_GUID); - return new MsiTokenProvider(port, tenantGuid); + return new MsiTokenProvider(conf.getInt(MSI_PORT, -1)); } private AccessTokenProvider getDeviceCodeTokenProvider( diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index e34da36d566..ca79321efdd 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -164,15 +164,11 @@ Identity extension within the VM. The advantage of doing this is that the credentials are managed by the extension, and do not have to be put into core-site.xml. -To use MSI, the following two steps are needed: -1. Modify the VM deployment template to specify the port number of the token - service exposed to localhost by the identity extension in the VM. -2. Get your Azure ActiveDirectory Tenant ID: - 1. Go to [the portal](https://portal.azure.com) - 2. Under services in left nav, look for Azure Active Directory and click on it. - 3. Click on Properties - 4. Note down the GUID shown under "Directory ID" - this is your AAD tenant ID - +To use MSI, modify the VM deployment template to use the identity extension. Note the +port number you specified in the template: this is the port number for the REST endpoint +of the token service exposed to localhost by the identity extension in the VM. The default +recommended port number is 50342 - if the recommended port number is used, then the msi.port +setting below can be omitted in the configuration. ##### Configure core-site.xml Add the following properties to your `core-site.xml` @@ -185,12 +181,7 @@ Add the following properties to your `core-site.xml` fs.adl.oauth2.msi.port - PORT NUMBER FROM STEP 1 ABOVE - - - - fs.adl.oauth2.msi.TenantGuid - AAD TENANT ID GUID FROM STEP 2 ABOVE + PORT NUMBER FROM ABOVE (if different from the default of 50342) ``` diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 929b33a0427..12c2e3ffc78 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -43,8 +43,6 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys import static org.apache.hadoop.fs.adl.AdlConfKeys .AZURE_AD_TOKEN_PROVIDER_TYPE_KEY; import static org.apache.hadoop.fs.adl.AdlConfKeys.DEVICE_CODE_CLIENT_APP_ID; -import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_PORT; -import static org.apache.hadoop.fs.adl.AdlConfKeys.MSI_TENANT_GUID; import static org.apache.hadoop.fs.adl.TokenProviderType.*; import static org.junit.Assert.assertEquals; @@ -107,8 +105,6 @@ public class TestAzureADTokenProvider { throws IOException, URISyntaxException { Configuration conf = new Configuration(); conf.setEnum(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY, MSI); - conf.set(MSI_PORT, "54321"); - conf.set(MSI_TENANT_GUID, "TENANT_GUID"); URI uri = new URI("adl://localhost:8080"); AdlFileSystem fileSystem = new AdlFileSystem();