diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 87b608b0839..9301f515fe5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -743,7 +743,20 @@ public class Configuration implements Iterable>, this.loadDefaults = other.loadDefaults; setQuietMode(other.getQuietMode()); } - + + /** + * Reload existing configuration instances. + */ + public static synchronized void reloadExistingConfigurations() { + if (LOG.isDebugEnabled()) { + LOG.debug("Reloading " + REGISTRY.keySet().size() + + " existing configurations"); + } + for (Configuration conf : REGISTRY.keySet()) { + conf.reloadConfiguration(); + } + } + /** * Add a default resource. Resources are loaded in the order of the resources * added. diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java index 8fc8e00b84c..d3a5565e6d5 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java @@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configuration.DeprecationDelta; /** * Constants. @@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability; @InterfaceStability.Evolving public final class AdlConfKeys { // OAuth2 Common Configuration - public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2" - + ".refresh.url"; + public static final String AZURE_AD_REFRESH_URL_KEY = + "fs.adl.oauth2.refresh.url"; // optional when provider type is refresh or client id. public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY = - "dfs.adls.oauth2.access.token.provider"; + "fs.adl.oauth2.access.token.provider"; public static final String AZURE_AD_CLIENT_ID_KEY = - "dfs.adls.oauth2.client.id"; + "fs.adl.oauth2.client.id"; public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY = - "dfs.adls.oauth2.access.token.provider.type"; + "fs.adl.oauth2.access.token.provider.type"; // OAuth Refresh Token Configuration public static final String AZURE_AD_REFRESH_TOKEN_KEY = - "dfs.adls.oauth2.refresh.token"; + "fs.adl.oauth2.refresh.token"; public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken"; // OAuth Client Cred Token Configuration public static final String AZURE_AD_CLIENT_SECRET_KEY = - "dfs.adls.oauth2.credential"; + "fs.adl.oauth2.credential"; public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED = "ClientCredential"; @@ -75,7 +77,7 @@ public final class AdlConfKeys { static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024; static final String LATENCY_TRACKER_KEY = - "adl.dfs.enable.client.latency.tracker"; + "adl.enable.client.latency.tracker"; static final boolean LATENCY_TRACKER_DEFAULT = true; static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY = @@ -90,6 +92,26 @@ public final class AdlConfKeys { "adl.feature.ownerandgroup.enableupn"; static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false; + public static void addDeprecatedKeys() { + Configuration.addDeprecations(new DeprecationDelta[]{ + new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type", + AZURE_AD_TOKEN_PROVIDER_TYPE_KEY), + new DeprecationDelta("dfs.adls.oauth2.client.id", + AZURE_AD_CLIENT_ID_KEY), + new DeprecationDelta("dfs.adls.oauth2.refresh.token", + AZURE_AD_REFRESH_TOKEN_KEY), + new DeprecationDelta("dfs.adls.oauth2.refresh.url", + AZURE_AD_REFRESH_URL_KEY), + new DeprecationDelta("dfs.adls.oauth2.credential", + AZURE_AD_CLIENT_SECRET_KEY), + new DeprecationDelta("dfs.adls.oauth2.access.token.provider", + AZURE_AD_TOKEN_PROVIDER_CLASS_KEY), + new DeprecationDelta("adl.dfs.enable.client.latency.tracker", + LATENCY_TRACKER_KEY) + }); + Configuration.reloadExistingConfigurations(); + } + private AdlConfKeys() { } } diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index e0e273e6f21..0b860b36592 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem { private AccessTokenProvider tokenProvider; private AzureADTokenProvider azureTokenProvider; + static { + AdlConfKeys.addDeprecatedKeys(); + } + @Override public String getScheme() { return SCHEME; diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md index de8fda25d17..d2da858dfc7 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md @@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml` ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type RefreshToken ``` @@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*] ```xml - dfs.adls.oauth2.client.id + fs.adl.oauth2.client.id - dfs.adls.oauth2.refresh.token + fs.adl.oauth2.refresh.token ``` @@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml` ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type ClientCredential - dfs.adls.oauth2.refresh.url + fs.adl.oauth2.refresh.url TOKEN ENDPOINT FROM STEP 7 ABOVE - dfs.adls.oauth2.client.id + fs.adl.oauth2.client.id CLIENT ID FROM STEP 7 ABOVE - dfs.adls.oauth2.credential + fs.adl.oauth2.credential PASSWORD FROM STEP 7 ABOVE ``` @@ -166,9 +166,9 @@ For additional reading on the credential provider API, see ##### Provisioning ```bash -hadoop credential create dfs.adls.oauth2.client.id -value 123 +hadoop credential create fs.adl.oauth2.client.id -value 123 -provider localjceks://file/home/foo/adls.jceks -hadoop credential create dfs.adls.oauth2.refresh.token -value 123 +hadoop credential create fs.adl.oauth2.refresh.token -value 123 -provider localjceks://file/home/foo/adls.jceks ``` @@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123 ```xml - dfs.adls.oauth2.access.token.provider.type + fs.adl.oauth2.access.token.provider.type RefreshToken @@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123 ```bash hadoop distcp - [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken + [-D fs.adl.oauth2.access.token.provider.type=RefreshToken -D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks] hdfs://:9001/user/foo/srcDir adl://.azuredatalakestore.net/tgtDir/ diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java index 70f2a7f992f..3867e74fa77 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java @@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -121,8 +122,8 @@ public class TestAzureADTokenProvider { Assert.fail("Initialization should have failed due no token provider " + "configuration"); } catch (IllegalArgumentException e) { - Assert.assertTrue( - e.getMessage().contains("dfs.adls.oauth2.access.token.provider")); + GenericTestUtils.assertExceptionContains( + AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e); } conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, CustomMockTokenProvider.class, AzureADTokenProvider.class); diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java index 4cabaa3d08d..3d51b42111e 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.adl; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.test.GenericTestUtils; import org.junit.Assert; import org.junit.Test; @@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys .TOKEN_PROVIDER_TYPE_REFRESH_TOKEN; import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + /** * Validate configuration keys defined for adl storage file system instance. */ @@ -64,18 +71,18 @@ public class TestValidateConfiguration { @Test public void validateConfigurationKeys() { Assert - .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY); - Assert.assertEquals("dfs.adls.oauth2.access.token.provider", + .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY); + Assert.assertEquals("fs.adl.oauth2.access.token.provider", AZURE_AD_TOKEN_PROVIDER_CLASS_KEY); - Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY); - Assert.assertEquals("dfs.adls.oauth2.refresh.token", + Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY); + Assert.assertEquals("fs.adl.oauth2.refresh.token", AZURE_AD_REFRESH_TOKEN_KEY); Assert - .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY); + .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY); Assert.assertEquals("adl.debug.override.localuserasfileowner", ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER); - Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type", + Assert.assertEquals("fs.adl.oauth2.access.token.provider.type", AZURE_AD_TOKEN_PROVIDER_TYPE_KEY); Assert.assertEquals("adl.feature.client.cache.readahead", @@ -88,7 +95,7 @@ public class TestValidateConfiguration { Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED); - Assert.assertEquals("adl.dfs.enable.client.latency.tracker", + Assert.assertEquals("adl.enable.client.latency.tracker", LATENCY_TRACKER_KEY); Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT); @@ -109,4 +116,66 @@ public class TestValidateConfiguration { Assert.assertEquals(false, ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT); } + + @Test + public void testSetDeprecatedKeys() throws ClassNotFoundException { + Configuration conf = new Configuration(true); + setDeprecatedKeys(conf); + + // Force AdlFileSystem static initialization to register deprecated keys. + Class.forName(AdlFileSystem.class.getName()); + + assertDeprecatedKeys(conf); + } + + @Test + public void testLoadDeprecatedKeys() + throws IOException, ClassNotFoundException { + Configuration saveConf = new Configuration(false); + setDeprecatedKeys(saveConf); + + final File testRootDir = GenericTestUtils.getTestDir(); + File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml"); + OutputStream out = new FileOutputStream(confXml); + saveConf.writeXml(out); + out.close(); + + Configuration conf = new Configuration(true); + conf.addResource(confXml.toURI().toURL()); + + // Trigger loading the configuration resources by getting any key. + conf.get("dummy.key"); + + // Force AdlFileSystem static initialization to register deprecated keys. + Class.forName(AdlFileSystem.class.getName()); + + assertDeprecatedKeys(conf); + } + + private void setDeprecatedKeys(Configuration conf) { + conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType"); + conf.set("dfs.adls.oauth2.client.id", "dummyClientId"); + conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken"); + conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl"); + conf.set("dfs.adls.oauth2.credential", "dummyCredential"); + conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass"); + conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker"); + } + + private void assertDeprecatedKeys(Configuration conf) { + Assert.assertEquals("dummyType", + conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY)); + Assert.assertEquals("dummyClientId", + conf.get(AZURE_AD_CLIENT_ID_KEY)); + Assert.assertEquals("dummyRefreshToken", + conf.get(AZURE_AD_REFRESH_TOKEN_KEY)); + Assert.assertEquals("dummyRefreshUrl", + conf.get(AZURE_AD_REFRESH_URL_KEY)); + Assert.assertEquals("dummyCredential", + conf.get(AZURE_AD_CLIENT_SECRET_KEY)); + Assert.assertEquals("dummyClass", + conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY)); + Assert.assertEquals("dummyTracker", + conf.get(LATENCY_TRACKER_KEY)); + } }