HADOOP-14038. Rename ADLS credential properties. Contributed by John Zhuge.

(cherry picked from commit 253e3e78ab)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java

@@ -743,7 +743,20 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
     this.loadDefaults = other.loadDefaults;
     setQuietMode(other.getQuietMode());
   }
-  
+
+  /**
+   * Reload existing configuration instances.
+   */
+  public static synchronized void reloadExistingConfigurations() {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Reloading " + REGISTRY.keySet().size()
+          + " existing configurations");
+    }
+    for (Configuration conf : REGISTRY.keySet()) {
+      conf.reloadConfiguration();
+    }
+  }
+
   /**
    * Add a default resource. Resources are loaded in the order of the resources 
    * added.

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java

@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.adl;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
 
 /**
  * Constants.
@@ -28,25 +30,25 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceStability.Evolving
 public final class AdlConfKeys {
   // OAuth2 Common Configuration
-  public static final String AZURE_AD_REFRESH_URL_KEY = "dfs.adls.oauth2"
+  public static final String AZURE_AD_REFRESH_URL_KEY =
-      + ".refresh.url";
+      "fs.adl.oauth2.refresh.url";
 
   // optional when provider type is refresh or client id.
   public static final String AZURE_AD_TOKEN_PROVIDER_CLASS_KEY =
-      "dfs.adls.oauth2.access.token.provider";
+      "fs.adl.oauth2.access.token.provider";
   public static final String AZURE_AD_CLIENT_ID_KEY =
-      "dfs.adls.oauth2.client.id";
+      "fs.adl.oauth2.client.id";
   public static final String AZURE_AD_TOKEN_PROVIDER_TYPE_KEY =
-      "dfs.adls.oauth2.access.token.provider.type";
+      "fs.adl.oauth2.access.token.provider.type";
 
   // OAuth Refresh Token Configuration
   public static final String AZURE_AD_REFRESH_TOKEN_KEY =
-      "dfs.adls.oauth2.refresh.token";
+      "fs.adl.oauth2.refresh.token";
 
   public static final String TOKEN_PROVIDER_TYPE_REFRESH_TOKEN = "RefreshToken";
   // OAuth Client Cred Token Configuration
   public static final String AZURE_AD_CLIENT_SECRET_KEY =
-      "dfs.adls.oauth2.credential";
+      "fs.adl.oauth2.credential";
   public static final String TOKEN_PROVIDER_TYPE_CLIENT_CRED =
       "ClientCredential";
 
@@ -75,7 +77,7 @@ public final class AdlConfKeys {
   static final int DEFAULT_WRITE_AHEAD_BUFFER_SIZE = 4 * 1024 * 1024;
 
   static final String LATENCY_TRACKER_KEY =
-      "adl.dfs.enable.client.latency.tracker";
+      "adl.enable.client.latency.tracker";
   static final boolean LATENCY_TRACKER_DEFAULT = true;
 
   static final String ADL_EXPERIMENT_POSITIONAL_READ_KEY =
@@ -90,6 +92,26 @@ public final class AdlConfKeys {
       "adl.feature.ownerandgroup.enableupn";
   static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
 
+  public static void addDeprecatedKeys() {
+    Configuration.addDeprecations(new DeprecationDelta[]{
+        new DeprecationDelta("dfs.adls.oauth2.access.token.provider.type",
+            AZURE_AD_TOKEN_PROVIDER_TYPE_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.client.id",
+            AZURE_AD_CLIENT_ID_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.refresh.token",
+            AZURE_AD_REFRESH_TOKEN_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.refresh.url",
+            AZURE_AD_REFRESH_URL_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.credential",
+            AZURE_AD_CLIENT_SECRET_KEY),
+        new DeprecationDelta("dfs.adls.oauth2.access.token.provider",
+            AZURE_AD_TOKEN_PROVIDER_CLASS_KEY),
+        new DeprecationDelta("adl.dfs.enable.client.latency.tracker",
+            LATENCY_TRACKER_KEY)
+    });
+    Configuration.reloadExistingConfigurations();
+  }
+
   private AdlConfKeys() {
   }
 }

hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java

@@ -88,6 +88,10 @@ public class AdlFileSystem extends FileSystem {
   private AccessTokenProvider tokenProvider;
   private AzureADTokenProvider azureTokenProvider;
 
+  static {
+    AdlConfKeys.addDeprecatedKeys();
+  }
+
   @Override
   public String getScheme() {
     return SCHEME;

hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md

@@ -83,7 +83,7 @@ Add the following properties to the cluster's `core-site.xml`
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.access.token.provider.type</name>
+  <name>fs.adl.oauth2.access.token.provider.type</name>
   <value>RefreshToken</value>
 </property>
 ```
@@ -95,12 +95,12 @@ service associated with the client id. See [*Active Directory Library For Java*]
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.client.id</name>
+  <name>fs.adl.oauth2.client.id</name>
   <value></value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.refresh.token</name>
+  <name>fs.adl.oauth2.refresh.token</name>
   <value></value>
 </property>
 ```
@@ -133,22 +133,22 @@ Add the following properties to your `core-site.xml`
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.access.token.provider.type</name>
+  <name>fs.adl.oauth2.access.token.provider.type</name>
   <value>ClientCredential</value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.refresh.url</name>
+  <name>fs.adl.oauth2.refresh.url</name>
   <value>TOKEN ENDPOINT FROM STEP 7 ABOVE</value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.client.id</name>
+  <name>fs.adl.oauth2.client.id</name>
   <value>CLIENT ID FROM STEP 7 ABOVE</value>
 </property>
 
 <property>
-  <name>dfs.adls.oauth2.credential</name>
+  <name>fs.adl.oauth2.credential</name>
   <value>PASSWORD FROM STEP 7 ABOVE</value>
 </property>
 ```
@@ -166,9 +166,9 @@ For additional reading on the credential provider API, see
 ##### Provisioning
 
 ```bash
-hadoop credential create dfs.adls.oauth2.client.id -value 123
+hadoop credential create fs.adl.oauth2.client.id -value 123
     -provider localjceks://file/home/foo/adls.jceks
-hadoop credential create dfs.adls.oauth2.refresh.token -value 123
+hadoop credential create fs.adl.oauth2.refresh.token -value 123
     -provider localjceks://file/home/foo/adls.jceks
 ```
 
@@ -176,7 +176,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
 
 ```xml
 <property>
-  <name>dfs.adls.oauth2.access.token.provider.type</name>
+  <name>fs.adl.oauth2.access.token.provider.type</name>
   <value>RefreshToken</value>
 </property>
 <property>
@@ -190,7 +190,7 @@ hadoop credential create dfs.adls.oauth2.refresh.token -value 123
 
 ```bash
 hadoop distcp
-    [-D dfs.adls.oauth2.access.token.provider.type=RefreshToken
+    [-D fs.adl.oauth2.access.token.provider.type=RefreshToken
      -D hadoop.security.credential.provider.path=localjceks://file/home/user/adls.jceks]
     hdfs://<NameNode Hostname>:9001/user/foo/srcDir
     adl://<Account Name>.azuredatalakestore.net/tgtDir/

hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestAzureADTokenProvider.java

@@ -46,6 +46,7 @@ import static org.junit.Assert.assertEquals;
 import org.apache.hadoop.security.ProviderUtils;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Rule;
 import org.junit.Test;
@@ -121,8 +122,8 @@ public class TestAzureADTokenProvider {
       Assert.fail("Initialization should have failed due no token provider "
           + "configuration");
     } catch (IllegalArgumentException e) {
-      Assert.assertTrue(
+      GenericTestUtils.assertExceptionContains(
-          e.getMessage().contains("dfs.adls.oauth2.access.token.provider"));
+          AZURE_AD_TOKEN_PROVIDER_CLASS_KEY, e);
     }
     conf.setClass(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY,
         CustomMockTokenProvider.class, AzureADTokenProvider.class);

hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java

@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.fs.adl;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -56,6 +58,11 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
     .TOKEN_PROVIDER_TYPE_REFRESH_TOKEN;
 import static org.apache.hadoop.fs.adl.AdlConfKeys.WRITE_BUFFER_SIZE_KEY;
 
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
 /**
  * Validate configuration keys defined for adl storage file system instance.
  */
@@ -64,18 +71,18 @@ public class TestValidateConfiguration {
   @Test
   public void validateConfigurationKeys() {
     Assert
-        .assertEquals("dfs.adls.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
+        .assertEquals("fs.adl.oauth2.refresh.url", AZURE_AD_REFRESH_URL_KEY);
-    Assert.assertEquals("dfs.adls.oauth2.access.token.provider",
+    Assert.assertEquals("fs.adl.oauth2.access.token.provider",
         AZURE_AD_TOKEN_PROVIDER_CLASS_KEY);
-    Assert.assertEquals("dfs.adls.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
+    Assert.assertEquals("fs.adl.oauth2.client.id", AZURE_AD_CLIENT_ID_KEY);
-    Assert.assertEquals("dfs.adls.oauth2.refresh.token",
+    Assert.assertEquals("fs.adl.oauth2.refresh.token",
         AZURE_AD_REFRESH_TOKEN_KEY);
     Assert
-        .assertEquals("dfs.adls.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
+        .assertEquals("fs.adl.oauth2.credential", AZURE_AD_CLIENT_SECRET_KEY);
     Assert.assertEquals("adl.debug.override.localuserasfileowner",
         ADL_DEBUG_OVERRIDE_LOCAL_USER_AS_OWNER);
 
-    Assert.assertEquals("dfs.adls.oauth2.access.token.provider.type",
+    Assert.assertEquals("fs.adl.oauth2.access.token.provider.type",
         AZURE_AD_TOKEN_PROVIDER_TYPE_KEY);
 
     Assert.assertEquals("adl.feature.client.cache.readahead",
@@ -88,7 +95,7 @@ public class TestValidateConfiguration {
 
     Assert.assertEquals("ClientCredential", TOKEN_PROVIDER_TYPE_CLIENT_CRED);
 
-    Assert.assertEquals("adl.dfs.enable.client.latency.tracker",
+    Assert.assertEquals("adl.enable.client.latency.tracker",
         LATENCY_TRACKER_KEY);
 
     Assert.assertEquals(true, LATENCY_TRACKER_DEFAULT);
@@ -109,4 +116,66 @@ public class TestValidateConfiguration {
     Assert.assertEquals(false,
         ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
   }
+
+  @Test
+  public void testSetDeprecatedKeys() throws ClassNotFoundException {
+    Configuration conf = new Configuration(true);
+    setDeprecatedKeys(conf);
+
+    // Force AdlFileSystem static initialization to register deprecated keys.
+    Class.forName(AdlFileSystem.class.getName());
+
+    assertDeprecatedKeys(conf);
+  }
+
+  @Test
+  public void testLoadDeprecatedKeys()
+      throws IOException, ClassNotFoundException {
+    Configuration saveConf = new Configuration(false);
+    setDeprecatedKeys(saveConf);
+
+    final File testRootDir = GenericTestUtils.getTestDir();
+    File confXml = new File(testRootDir, "testLoadDeprecatedKeys.xml");
+    OutputStream out = new FileOutputStream(confXml);
+    saveConf.writeXml(out);
+    out.close();
+
+    Configuration conf = new Configuration(true);
+    conf.addResource(confXml.toURI().toURL());
+
+    // Trigger loading the configuration resources by getting any key.
+    conf.get("dummy.key");
+
+    // Force AdlFileSystem static initialization to register deprecated keys.
+    Class.forName(AdlFileSystem.class.getName());
+
+    assertDeprecatedKeys(conf);
+  }
+
+  private void setDeprecatedKeys(Configuration conf) {
+    conf.set("dfs.adls.oauth2.access.token.provider.type", "dummyType");
+    conf.set("dfs.adls.oauth2.client.id", "dummyClientId");
+    conf.set("dfs.adls.oauth2.refresh.token", "dummyRefreshToken");
+    conf.set("dfs.adls.oauth2.refresh.url", "dummyRefreshUrl");
+    conf.set("dfs.adls.oauth2.credential", "dummyCredential");
+    conf.set("dfs.adls.oauth2.access.token.provider", "dummyClass");
+    conf.set("adl.dfs.enable.client.latency.tracker", "dummyTracker");
+  }
+
+  private void assertDeprecatedKeys(Configuration conf) {
+    Assert.assertEquals("dummyType",
+        conf.get(AZURE_AD_TOKEN_PROVIDER_TYPE_KEY));
+    Assert.assertEquals("dummyClientId",
+        conf.get(AZURE_AD_CLIENT_ID_KEY));
+    Assert.assertEquals("dummyRefreshToken",
+        conf.get(AZURE_AD_REFRESH_TOKEN_KEY));
+    Assert.assertEquals("dummyRefreshUrl",
+        conf.get(AZURE_AD_REFRESH_URL_KEY));
+    Assert.assertEquals("dummyCredential",
+        conf.get(AZURE_AD_CLIENT_SECRET_KEY));
+    Assert.assertEquals("dummyClass",
+        conf.get(AZURE_AD_TOKEN_PROVIDER_CLASS_KEY));
+    Assert.assertEquals("dummyTracker",
+        conf.get(LATENCY_TRACKER_KEY));
+  }
 }