HADOOP-11176. KMSClientProvider authentication fails when both currentUgi and loginUgi are a proxied user. Contributed by Arun Suresh.

(cherry picked from commit 0e57aa3bf6)
(cherry picked from commit f3132eee10)

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -466,6 +466,9 @@ Release 2.6.0 - UNRELEASED
     HADOOP-11193. Fix uninitialized variables in NativeIO.c
     (Xiaoyu Yao via wheat9)
 
+    HADOOP-11176. KMSClientProvider authentication fails when both currentUgi
+    and loginUgi are a proxied user. (Arun Suresh via atm)
+
     BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
   
       HADOOP-10734. Implement high-performance secure random number sources.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java

@@ -250,8 +250,8 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
   private SSLFactory sslFactory;
   private ConnectionConfigurator configurator;
   private DelegationTokenAuthenticatedURL.Token authToken;
-  private UserGroupInformation loginUgi;
   private final int authRetry;
+  private final UserGroupInformation actualUgi;
 
   @Override
   public String toString() {
@@ -335,7 +335,11 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
                     KMS_CLIENT_ENC_KEY_CACHE_NUM_REFILL_THREADS_DEFAULT),
             new EncryptedQueueRefiller());
     authToken = new DelegationTokenAuthenticatedURL.Token();
-    loginUgi = UserGroupInformation.getCurrentUser();
+    actualUgi =
+        (UserGroupInformation.getCurrentUser().getAuthenticationMethod() ==
+        UserGroupInformation.AuthenticationMethod.PROXY) ? UserGroupInformation
+            .getCurrentUser().getRealUser() : UserGroupInformation
+            .getCurrentUser();
   }
 
   private String createServiceURL(URL url) throws IOException {
@@ -406,7 +410,7 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
                               ? currentUgi.getShortUserName() : null;
 
       // creating the HTTP connection using the current UGI at constructor time
-      conn = loginUgi.doAs(new PrivilegedExceptionAction<HttpURLConnection>() {
+      conn = actualUgi.doAs(new PrivilegedExceptionAction<HttpURLConnection>() {
         @Override
         public HttpURLConnection run() throws Exception {
           DelegationTokenAuthenticatedURL authUrl =
@@ -456,8 +460,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
       // WWW-Authenticate header as well)..
       KMSClientProvider.this.authToken =
           new DelegationTokenAuthenticatedURL.Token();
-      KMSClientProvider.this.loginUgi =
-          UserGroupInformation.getCurrentUser();
       if (authRetryCount > 0) {
         String contentType = conn.getRequestProperty(CONTENT_TYPE);
         String requestMethod = conn.getRequestMethod();
@@ -474,9 +476,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
       // Ignore the AuthExceptions.. since we are just using the method to
       // extract and set the authToken.. (Workaround till we actually fix
       // AuthenticatedURL properly to set authToken post initialization)
-    } finally {
-      KMSClientProvider.this.loginUgi =
-          UserGroupInformation.getCurrentUser();
     }
     HttpExceptionUtils.validateResponse(conn, expectedResponse);
     if (APPLICATION_JSON_MIME.equalsIgnoreCase(conn.getContentType())

hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java

@@ -1585,22 +1585,34 @@ public class TestKMS {
   }
 
   @Test
-  public void testProxyUser() throws Exception {
+  public void testProxyUserKerb() throws Exception {
+    doProxyUserTest(true);
+  }
+
+  @Test
+  public void testProxyUserSimple() throws Exception {
+    doProxyUserTest(false);
+  }
+
+  public void doProxyUserTest(final boolean kerberos) throws Exception {
     Configuration conf = new Configuration();
     conf.set("hadoop.security.authentication", "kerberos");
     UserGroupInformation.setConfiguration(conf);
     final File testDir = getTestDir();
     conf = createBaseKMSConf(testDir);
-    conf.set("hadoop.kms.authentication.type", "kerberos");
+    if (kerberos) {
+      conf.set("hadoop.kms.authentication.type", "kerberos");
+    }
     conf.set("hadoop.kms.authentication.kerberos.keytab",
         keytab.getAbsolutePath());
     conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
     conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
-    conf.set("hadoop.kms.proxyuser.client.users", "foo");
+    conf.set("hadoop.kms.proxyuser.client.users", "foo,bar");
     conf.set("hadoop.kms.proxyuser.client.hosts", "*");
-    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kAA.ALL", "*");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kAA.ALL", "client");
-    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kBB.ALL", "*");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kBB.ALL", "foo");
-    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kCC.ALL", "*");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kCC.ALL", "foo1");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kDD.ALL", "bar");
 
     writeConf(testDir, conf);
 
@@ -1611,9 +1623,16 @@ public class TestKMS {
         conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 64);
         final URI uri = createKMSUri(getKMSUrl());
 
-        // proxyuser client using kerberos credentials
+        UserGroupInformation proxyUgi = null;
-        final UserGroupInformation clientUgi = UserGroupInformation.
+        if (kerberos) {
-            loginUserFromKeytabAndReturnUGI("client", keytab.getAbsolutePath());
+          // proxyuser client using kerberos credentials
+          proxyUgi = UserGroupInformation.
+              loginUserFromKeytabAndReturnUGI("client", keytab.getAbsolutePath());
+        } else {
+          proxyUgi = UserGroupInformation.createRemoteUser("client");
+        }
+
+        final UserGroupInformation clientUgi = proxyUgi; 
         clientUgi.doAs(new PrivilegedExceptionAction<Void>() {
           @Override
           public Void run() throws Exception {
@@ -1649,6 +1668,123 @@ public class TestKMS {
                 return null;
               }
             });
+
+            // authorized proxyuser
+            UserGroupInformation barUgi =
+                UserGroupInformation.createProxyUser("bar", clientUgi);
+            barUgi.doAs(new PrivilegedExceptionAction<Void>() {
+              @Override
+              public Void run() throws Exception {
+                Assert.assertNotNull(kp.createKey("kDD",
+                    new KeyProvider.Options(conf)));
+                return null;
+              }
+            });
+            return null;
+          }
+        });
+
+        return null;
+      }
+    });
+  }
+
+  @Test
+  public void testWebHDFSProxyUserKerb() throws Exception {
+    doWebHDFSProxyUserTest(true);
+  }
+
+  @Test
+  public void testWebHDFSProxyUserSimple() throws Exception {
+    doWebHDFSProxyUserTest(false);
+  }
+
+  public void doWebHDFSProxyUserTest(final boolean kerberos) throws Exception {
+    Configuration conf = new Configuration();
+    conf.set("hadoop.security.authentication", "kerberos");
+    UserGroupInformation.setConfiguration(conf);
+    final File testDir = getTestDir();
+    conf = createBaseKMSConf(testDir);
+    if (kerberos) {
+      conf.set("hadoop.kms.authentication.type", "kerberos");
+    }
+    conf.set("hadoop.kms.authentication.kerberos.keytab",
+        keytab.getAbsolutePath());
+    conf.set("hadoop.kms.authentication.kerberos.principal", "HTTP/localhost");
+    conf.set("hadoop.kms.authentication.kerberos.name.rules", "DEFAULT");
+    conf.set("hadoop.security.kms.client.timeout", "300");
+    conf.set("hadoop.kms.proxyuser.client.users", "foo,bar");
+    conf.set("hadoop.kms.proxyuser.client.hosts", "*");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kAA.ALL", "foo");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kBB.ALL", "foo1");
+    conf.set(KeyAuthorizationKeyProvider.KEY_ACL + "kCC.ALL", "bar");
+
+    writeConf(testDir, conf);
+
+    runServer(null, null, testDir, new KMSCallable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        final Configuration conf = new Configuration();
+        conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 64);
+        final URI uri = createKMSUri(getKMSUrl());
+
+        UserGroupInformation proxyUgi = null;
+        if (kerberos) {
+          // proxyuser client using kerberos credentials
+          proxyUgi = UserGroupInformation.
+              loginUserFromKeytabAndReturnUGI("client", keytab.getAbsolutePath());
+        } else {
+          proxyUgi = UserGroupInformation.createRemoteUser("client");
+        }
+
+        final UserGroupInformation clientUgi = proxyUgi; 
+        clientUgi.doAs(new PrivilegedExceptionAction<Void>() {
+          @Override
+          public Void run() throws Exception {
+
+            // authorized proxyuser
+            UserGroupInformation fooUgi =
+                UserGroupInformation.createProxyUser("foo", clientUgi);
+            fooUgi.doAs(new PrivilegedExceptionAction<Void>() {
+              @Override
+              public Void run() throws Exception {
+                KeyProvider kp = new KMSClientProvider(uri, conf);
+                Assert.assertNotNull(kp.createKey("kAA",
+                    new KeyProvider.Options(conf)));
+                return null;
+              }
+            });
+
+            // unauthorized proxyuser
+            UserGroupInformation foo1Ugi =
+                UserGroupInformation.createProxyUser("foo1", clientUgi);
+            foo1Ugi.doAs(new PrivilegedExceptionAction<Void>() {
+              @Override
+              public Void run() throws Exception {
+                try {
+                  KeyProvider kp = new KMSClientProvider(uri, conf);
+                  kp.createKey("kBB", new KeyProvider.Options(conf));
+                  Assert.fail();
+                } catch (Exception ex) {
+                  Assert.assertTrue(ex.getMessage(), ex.getMessage().contains("Forbidden"));
+                }
+                return null;
+              }
+            });
+
+            // authorized proxyuser
+            UserGroupInformation barUgi =
+                UserGroupInformation.createProxyUser("bar", clientUgi);
+            barUgi.doAs(new PrivilegedExceptionAction<Void>() {
+              @Override
+              public Void run() throws Exception {
+                KeyProvider kp = new KMSClientProvider(uri, conf);
+                Assert.assertNotNull(kp.createKey("kCC",
+                    new KeyProvider.Options(conf)));
+                return null;
+              }
+            });
+
             return null;
           }
         });