From f5e6be337b281cc04f13d5e68f9d5ba8fbba8761 Mon Sep 17 00:00:00 2001
From: Issac Buenrostro <issac.buenrostro@gmail.com>
Date: Thu, 12 Nov 2020 14:57:26 -0800
Subject: [PATCH] =?UTF-8?q?HDFS-15545=20-=20Allow=20WebHdfsFileSystem=20to?=
 =?UTF-8?q?=20read=20a=20new=20delegation=20token=20f=E2=80=A6=20(#2255)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Issac Buenrostro <ibuenros@linkedin.com>
---
 .../hadoop/hdfs/web/WebHdfsFileSystem.java    | 18 ++++++++++++++++--
 .../hadoop/hdfs/web/TestWebHdfsTokens.java    | 19 ++++++++++++++++++-
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 080fba87e83..d7dd9117284 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -368,8 +368,8 @@ public class WebHdfsFileSystem extends FileSystem
       Token<?> token = tokenSelector.selectToken(
           new Text(getCanonicalServiceName()), ugi.getTokens());
       // ugi tokens are usually indicative of a task which can't
-      // refetch tokens.  even if ugi has credentials, don't attempt
-      // to get another token to match hdfs/rpc behavior
+      // refetch tokens.  Don't attempt to fetch tokens from the
+      // namenode in this situation.
       if (token != null) {
         LOG.debug("Using UGI token: {}", token);
         canRefreshDelegationToken = false;
@@ -392,6 +392,9 @@ public class WebHdfsFileSystem extends FileSystem
   @VisibleForTesting
   synchronized boolean replaceExpiredDelegationToken() throws IOException {
     boolean replaced = false;
+    if (attemptReplaceDelegationTokenFromUGI()) {
+      return true;
+    }
     if (canRefreshDelegationToken) {
       Token<?> token = getDelegationToken(null);
       LOG.debug("Replaced expired token: {}", token);
@@ -401,6 +404,17 @@ public class WebHdfsFileSystem extends FileSystem
     return replaced;
   }
 
+  private synchronized boolean attemptReplaceDelegationTokenFromUGI() {
+    Token<?> token = tokenSelector.selectToken(
+            new Text(getCanonicalServiceName()), ugi.getTokens());
+    if (token != null && !token.equals(delegationToken)) {
+      LOG.debug("Replaced expired token with new UGI token: {}", token);
+      setDelegationToken(token);
+      return true;
+    }
+    return false;
+  }
+
   @Override
   protected int getDefaultPort() {
     return HdfsClientConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
index 0b8c201ddb2..cb35bd9e68c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
@@ -522,7 +522,8 @@ public class TestWebHdfsTokens {
     Assert.assertSame(token, token2);
     reset(fs);
 
-    // verify an expired ugi token is NOT replaced with a new token
+    // verify an expired ugi token is not replaced with a new token if the
+    // ugi does not contain a new token
     fs.cancelDelegationToken(token);
     for (int i=0; i<2; i++) {
       try {
@@ -543,6 +544,22 @@ public class TestWebHdfsTokens {
       reset(fs);
     }
     
+    // verify an expired ugi token is replaced with a new token
+    // if one is available in UGI
+    token = fs.getDelegationToken(null);
+    ugi.addToken(token);
+    reset(fs);
+    fs.getFileStatus(new Path("/"));
+    verify(fs, times(2)).getDelegationToken(); // first bad, then good
+    verify(fs, times(1)).replaceExpiredDelegationToken();
+    verify(fs, never()).getDelegationToken(anyString());
+    verify(fs, times(1)).setDelegationToken(eq(token));
+    token2 = fs.getRenewToken();
+    Assert.assertNotNull(token2);
+    Assert.assertEquals(fs.getTokenKind(), token.getKind());
+    Assert.assertSame(token, token2);
+    reset(fs);
+
     // verify fs close does NOT cancel the ugi token
     fs.close();
     verify(fs, never()).getDelegationToken();