HADOOP-17873. ABFS: Fix transient failures in ITestAbfsStreamStatistics and ITestAbfsRestOperationException (#3699)

Successor for the reverted PR #3341, using the hadoop @VisibleForTesting attribute

Contributed by Sumangala Patki

hadoop-tools/hadoop-azure/pom.xml

@@ -604,6 +604,7 @@
                     <exclude>**/azurebfs/ITestAzureBlobFileSystemListStatus.java</exclude>
                     <exclude>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</exclude>
                     <exclude>**/azurebfs/ITestSmallWriteOptimization.java</exclude>
+                    <exclude>**/azurebfs/ITestAbfsStreamStatistics*.java</exclude>
                     <exclude>**/azurebfs/services/ITestReadBufferManager.java</exclude>
                     <exclude>**/azurebfs/commit/*.java</exclude>
                   </excludes>
@@ -647,6 +648,7 @@
                     <include>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</include>
                     <include>**/azurebfs/ITestSmallWriteOptimization.java</include>
                     <include>**/azurebfs/services/ITestReadBufferManager.java</include>
+                    <include>**/azurebfs/ITestAbfsStreamStatistics*.java</include>
                     <include>**/azurebfs/commit/*.java</include>
                   </includes>
                 </configuration>

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java

@@ -22,6 +22,7 @@ package org.apache.hadoop.fs.azurebfs.oauth2;
 import java.io.IOException;
 import java.net.URI;
 
+import org.apache.hadoop.classification.VisibleForTesting;
 import org.apache.hadoop.util.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -138,4 +139,9 @@ public final class CustomTokenProviderAdapter extends AccessTokenProvider
     String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, "");
     return suffix != null ? suffix : "";
   }
+
+  @VisibleForTesting
+  protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() {
+    return adaptee;
+  }
 }

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java

@@ -1289,4 +1289,9 @@ public class AbfsClient implements Closeable {
   public <V> void addCallback(ListenableFuture<V> future, FutureCallback<V> callback) {
     Futures.addCallback(future, callback, executorService);
   }
+
+  @VisibleForTesting
+  protected AccessTokenProvider getTokenProvider() {
+    return tokenProvider;
+  }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java

@@ -37,10 +37,12 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
+import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
 import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
 import org.apache.hadoop.fs.azurebfs.services.AbfsClient;
 import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
 import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
 import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
 import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@@ -251,6 +253,9 @@ public abstract class AbstractAbfsIntegrationTest extends
     }
   }
 
+  public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) {
+    return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
+  }
 
   public void loadConfiguredFileSystem() throws Exception {
       // disable auto-creation of filesystem

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java

@@ -112,7 +112,10 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
     final AzureBlobFileSystem fs1 =
         (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
         config);
-    RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
+    RetryTestTokenProvider retryTestTokenProvider
+        = RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
+        getAccessTokenProvider(fs1));
+    retryTestTokenProvider.resetStatusToFirstTokenFetch();
 
     intercept(Exception.class,
         ()-> {
@@ -120,10 +123,10 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
         });
 
     // Number of retries done should be as configured
-    Assert.assertTrue(
+    Assert.assertEquals(
-        "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
+        "Number of token fetch retries done does not match with fs.azure"
-            + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries
+            + ".custom.token.fetch.retry.count configured", numOfRetries,
-            + ")", RetryTestTokenProvider.reTryCount == numOfRetries);
+        retryTestTokenProvider.getRetryCount());
   }
 
   @Test

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java

@@ -30,12 +30,12 @@ import org.slf4j.LoggerFactory;
  */
 public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
 
-  // Need to track first token fetch otherwise will get counted as a retry too.
+  private static final Logger LOG = LoggerFactory.getLogger(
-  private static boolean isThisFirstTokenFetch = true;
+      RetryTestTokenProvider.class);
-  public static int reTryCount = 0;
 
-  private static final Logger LOG = LoggerFactory
+  // Need to track first token fetch otherwise will get counted as a retry too.
-      .getLogger(RetryTestTokenProvider.class);
+  private boolean isThisFirstTokenFetch = true;
+  private int retryCount = 0;
 
   @Override
   public void initialize(Configuration configuration, String accountName)
@@ -43,9 +43,13 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
 
   }
 
-  public static void ResetStatusToFirstTokenFetch() {
+  /**
+   * Clear earlier retry details and reset RetryTestTokenProvider instance to
+   * state of first access token fetch call.
+   */
+  public void resetStatusToFirstTokenFetch() {
     isThisFirstTokenFetch = true;
-    reTryCount = 0;
+    retryCount = 0;
   }
 
   @Override
@@ -53,7 +57,7 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
     if (isThisFirstTokenFetch) {
       isThisFirstTokenFetch = false;
     } else {
-      reTryCount++;
+      retryCount++;
     }
 
     LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens");
@@ -64,4 +68,13 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
   public Date getExpiryTime() {
     return new Date();
   }
+
+  public static RetryTestTokenProvider getCurrentRetryTestProviderInstance(
+      AccessTokenProvider customTokenProvider) {
+    return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee();
+  }
+
+  public int getRetryCount() {
+    return retryCount;
+  }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java

@@ -395,4 +395,8 @@ public final class TestAbfsClient {
         url,
         requestHeaders);
   }
+
+  public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) {
+    return client.getTokenProvider();
+  }
 }