From 7b632dd22b5f1878357b04d97ed7f63a62fbb9de Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Fri, 5 Nov 2021 14:22:07 +0000 Subject: [PATCH] Revert "HADOOP-17873. ABFS: Fix transient failures in ITestAbfsStreamStatistics and ITestAbfsRestOperationException (#3341)" This reverts commit 0379aebafe68e014d3556ed55f1aa15c4d787944. --- hadoop-tools/hadoop-azure/pom.xml | 2 -- .../oauth2/CustomTokenProviderAdapter.java | 6 ---- .../fs/azurebfs/services/AbfsClient.java | 5 ---- .../azurebfs/AbstractAbfsIntegrationTest.java | 5 ---- .../ITestAbfsRestOperationException.java | 13 ++++----- .../oauth2/RetryTestTokenProvider.java | 29 +++++-------------- .../fs/azurebfs/services/TestAbfsClient.java | 4 --- 7 files changed, 13 insertions(+), 51 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index d6d1d1831d3..f610295715f 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -555,7 +555,6 @@ **/azurebfs/ITestAzureBlobFileSystemListStatus.java **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java - **/azurebfs/ITestAbfsStreamStatistics*.java **/azurebfs/services/ITestReadBufferManager.java @@ -598,7 +597,6 @@ **/azurebfs/extensions/ITestAbfsDelegationTokens.java **/azurebfs/ITestSmallWriteOptimization.java **/azurebfs/services/ITestReadBufferManager.java - **/azurebfs/ITestAbfsStreamStatistics*.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java index 6ae0962fbfd..14914101e5c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java @@ -22,7 +22,6 @@ package org.apache.hadoop.fs.azurebfs.oauth2; import java.io.IOException; import java.net.URI; -import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,9 +138,4 @@ public final class CustomTokenProviderAdapter extends AccessTokenProvider String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, ""); return suffix != null ? suffix : ""; } - - @VisibleForTesting - protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() { - return adaptee; - } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 50e224672d4..a473b36d82c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1223,9 +1223,4 @@ public class AbfsClient implements Closeable { public void addCallback(ListenableFuture future, FutureCallback callback) { Futures.addCallback(future, callback, executorService); } - - @VisibleForTesting - protected AccessTokenProvider getTokenProvider() { - return tokenProvider; - } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index c1387681c7a..fd2f2690dae 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -37,11 +37,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException; -import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation; @@ -241,9 +239,6 @@ public abstract class AbstractAbfsIntegrationTest extends } } - public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) { - return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient()); - } public void loadConfiguredFileSystem() throws Exception { // disable auto-creation of filesystem diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java index b1705cad015..a71e7bc815f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java @@ -111,10 +111,7 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest final AzureBlobFileSystem fs1 = (AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(), config); - RetryTestTokenProvider retryTestTokenProvider - = RetryTestTokenProvider.getCurrentRetryTestProviderInstance( - getAccessTokenProvider(fs1)); - retryTestTokenProvider.resetStatusToFirstTokenFetch(); + RetryTestTokenProvider.ResetStatusToFirstTokenFetch(); intercept(Exception.class, ()-> { @@ -122,10 +119,10 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest }); // Number of retries done should be as configured - Assert.assertEquals( - "Number of token fetch retries done does not match with fs.azure" - + ".custom.token.fetch.retry.count configured", numOfRetries, - retryTestTokenProvider.getRetryCount()); + Assert.assertTrue( + "Number of token fetch retries (" + RetryTestTokenProvider.reTryCount + + ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries + + ")", RetryTestTokenProvider.reTryCount == numOfRetries); } @Test diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java index 7427add2908..3566ebbaaaa 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java @@ -30,12 +30,12 @@ import org.slf4j.LoggerFactory; */ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { - private static final Logger LOG = LoggerFactory.getLogger( - RetryTestTokenProvider.class); - // Need to track first token fetch otherwise will get counted as a retry too. - private boolean isThisFirstTokenFetch = true; - private int retryCount = 0; + private static boolean isThisFirstTokenFetch = true; + public static int reTryCount = 0; + + private static final Logger LOG = LoggerFactory + .getLogger(RetryTestTokenProvider.class); @Override public void initialize(Configuration configuration, String accountName) @@ -43,13 +43,9 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { } - /** - * Clear earlier retry details and reset RetryTestTokenProvider instance to - * state of first access token fetch call. - */ - public void resetStatusToFirstTokenFetch() { + public static void ResetStatusToFirstTokenFetch() { isThisFirstTokenFetch = true; - retryCount = 0; + reTryCount = 0; } @Override @@ -57,7 +53,7 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { if (isThisFirstTokenFetch) { isThisFirstTokenFetch = false; } else { - retryCount++; + reTryCount++; } LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens"); @@ -68,13 +64,4 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee { public Date getExpiryTime() { return new Date(); } - - public static RetryTestTokenProvider getCurrentRetryTestProviderInstance( - AccessTokenProvider customTokenProvider) { - return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee(); - } - - public int getRetryCount() { - return retryCount; - } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index 0a1dca7e7d8..a725bf3175a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -395,8 +395,4 @@ public final class TestAbfsClient { url, requestHeaders); } - - public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) { - return client.getTokenProvider(); - } }