Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "HADOOP-17873. ABFS: Fix transient failures in ITestAbfsStreamStatistics and ITestAbfsRestOperationException (#3341)" 

This reverts commit 0379aebafe.
This commit is contained in:
Steve Loughran 2021-11-05 14:22:07 +00:00
parent 689dd7bf17
commit 7b632dd22b
7 changed files with 13 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-tools/hadoop-azure/pom.xml
View File

@ -555,7 +555,6 @@
<exclude>**/azurebfs/ITestAzureBlobFileSystemListStatus.java</exclude>
<exclude>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</exclude>
<exclude>**/azurebfs/ITestSmallWriteOptimization.java</exclude>
<exclude>**/azurebfs/ITestAbfsStreamStatistics*.java</exclude>
<exclude>**/azurebfs/services/ITestReadBufferManager.java</exclude>
</excludes>
@ -598,7 +597,6 @@
<include>**/azurebfs/extensions/ITestAbfsDelegationTokens.java</include>
<include>**/azurebfs/ITestSmallWriteOptimization.java</include>
<include>**/azurebfs/services/ITestReadBufferManager.java</include>
<include>**/azurebfs/ITestAbfsStreamStatistics*.java</include>
</includes>
</configuration>
</execution>

6
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/CustomTokenProviderAdapter.java
View File

@ -22,7 +22,6 @@ package org.apache.hadoop.fs.azurebfs.oauth2;
import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -139,9 +138,4 @@ public final class CustomTokenProviderAdapter extends AccessTokenProvider
String suffix = ExtensionHelper.getUserAgentSuffix(adaptee, "");
return suffix != null ? suffix : "";
}
@VisibleForTesting
protected CustomTokenProviderAdaptee getCustomTokenProviderAdaptee() {
return adaptee;
}
}

5
hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java
View File

@ -1223,9 +1223,4 @@ public class AbfsClient implements Closeable {
public <V> void addCallback(ListenableFuture<V> future, FutureCallback<V> callback) {
Futures.addCallback(future, callback, executorService);
}
@VisibleForTesting
protected AccessTokenProvider getTokenProvider() {
return tokenProvider;
}
}

5
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java
View File

@ -37,11 +37,9 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.azurebfs.constants.FSOperationType;
import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider;
import org.apache.hadoop.fs.azurebfs.security.AbfsDelegationTokenManager;
import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
import org.apache.hadoop.fs.azurebfs.services.AuthType;
import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient;
import org.apache.hadoop.fs.azure.AzureNativeFileSystemStore;
import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
import org.apache.hadoop.fs.azure.metrics.AzureFileSystemInstrumentation;
@ -241,9 +239,6 @@ public abstract class AbstractAbfsIntegrationTest extends
}
}
public AccessTokenProvider getAccessTokenProvider(final AzureBlobFileSystem fs) {
return TestAbfsClient.getAccessTokenProvider(fs.getAbfsStore().getClient());
}
public void loadConfiguredFileSystem() throws Exception {
// disable auto-creation of filesystem

13
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsRestOperationException.java
View File

@ -111,10 +111,7 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
final AzureBlobFileSystem fs1 =
(AzureBlobFileSystem) FileSystem.newInstance(fs.getUri(),
config);
RetryTestTokenProvider retryTestTokenProvider
= RetryTestTokenProvider.getCurrentRetryTestProviderInstance(
getAccessTokenProvider(fs1));
retryTestTokenProvider.resetStatusToFirstTokenFetch();
RetryTestTokenProvider.ResetStatusToFirstTokenFetch();
intercept(Exception.class,
()-> {
@ -122,10 +119,10 @@ public class ITestAbfsRestOperationException extends AbstractAbfsIntegrationTest
});
// Number of retries done should be as configured
Assert.assertEquals(
"Number of token fetch retries done does not match with fs.azure"
+ ".custom.token.fetch.retry.count configured", numOfRetries,
retryTestTokenProvider.getRetryCount());
Assert.assertTrue(
"Number of token fetch retries (" + RetryTestTokenProvider.reTryCount
+ ") done, does not match with fs.azure.custom.token.fetch.retry.count configured (" + numOfRetries
+ ")", RetryTestTokenProvider.reTryCount == numOfRetries);
}
@Test

29
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/oauth2/RetryTestTokenProvider.java
View File

@ -30,12 +30,12 @@ import org.slf4j.LoggerFactory;
*/
public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
private static final Logger LOG = LoggerFactory.getLogger(
RetryTestTokenProvider.class);
// Need to track first token fetch otherwise will get counted as a retry too.
private boolean isThisFirstTokenFetch = true;
private int retryCount = 0;
private static boolean isThisFirstTokenFetch = true;
public static int reTryCount = 0;
private static final Logger LOG = LoggerFactory
.getLogger(RetryTestTokenProvider.class);
@Override
public void initialize(Configuration configuration, String accountName)
@ -43,13 +43,9 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
}
/**
* Clear earlier retry details and reset RetryTestTokenProvider instance to
* state of first access token fetch call.
*/
public void resetStatusToFirstTokenFetch() {
public static void ResetStatusToFirstTokenFetch() {
isThisFirstTokenFetch = true;
retryCount = 0;
reTryCount = 0;
}
@Override
@ -57,7 +53,7 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
if (isThisFirstTokenFetch) {
isThisFirstTokenFetch = false;
} else {
retryCount++;
reTryCount++;
}
LOG.debug("RetryTestTokenProvider: Throw an exception in fetching tokens");
@ -68,13 +64,4 @@ public class RetryTestTokenProvider implements CustomTokenProviderAdaptee {
public Date getExpiryTime() {
return new Date();
}
public static RetryTestTokenProvider getCurrentRetryTestProviderInstance(
AccessTokenProvider customTokenProvider) {
return (RetryTestTokenProvider) ((CustomTokenProviderAdapter) customTokenProvider).getCustomTokenProviderAdaptee();
}
public int getRetryCount() {
return retryCount;
}
}

4
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java
View File

@ -395,8 +395,4 @@ public final class TestAbfsClient {
url,
requestHeaders);
}
public static AccessTokenProvider getAccessTokenProvider(AbfsClient client) {
return client.getTokenProvider();
}
}