From 35e4c31fff9946d35a3543481585031558e9f5d5 Mon Sep 17 00:00:00 2001 From: snehavarma Date: Sun, 13 Jun 2021 23:52:29 +0530 Subject: [PATCH] HADOOP-17714 ABFS: testBlobBackCompatibility, testRandomRead & WasbAbfsCompatibility tests fail when triggered with default configs (#3035) --- .../fs/azurebfs/AbstractAbfsIntegrationTest.java | 16 +++++++++------- .../ITestAzureBlobFileSystemBackCompat.java | 6 ++++++ .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 2 +- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 7b3b5c10991..3befee48493 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -149,7 +149,7 @@ public abstract class AbstractAbfsIntegrationTest extends if (!isIPAddress && (abfsConfig.getAuthType(accountName) != AuthType.SAS) && !abfs.getIsNamespaceEnabled()) { - final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); + final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl(), abfsConfig.isHttpsAlwaysUsed())); final AzureNativeFileSystemStore azureNativeFileSystemStore = new AzureNativeFileSystemStore(); @@ -355,13 +355,13 @@ public abstract class AbstractAbfsIntegrationTest extends protected static String wasbUrlToAbfsUrl(final String wasbUrl) { return convertTestUrls( wasbUrl, FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX, - FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX); + FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX, false); } - protected static String abfsUrlToWasbUrl(final String abfsUrl) { + protected static String abfsUrlToWasbUrl(final String abfsUrl, final boolean isAlwaysHttpsUsed) { return convertTestUrls( abfsUrl, FileSystemUriSchemes.ABFS_SCHEME, FileSystemUriSchemes.ABFS_SECURE_SCHEME, FileSystemUriSchemes.ABFS_DNS_PREFIX, - FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX); + FileSystemUriSchemes.WASB_SCHEME, FileSystemUriSchemes.WASB_SECURE_SCHEME, FileSystemUriSchemes.WASB_DNS_PREFIX, isAlwaysHttpsUsed); } private static String convertTestUrls( @@ -371,15 +371,17 @@ public abstract class AbstractAbfsIntegrationTest extends final String fromDnsPrefix, final String toNonSecureScheme, final String toSecureScheme, - final String toDnsPrefix) { + final String toDnsPrefix, + final boolean isAlwaysHttpsUsed) { String data = null; - if (url.startsWith(fromNonSecureScheme + "://")) { + if (url.startsWith(fromNonSecureScheme + "://") && isAlwaysHttpsUsed) { + data = url.replace(fromNonSecureScheme + "://", toSecureScheme + "://"); + } else if (url.startsWith(fromNonSecureScheme + "://")) { data = url.replace(fromNonSecureScheme + "://", toNonSecureScheme + "://"); } else if (url.startsWith(fromSecureScheme + "://")) { data = url.replace(fromSecureScheme + "://", toSecureScheme + "://"); } - if (data != null) { data = data.replace("." + fromDnsPrefix + ".", "." + toDnsPrefix + "."); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index 5ac16b45a45..ffa3c8cf7d8 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -74,6 +74,12 @@ public class ITestAzureBlobFileSystemBackCompat extends + ";AccountName=" + this.getAccountName().split("\\.")[0] + ";AccountKey=" + this.getAccountKey(); } + else if (this.getConfiguration().isHttpsAlwaysUsed()) { + connectionString = "DefaultEndpointsProtocol=https;BlobEndpoint=https://" + + this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.") + + ";AccountName=" + this.getAccountName().split("\\.")[0] + + ";AccountKey=" + this.getAccountKey(); + } else { connectionString = "DefaultEndpointsProtocol=http;BlobEndpoint=http://" + this.getAccountName().replaceFirst("\\.dfs\\.", ".blob.") diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index e8f845c9a39..609512d9c04 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -159,7 +159,7 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { String abfsUrl = "abfs://abcde-1111-1111-1111-1111@xxxx.dfs.xxx.xxx.xxxx.xxxx"; String wabsUrl = "wasb://abcde-1111-1111-1111-1111@xxxx.blob.xxx.xxx.xxxx.xxxx"; assertEquals(abfsUrl, wasbUrlToAbfsUrl(wabsUrl)); - assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl)); + assertEquals(wabsUrl, abfsUrlToWasbUrl(abfsUrl, false)); } @Test