From baa8670256ffd12a2c0a966f4ba1efff217489c5 Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Tue, 9 Oct 2018 20:02:12 +0100 Subject: [PATCH] HADOOP-15825. ABFS: Enable some tests for namespace not enabled account using OAuth. Contributed by Da Zhou. (cherry picked from commit bd50fa956b1ca25bb2136977b98a6aa6895eff8b) --- .../fs/azurebfs/AbstractAbfsIntegrationTest.java | 3 ++- .../ITestAzureBlobFileSystemBackCompat.java | 6 ++---- .../ITestAzureBlobFileSystemFileStatus.java | 12 ++++++------ .../azurebfs/ITestAzureBlobFileSystemFinalize.java | 8 +++----- .../fs/azurebfs/ITestAzureBlobFileSystemFlush.java | 5 ----- .../ITestAzureBlobFileSystemInitAndCreate.java | 3 +-- .../ITestAzureBlobFileSystemRandomRead.java | 12 +++++------- .../fs/azurebfs/ITestWasbAbfsCompatibility.java | 14 ++++++++------ 8 files changed, 27 insertions(+), 36 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java index 568fdd92336..cb9549d81de 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java @@ -133,7 +133,8 @@ public abstract class AbstractAbfsIntegrationTest extends //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem. createFileSystem(); - if (!isIPAddress && authType == AuthType.SharedKey) { + // Only live account without namespace support can run ABFS&WASB compatibility tests + if (!isIPAddress && !abfs.getIsNamespaceEnabled()) { final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl())); final AzureNativeFileSystemStore azureNativeFileSystemStore = new AzureNativeFileSystemStore(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index 22d4990cc42..5ac16b45a45 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -26,7 +26,6 @@ import com.microsoft.azure.storage.blob.CloudBlockBlob; import org.junit.Assume; import org.junit.Test; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; @@ -38,14 +37,13 @@ public class ITestAzureBlobFileSystemBackCompat extends public ITestAzureBlobFileSystemBackCompat() throws Exception { super(); - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test public void testBlobBackCompat() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); - // test only valid for non-namespace enabled account - Assume.assumeFalse(fs.getIsNamespaceEnabled()); + Assume.assumeFalse("This test does not support namespace enabled account", + this.getFileSystem().getIsNamespaceEnabled()); String storageConnectionString = getBlobConnectionString(); CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index 707a1452d50..f5146963304 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -21,8 +21,6 @@ package org.apache.hadoop.fs.azurebfs; import java.io.IOException; import org.apache.hadoop.fs.CommonConfigurationKeys; -import org.apache.hadoop.fs.azurebfs.services.AuthType; -import org.junit.Ignore; import org.junit.Test; import org.apache.hadoop.fs.FileStatus; @@ -37,6 +35,7 @@ public class ITestAzureBlobFileSystemFileStatus extends private static final String DEFAULT_FILE_PERMISSION_VALUE = "640"; private static final String DEFAULT_DIR_PERMISSION_VALUE = "750"; private static final String DEFAULT_UMASK_VALUE = "027"; + private static final String FULL_PERMISSION = "777"; private static final Path TEST_FILE = new Path("testFile"); private static final Path TEST_FOLDER = new Path("testDir"); @@ -54,7 +53,6 @@ public class ITestAzureBlobFileSystemFileStatus extends assertEquals("root listing", 0, rootls.length); } - @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.") @Test public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); @@ -69,13 +67,16 @@ public class ITestAzureBlobFileSystemFileStatus extends String errorInStatus = "error in " + fileStatus + " from " + fs; - // When running with Oauth, the owner and group info retrieved from server will be digit ids. - if (this.getAuthType() != AuthType.OAuth && !fs.isSecureScheme()) { + if (!fs.getIsNamespaceEnabled()) { assertEquals(errorInStatus + ": owner", fs.getOwnerUser(), fileStatus.getOwner()); assertEquals(errorInStatus + ": group", fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup()); + assertEquals(new FsPermission(FULL_PERMISSION), fileStatus.getPermission()); } else { + // When running with namespace enabled account, + // the owner and group info retrieved from server will be digit ids. + // hence skip the owner and group validation if (isDir) { assertEquals(errorInStatus + ": permission", new FsPermission(DEFAULT_DIR_PERMISSION_VALUE), fileStatus.getPermission()); @@ -88,7 +89,6 @@ public class ITestAzureBlobFileSystemFileStatus extends return fileStatus; } - @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.") @Test public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java index 9d1738857c6..3c21525549b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java @@ -25,14 +25,13 @@ import org.junit.Test; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.azurebfs.services.AuthType; /** * Test finalize() method when "fs.abfs.impl.disable.cache" is enabled. */ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ static final String DISABLE_ABFS_CACHE_KEY = "fs.abfs.impl.disable.cache"; - static final String DISABLE_ABFSSS_CACHE_KEY = "fs.abfss.impl.disable.cache"; + static final String DISABLE_ABFSS_CACHE_KEY = "fs.abfss.impl.disable.cache"; public ITestAzureBlobFileSystemFinalize() throws Exception { super(); @@ -42,9 +41,8 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{ public void testFinalize() throws Exception { // Disable the cache for filesystem to make sure there is no reference. Configuration rawConfig = this.getRawConfiguration(); - rawConfig.setBoolean( - this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY, - true); + rawConfig.setBoolean(DISABLE_ABFS_CACHE_KEY, true); + rawConfig.setBoolean(DISABLE_ABFSS_CACHE_KEY, true); AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 23a1ab5bb72..d60cae868f0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; import org.hamcrest.core.IsEqual; import org.hamcrest.core.IsNot; -import org.junit.Assume; import org.junit.Test; import org.apache.hadoop.fs.FSDataInputStream; @@ -42,8 +41,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.azurebfs.services.AuthType; - /** * Test flush operation. * This class cannot be run in parallel test mode--check comments in @@ -221,8 +218,6 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest { } private void testFlush(boolean flushEnabled) throws Exception { - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); - final AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem(); // Simulate setting "fs.azure.enable.flush" to true or false diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index 5f08721ada0..5c4b87b0d2f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -46,8 +46,7 @@ public class ITestAzureBlobFileSystemInitAndCreate extends @Test (expected = FileNotFoundException.class) public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception { - super.setup(); - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = this.createFileSystem(); FileStatus[] fileStatuses = fs.listStatus(new Path("/")); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java index 768f4bbc5d9..e5f64b5f2c0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java @@ -34,7 +34,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import org.apache.hadoop.fs.contract.ContractTestUtils; import static org.apache.hadoop.test.LambdaTestUtils.intercept; @@ -67,7 +66,6 @@ public class ITestAzureBlobFileSystemRandomRead extends public ITestAzureBlobFileSystemRandomRead() throws Exception { super(); - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test @@ -98,6 +96,8 @@ public class ITestAzureBlobFileSystemRandomRead extends */ @Test public void testRandomRead() throws Exception { + Assume.assumeFalse("This test does not support namespace enabled account", + this.getFileSystem().getIsNamespaceEnabled()); assumeHugeFileExists(); try ( FSDataInputStream inputStreamV1 @@ -413,6 +413,8 @@ public class ITestAzureBlobFileSystemRandomRead extends @Test public void testRandomReadPerformance() throws Exception { + Assume.assumeFalse("This test does not support namespace enabled account", + this.getFileSystem().getIsNamespaceEnabled()); createTestFile(); assumeHugeFileExists(); @@ -523,11 +525,7 @@ public class ITestAzureBlobFileSystemRandomRead extends } private void createTestFile() throws Exception { - final AzureBlobFileSystem abFs = this.getFileSystem(); - // test only valid for non-namespace enabled account - Assume.assumeFalse(abFs.getIsNamespaceEnabled()); - FileSystem fs = this.getWasbFileSystem(); - + final AzureBlobFileSystem fs = this.getFileSystem(); if (fs.exists(TEST_FILE_PATH)) { FileStatus status = fs.getFileStatus(TEST_FILE_PATH); if (status.getLen() >= TEST_FILE_SIZE) { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java index fdf9788c36f..e8f845c9a39 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java @@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.azurebfs.services.AuthType; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted; import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory; @@ -51,7 +50,6 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { public ITestWasbAbfsCompatibility() throws Exception { Assume.assumeFalse("Emulator is not supported", isIPAddress()); - Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey); } @Test @@ -59,7 +57,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { // crate file using abfs AzureBlobFileSystem fs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(fs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test,", + fs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem(); @@ -93,7 +92,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { AzureBlobFileSystem abfs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(abfs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test", + abfs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem(); @@ -132,7 +132,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { AzureBlobFileSystem abfs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(abfs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test", + abfs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem(); @@ -166,7 +167,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest { //create folders AzureBlobFileSystem abfs = getFileSystem(); // test only valid for non-namespace enabled account - Assume.assumeFalse(abfs.getIsNamespaceEnabled()); + Assume.assumeFalse("Namespace enabled account does not support this test", + abfs.getIsNamespaceEnabled()); NativeAzureFileSystem wasb = getWasbFileSystem();