HADOOP-15825. ABFS: Enable some tests for namespace not enabled account using OAuth.

Contributed by Da Zhou.

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsIntegrationTest.java

@@ -133,7 +133,8 @@ public abstract class AbstractAbfsIntegrationTest extends
     //Create filesystem first to make sure getWasbFileSystem() can return an existing filesystem.
     createFileSystem();
 
-    if (!isIPAddress && authType == AuthType.SharedKey) {
+    // Only live account without namespace support can run ABFS&WASB compatibility tests
+    if (!isIPAddress && !abfs.getIsNamespaceEnabled()) {
       final URI wasbUri = new URI(abfsUrlToWasbUrl(getTestUrl()));
       final AzureNativeFileSystemStore azureNativeFileSystemStore =
           new AzureNativeFileSystemStore();

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java

@@ -26,7 +26,6 @@ import com.microsoft.azure.storage.blob.CloudBlockBlob;
 import org.junit.Assume;
 import org.junit.Test;
 
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 
@@ -38,14 +37,13 @@ public class ITestAzureBlobFileSystemBackCompat extends
 
   public ITestAzureBlobFileSystemBackCompat() throws Exception {
     super();
-    Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
   }
 
   @Test
   public void testBlobBackCompat() throws Exception {
     final AzureBlobFileSystem fs = this.getFileSystem();
-    // test only valid for non-namespace enabled account
+    Assume.assumeFalse("This test does not support namespace enabled account",
-    Assume.assumeFalse(fs.getIsNamespaceEnabled());
+            this.getFileSystem().getIsNamespaceEnabled());
     String storageConnectionString = getBlobConnectionString();
     CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
     CloudBlobClient blobClient = storageAccount.createCloudBlobClient();

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java

@@ -21,8 +21,6 @@ package org.apache.hadoop.fs.azurebfs;
 import java.io.IOException;
 
 import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.FileStatus;
@@ -37,6 +35,7 @@ public class ITestAzureBlobFileSystemFileStatus extends
   private static final String DEFAULT_FILE_PERMISSION_VALUE = "640";
   private static final String DEFAULT_DIR_PERMISSION_VALUE = "750";
   private static final String DEFAULT_UMASK_VALUE = "027";
+  private static final String FULL_PERMISSION = "777";
 
   private static final Path TEST_FILE = new Path("testFile");
   private static final Path TEST_FOLDER = new Path("testDir");
@@ -54,7 +53,6 @@ public class ITestAzureBlobFileSystemFileStatus extends
     assertEquals("root listing", 0, rootls.length);
   }
 
-  @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.")
   @Test
   public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception {
     final AzureBlobFileSystem fs = this.getFileSystem();
@@ -69,13 +67,16 @@ public class ITestAzureBlobFileSystemFileStatus extends
 
     String errorInStatus = "error in " + fileStatus + " from " + fs;
 
-    // When running with Oauth, the owner and group info retrieved from server will be digit ids.
+    if (!fs.getIsNamespaceEnabled()) {
-    if (this.getAuthType() != AuthType.OAuth && !fs.isSecureScheme()) {
       assertEquals(errorInStatus + ": owner",
               fs.getOwnerUser(), fileStatus.getOwner());
       assertEquals(errorInStatus + ": group",
               fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup());
+      assertEquals(new FsPermission(FULL_PERMISSION), fileStatus.getPermission());
     } else {
+      // When running with namespace enabled account,
+      // the owner and group info retrieved from server will be digit ids.
+      // hence skip the owner and group validation
       if (isDir) {
         assertEquals(errorInStatus + ": permission",
                 new FsPermission(DEFAULT_DIR_PERMISSION_VALUE), fileStatus.getPermission());
@@ -88,7 +89,6 @@ public class ITestAzureBlobFileSystemFileStatus extends
     return fileStatus;
   }
 
-  @Ignore("When running against live abfs with Oauth account, this test will fail. Need to check the tenant.")
   @Test
   public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception {
     final AzureBlobFileSystem fs = this.getFileSystem();

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFinalize.java

@@ -25,14 +25,13 @@ import org.junit.Test;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
 
 /**
  * Test finalize() method when "fs.abfs.impl.disable.cache" is enabled.
  */
 public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
   static final String DISABLE_ABFS_CACHE_KEY = "fs.abfs.impl.disable.cache";
-  static final String DISABLE_ABFSSS_CACHE_KEY = "fs.abfss.impl.disable.cache";
+  static final String DISABLE_ABFSS_CACHE_KEY = "fs.abfss.impl.disable.cache";
 
   public ITestAzureBlobFileSystemFinalize() throws Exception {
     super();
@@ -42,9 +41,8 @@ public class ITestAzureBlobFileSystemFinalize extends AbstractAbfsScaleTest{
   public void testFinalize() throws Exception {
     // Disable the cache for filesystem to make sure there is no reference.
     Configuration rawConfig = this.getRawConfiguration();
-    rawConfig.setBoolean(
+    rawConfig.setBoolean(DISABLE_ABFS_CACHE_KEY, true);
-            this.getAuthType() == AuthType.SharedKey ? DISABLE_ABFS_CACHE_KEY : DISABLE_ABFSSS_CACHE_KEY,
+    rawConfig.setBoolean(DISABLE_ABFSS_CACHE_KEY, true);
-    true);
 
     AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(rawConfig);
 

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java

@@ -33,7 +33,6 @@ import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
 import org.hamcrest.core.IsEqual;
 import org.hamcrest.core.IsNot;
-import org.junit.Assume;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -42,8 +41,6 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
-
 /**
  * Test flush operation.
  * This class cannot be run in parallel test mode--check comments in
@@ -221,8 +218,6 @@ public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest {
   }
 
   private void testFlush(boolean flushEnabled) throws Exception {
-    Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
-
     final AzureBlobFileSystem fs = (AzureBlobFileSystem) getFileSystem();
 
     // Simulate setting "fs.azure.enable.flush" to true or false

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java

@@ -46,8 +46,7 @@ public class ITestAzureBlobFileSystemInitAndCreate extends
 
   @Test (expected = FileNotFoundException.class)
   public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception {
-    super.setup();
+    final AzureBlobFileSystem fs = this.createFileSystem();
-    final AzureBlobFileSystem fs = this.getFileSystem();
     FileStatus[] fileStatuses = fs.listStatus(new Path("/"));
   }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java

@@ -34,7 +34,6 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -67,7 +66,6 @@ public class ITestAzureBlobFileSystemRandomRead extends
 
   public ITestAzureBlobFileSystemRandomRead() throws Exception {
     super();
-    Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
   }
 
   @Test
@@ -98,6 +96,8 @@ public class ITestAzureBlobFileSystemRandomRead extends
    */
   @Test
   public void testRandomRead() throws Exception {
+    Assume.assumeFalse("This test does not support namespace enabled account",
+            this.getFileSystem().getIsNamespaceEnabled());
     assumeHugeFileExists();
     try (
             FSDataInputStream inputStreamV1
@@ -413,6 +413,8 @@ public class ITestAzureBlobFileSystemRandomRead extends
 
   @Test
   public void testRandomReadPerformance() throws Exception {
+    Assume.assumeFalse("This test does not support namespace enabled account",
+            this.getFileSystem().getIsNamespaceEnabled());
     createTestFile();
     assumeHugeFileExists();
 
@@ -523,11 +525,7 @@ public class ITestAzureBlobFileSystemRandomRead extends
   }
 
   private void createTestFile() throws Exception {
-    final AzureBlobFileSystem abFs = this.getFileSystem();
+    final AzureBlobFileSystem fs = this.getFileSystem();
-    // test only valid for non-namespace enabled account
-    Assume.assumeFalse(abFs.getIsNamespaceEnabled());
-    FileSystem fs = this.getWasbFileSystem();
-
     if (fs.exists(TEST_FILE_PATH)) {
       FileStatus status = fs.getFileStatus(TEST_FILE_PATH);
       if (status.getLen() >= TEST_FILE_SIZE) {

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestWasbAbfsCompatibility.java

@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.azure.NativeAzureFileSystem;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.apache.hadoop.fs.azurebfs.services.AuthType;
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory;
@@ -51,7 +50,6 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
 
   public ITestWasbAbfsCompatibility() throws Exception {
     Assume.assumeFalse("Emulator is not supported", isIPAddress());
-    Assume.assumeTrue(this.getAuthType() == AuthType.SharedKey);
   }
 
   @Test
@@ -59,7 +57,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
     // crate file using abfs
     AzureBlobFileSystem fs = getFileSystem();
     // test only valid for non-namespace enabled account
-    Assume.assumeFalse(fs.getIsNamespaceEnabled());
+    Assume.assumeFalse("Namespace enabled account does not support this test,",
+            fs.getIsNamespaceEnabled());
 
     NativeAzureFileSystem wasb = getWasbFileSystem();
 
@@ -93,7 +92,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
 
     AzureBlobFileSystem abfs = getFileSystem();
     // test only valid for non-namespace enabled account
-    Assume.assumeFalse(abfs.getIsNamespaceEnabled());
+    Assume.assumeFalse("Namespace enabled account does not support this test",
+            abfs.getIsNamespaceEnabled());
 
     NativeAzureFileSystem wasb = getWasbFileSystem();
 
@@ -132,7 +132,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
 
     AzureBlobFileSystem abfs = getFileSystem();
     // test only valid for non-namespace enabled account
-    Assume.assumeFalse(abfs.getIsNamespaceEnabled());
+    Assume.assumeFalse("Namespace enabled account does not support this test",
+            abfs.getIsNamespaceEnabled());
 
     NativeAzureFileSystem wasb = getWasbFileSystem();
 
@@ -166,7 +167,8 @@ public class ITestWasbAbfsCompatibility extends AbstractAbfsIntegrationTest {
     //create folders
     AzureBlobFileSystem abfs = getFileSystem();
     // test only valid for non-namespace enabled account
-    Assume.assumeFalse(abfs.getIsNamespaceEnabled());
+    Assume.assumeFalse("Namespace enabled account does not support this test",
+            abfs.getIsNamespaceEnabled());
 
     NativeAzureFileSystem wasb = getWasbFileSystem();