From f4329990250bed62efdebe3ce2bc740092cf9573 Mon Sep 17 00:00:00 2001 From: Mingliang Liu Date: Thu, 16 Feb 2017 15:14:25 -0800 Subject: [PATCH] HADOOP-14049. Honour AclBit flag associated to file/folder permission for Azure datalake account. Contributed by Vishwajeet Dusane --- .../apache/hadoop/fs/adl/AdlFileSystem.java | 6 +++- .../hadoop/fs/adl/TestADLResponseData.java | 21 ++++++++++++ .../hadoop/fs/adl/TestGetFileStatus.java | 25 +++++++++++++++ .../apache/hadoop/fs/adl/TestListStatus.java | 32 +++++++++++++++++++ 4 files changed, 83 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java index 303b7bcbdfc..fb0feda886e 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java +++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java @@ -593,7 +593,11 @@ private FileStatus toFileStatus(final DirectoryEntry entry, final Path f) { boolean isDirectory = entry.type == DirectoryEntryType.DIRECTORY; long lastModificationData = entry.lastModifiedTime.getTime(); long lastAccessTime = entry.lastAccessTime.getTime(); - FsPermission permission = new AdlPermission(aclBitStatus, + // set aclBit from ADLS backend response if + // ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION is true. + final boolean aclBit = aclBitStatus ? entry.aclBit : false; + + FsPermission permission = new AdlPermission(aclBit, Short.valueOf(entry.permission, 8)); String user = entry.user; String group = entry.group; diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java index 24eb31477ff..788242e3cd6 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestADLResponseData.java @@ -66,6 +66,15 @@ public static String getGetFileStatusJSONResponse(long length) { "\"owner\":\"NotSupportYet\",\"group\":\"NotSupportYet\"}}"; } + public static String getGetFileStatusJSONResponse(boolean aclBit) { + return "{\"FileStatus\":{\"length\":1024," + + "\"pathSuffix\":\"\",\"type\":\"FILE\",\"blockSize\":268435456," + + "\"accessTime\":1452103827023,\"modificationTime\":1452103827023," + + "\"replication\":0,\"permission\":\"777\"," + + "\"owner\":\"NotSupportYet\",\"group\":\"NotSupportYet\",\"aclBit\":\"" + + aclBit + "\"}}"; + } + public static String getListFileStatusJSONResponse(int dirSize) { String list = ""; for (int i = 0; i < dirSize; ++i) { @@ -81,6 +90,18 @@ public static String getListFileStatusJSONResponse(int dirSize) { return "{\"FileStatuses\":{\"FileStatus\":[" + list + "]}}"; } + public static String getListFileStatusJSONResponse(boolean aclBit) { + return "{\"FileStatuses\":{\"FileStatus\":[{\"length\":0,\"pathSuffix\":\"" + + java.util.UUID.randomUUID() + + "\",\"type\":\"DIRECTORY\",\"blockSize\":0," + + "\"accessTime\":1481184513488," + + "\"modificationTime\":1481184513488,\"replication\":0," + + "\"permission\":\"770\"," + + "\"owner\":\"4b27fe1a-d9ab-4a04-ad7a-4bba72cd9e6c\"," + + "\"group\":\"4b27fe1a-d9ab-4a04-ad7a-4bba72cd9e6c\",\"aclBit\":\"" + + aclBit + "\"}]}}"; + } + public static String getJSONResponse(boolean status) { return "{\"boolean\":" + status + "}"; } diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java index 08c805ea814..78ef9313bac 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java @@ -67,4 +67,29 @@ public void getFileStatusReturnsAsExpected() Assert.assertEquals("NotSupportYet", fileStatus.getGroup()); } + @Test + public void getFileStatusAclBit() + throws URISyntaxException, IOException { + // With ACLBIT set to true + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true))); + long startTime = Time.monotonicNow(); + FileStatus fileStatus = getMockAdlFileSystem() + .getFileStatus(new Path("/test1/test2")); + long endTime = Time.monotonicNow(); + LOG.debug("Time : " + (endTime - startTime)); + Assert.assertTrue(fileStatus.isFile()); + Assert.assertEquals(true, fileStatus.getPermission().getAclBit()); + + // With ACLBIT set to false + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false))); + startTime = Time.monotonicNow(); + fileStatus = getMockAdlFileSystem() + .getFileStatus(new Path("/test1/test2")); + endTime = Time.monotonicNow(); + LOG.debug("Time : " + (endTime - startTime)); + Assert.assertTrue(fileStatus.isFile()); + Assert.assertEquals(false, fileStatus.getPermission().getAclBit()); + } } diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java index c151e8940f9..dac8886a4c6 100644 --- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java @@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.net.URISyntaxException; /** * This class is responsible for testing local listStatus implementation to @@ -100,4 +101,35 @@ public void listStatusOnFailure() throws IOException { LOG.debug("Time : " + (endTime - startTime)); } + @Test + public void listStatusAclBit() + throws URISyntaxException, IOException { + // With ACLBIT set to true + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getListFileStatusJSONResponse(true))); + FileStatus[] ls = null; + long startTime = Time.monotonicNow(); + ls = getMockAdlFileSystem() + .listStatus(new Path("/test1/test2")); + long endTime = Time.monotonicNow(); + LOG.debug("Time : " + (endTime - startTime)); + for (int i = 0; i < ls.length; i++) { + Assert.assertTrue(ls[i].isDirectory()); + Assert.assertEquals(true, ls[i].getPermission().getAclBit()); + } + + // With ACLBIT set to false + ls = null; + getMockServer().enqueue(new MockResponse().setResponseCode(200) + .setBody(TestADLResponseData.getListFileStatusJSONResponse(false))); + startTime = Time.monotonicNow(); + ls = getMockAdlFileSystem() + .listStatus(new Path("/test1/test2")); + endTime = Time.monotonicNow(); + LOG.debug("Time : " + (endTime - startTime)); + for (int i = 0; i < ls.length; i++) { + Assert.assertTrue(ls[i].isDirectory()); + Assert.assertEquals(false, ls[i].getPermission().getAclBit()); + } + } }