diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java index a85e8d7191c..c5ee933ea19 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java @@ -19,15 +19,22 @@ package org.apache.hadoop.fs.shell; import java.io.IOException; +import java.net.URI; import java.text.SimpleDateFormat; import java.util.Date; import java.util.LinkedList; +import java.util.Set; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.RpcNoSuchMethodException; + +import com.google.common.collect.Sets; /** * Get a listing of all files in that match the file patterns. @@ -66,6 +73,8 @@ class Ls extends FsCommand { protected boolean dirRecurse; protected boolean humanReadable = false; + private Set aclNotSupportedFsSet = Sets.newHashSet(); + protected String formatSize(long size) { return humanReadable ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1) @@ -108,7 +117,7 @@ class Ls extends FsCommand { FileStatus stat = item.stat; String line = String.format(lineFormat, (stat.isDirectory() ? "d" : "-"), - stat.getPermission(), + stat.getPermission() + (hasAcl(item) ? "+" : ""), (stat.isFile() ? stat.getReplication() : "-"), stat.getOwner(), stat.getGroup(), @@ -146,6 +155,49 @@ class Ls extends FsCommand { lineFormat = fmt.toString(); } + /** + * Calls getAclStatus to determine if the given item has an ACL. For + * compatibility, this method traps errors caused by the RPC method missing + * from the server side. This would happen if the client was connected to an + * old NameNode that didn't have the ACL APIs. This method also traps the + * case of the client-side FileSystem not implementing the ACL APIs. + * FileSystem instances that do not support ACLs are remembered. This + * prevents the client from sending multiple failing RPC calls during a + * recursive ls. + * + * @param item PathData item to check + * @return boolean true if item has an ACL + * @throws IOException if there is a failure + */ + private boolean hasAcl(PathData item) throws IOException { + FileSystem fs = item.fs; + if (aclNotSupportedFsSet.contains(fs.getUri())) { + // This FileSystem failed to run the ACL API in an earlier iteration. + return false; + } + try { + return !fs.getAclStatus(item.path).getEntries().isEmpty(); + } catch (RemoteException e) { + // If this is a RpcNoSuchMethodException, then the client is connected to + // an older NameNode that doesn't support ACLs. Keep going. + IOException e2 = e.unwrapRemoteException(RpcNoSuchMethodException.class); + if (!(e2 instanceof RpcNoSuchMethodException)) { + throw e; + } + } catch (IOException e) { + // The NameNode supports ACLs, but they are not enabled. Keep going. + String message = e.getMessage(); + if (message != null && !message.contains("ACLs has been disabled")) { + throw e; + } + } catch (UnsupportedOperationException e) { + // The underlying FileSystem doesn't implement ACLs. Keep going. + } + // Remember that this FileSystem cannot support ACLs. + aclNotSupportedFsSet.add(fs.getUri()); + return false; + } + private int maxLength(int n, Object value) { return Math.max(n, (value != null) ? String.valueOf(value).length() : 0); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestAclCommands.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestAclCommands.java index 7382bfff208..34d9a9613a2 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestAclCommands.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestAclCommands.java @@ -20,15 +20,27 @@ package org.apache.hadoop.fs.shell; import static org.junit.Assert.*; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsShell; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; +import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.hadoop.ipc.RpcNoSuchMethodException; +import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ToolRunner; import org.junit.Before; import org.junit.Test; @@ -128,6 +140,97 @@ public class TestAclCommands { assertEquals("Parsed Acl not correct", expectedList, parsedList); } + @Test + public void testLsNoRpcForGetAclStatus() throws Exception { + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "stubfs:///"); + conf.setClass("fs.stubfs.impl", StubFileSystem.class, FileSystem.class); + conf.setBoolean("stubfs.noRpcForGetAclStatus", true); + assertEquals("ls must succeed even if getAclStatus RPC does not exist.", + 0, ToolRunner.run(conf, new FsShell(), new String[] { "-ls", "/" })); + } + + @Test + public void testLsAclsUnsupported() throws Exception { + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "stubfs:///"); + conf.setClass("fs.stubfs.impl", StubFileSystem.class, FileSystem.class); + assertEquals("ls must succeed even if FileSystem does not implement ACLs.", + 0, ToolRunner.run(conf, new FsShell(), new String[] { "-ls", "/" })); + } + + public static class StubFileSystem extends FileSystem { + + public FSDataOutputStream append(Path f, int bufferSize, + Progressable progress) throws IOException { + return null; + } + + public FSDataOutputStream create(Path f, FsPermission permission, + boolean overwrite, int bufferSize, short replication, long blockSize, + Progressable progress) throws IOException { + return null; + } + + @Override + public boolean delete(Path f, boolean recursive) throws IOException { + return false; + } + + public AclStatus getAclStatus(Path path) throws IOException { + if (getConf().getBoolean("stubfs.noRpcForGetAclStatus", false)) { + throw new RemoteException(RpcNoSuchMethodException.class.getName(), + "test exception"); + } + return super.getAclStatus(path); + } + + @Override + public FileStatus getFileStatus(Path f) throws IOException { + return null; + } + + @Override + public URI getUri() { + return URI.create("stubfs:///"); + } + + @Override + public Path getWorkingDirectory() { + return null; + } + + @Override + public FileStatus[] listStatus(Path f) throws IOException { + FsPermission perm = new FsPermission(FsAction.ALL, FsAction.READ_EXECUTE, + FsAction.READ_EXECUTE); + Path path = new Path("/foo"); + FileStatus stat = new FileStatus(1000, true, 3, 1000, 0, 0, perm, "owner", + "group", path); + return new FileStatus[] { stat }; + } + + @Override + public boolean mkdirs(Path f, FsPermission permission) + throws IOException { + return false; + } + + @Override + public FSDataInputStream open(Path f, int bufferSize) throws IOException { + return null; + } + + @Override + public boolean rename(Path src, Path dst) throws IOException { + return false; + } + + @Override + public void setWorkingDirectory(Path dir) { + } + } + private int runCommand(String[] commands) throws Exception { return ToolRunner.run(conf, new FsShell(), commands); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-4685.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-4685.txt index 20139bf690b..bbfc5d6275f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-4685.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-4685.txt @@ -80,6 +80,8 @@ HDFS-4685 (Unreleased) HDFS-5933. Optimize the FSImage layout for ACLs (Haohui Mai via cnauroth) + HDFS-5932. Ls should display the ACL bit (Chris Nauroth via wheat9) + OPTIMIZATIONS BUG FIXES