From 038f32f74e94ba3f8d00022728b51ae5bd9b7b9a Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Fri, 25 Apr 2014 05:56:51 +0000 Subject: [PATCH] HDFS-6210. Merge r1589933 from trunk. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1589934 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../offlineImageViewer/FSImageHandler.java | 2 + .../offlineImageViewer/FSImageLoader.java | 76 +++++++ .../TestOfflineImageViewerForAcl.java | 206 ++++++++++++++++++ 4 files changed, 287 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 04be9e47cf3..3f1b1c514d4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -74,6 +74,9 @@ Release 2.5.0 - UNRELEASED HDFS-6266. Identify full path for a given INode. (jing9) + HDFS-6210. Support GETACLSTATUS operation in WebImageViewer. + (Akira Ajisaka via wheat9) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java index 9e73003c012..b6d760aab9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageHandler.java @@ -104,6 +104,8 @@ private void handleOperation(String op, String path, MessageEvent e) content = loader.getFileStatus(path); } else if (op.equals("LISTSTATUS")) { content = loader.listStatus(path); + } else if (op.equals("GETACLSTATUS")) { + content = loader.getAclStatus(path); } else { response.setStatus(HttpResponseStatus.BAD_REQUEST); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java index 4b03b4f2baa..70c891258a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java @@ -31,6 +31,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; @@ -270,6 +271,81 @@ private List> getFileStatusList(String path) { return list; } + /** + * Return the JSON formatted ACL status of the specified file. + * @param path a path specifies a file + * @return JSON formatted AclStatus + * @throws IOException if failed to serialize fileStatus to JSON. + */ + String getAclStatus(String path) throws IOException { + StringBuilder sb = new StringBuilder(); + List aclEntryList = getAclEntryList(path); + PermissionStatus p = getPermissionStatus(path); + sb.append("{\"AclStatus\":{\"entries\":["); + int i = 0; + for (AclEntry aclEntry : aclEntryList) { + if (i++ != 0) { + sb.append(','); + } + sb.append('"'); + sb.append(aclEntry.toString()); + sb.append('"'); + } + sb.append("],\"group\": \""); + sb.append(p.getGroupName()); + sb.append("\",\"owner\": \""); + sb.append(p.getUserName()); + sb.append("\",\"stickyBit\": "); + sb.append(p.getPermission().getStickyBit()); + sb.append("}}\n"); + return sb.toString(); + } + + private List getAclEntryList(String path) { + long id = getINodeId(path); + FsImageProto.INodeSection.INode inode = inodes.get(id); + switch (inode.getType()) { + case FILE: { + FsImageProto.INodeSection.INodeFile f = inode.getFile(); + return FSImageFormatPBINode.Loader.loadAclEntries( + f.getAcl(), stringTable); + } + case DIRECTORY: { + FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory(); + return FSImageFormatPBINode.Loader.loadAclEntries( + d.getAcl(), stringTable); + } + default: { + return new ArrayList(); + } + } + } + + private PermissionStatus getPermissionStatus(String path) { + long id = getINodeId(path); + FsImageProto.INodeSection.INode inode = inodes.get(id); + switch (inode.getType()) { + case FILE: { + FsImageProto.INodeSection.INodeFile f = inode.getFile(); + return FSImageFormatPBINode.Loader.loadPermission( + f.getPermission(), stringTable); + } + case DIRECTORY: { + FsImageProto.INodeSection.INodeDirectory d = inode.getDirectory(); + return FSImageFormatPBINode.Loader.loadPermission( + d.getPermission(), stringTable); + } + case SYMLINK: { + FsImageProto.INodeSection.INodeSymlink s = inode.getSymlink(); + return FSImageFormatPBINode.Loader.loadPermission( + s.getPermission(), stringTable); + } + default: { + return null; + } + } + } + /** * Return the INodeId of the specified path. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java new file mode 100644 index 00000000000..7b4804ab8eb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForAcl.java @@ -0,0 +1,206 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.tools.offlineImageViewer; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.HashMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.AclStatus; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; +import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; +import org.apache.hadoop.net.NetUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryScope.DEFAULT; +import static org.apache.hadoop.fs.permission.AclEntryType.GROUP; +import static org.apache.hadoop.fs.permission.AclEntryType.OTHER; +import static org.apache.hadoop.fs.permission.AclEntryType.USER; +import static org.apache.hadoop.fs.permission.FsAction.ALL; +import static org.apache.hadoop.fs.permission.FsAction.READ; +import static org.apache.hadoop.fs.permission.FsAction.READ_WRITE; +import static org.apache.hadoop.fs.permission.FsAction.READ_EXECUTE; +import static org.apache.hadoop.fs.permission.FsAction.NONE; +import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +/** + * Tests OfflineImageViewer if the input fsimage has HDFS ACLs + */ +public class TestOfflineImageViewerForAcl { + + private static final Log LOG = + LogFactory.getLog(TestOfflineImageViewerForAcl.class); + + private static File originalFsimage = null; + + // ACLs as set to dfs, to be compared with viewer's output + final static HashMap writtenAcls = Maps.newHashMap(); + + /** + * Create a populated namespace for later testing. Save its contents to a + * data structure and store its fsimage location. + * We only want to generate the fsimage file once and use it for + * multiple tests. + */ + @BeforeClass + public static void createOriginalFSImage() throws IOException { + MiniDFSCluster cluster = null; + try { + Configuration conf = new Configuration(); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); + cluster = new MiniDFSCluster.Builder(conf).build(); + cluster.waitActive(); + DistributedFileSystem hdfs = cluster.getFileSystem(); + + // Create a reasonable namespace with ACLs + Path dir = new Path("/dirWithNoAcl"); + hdfs.mkdirs(dir); + writtenAcls.put(dir.toString(), hdfs.getAclStatus(dir)); + + dir = new Path("/dirWithDefaultAcl"); + hdfs.mkdirs(dir); + hdfs.setAcl(dir, Lists.newArrayList( + aclEntry(DEFAULT, USER, ALL), + aclEntry(DEFAULT, USER, "foo", ALL), + aclEntry(DEFAULT, GROUP, READ_EXECUTE), + aclEntry(DEFAULT, OTHER, NONE))); + writtenAcls.put(dir.toString(), hdfs.getAclStatus(dir)); + + Path file = new Path("/noAcl"); + FSDataOutputStream o = hdfs.create(file); + o.write(23); + o.close(); + writtenAcls.put(file.toString(), hdfs.getAclStatus(file)); + + file = new Path("/withAcl"); + o = hdfs.create(file); + o.write(23); + o.close(); + hdfs.setAcl(file, Lists.newArrayList( + aclEntry(ACCESS, USER, READ_WRITE), + aclEntry(ACCESS, USER, "foo", READ), + aclEntry(ACCESS, GROUP, READ), + aclEntry(ACCESS, OTHER, NONE))); + writtenAcls.put(file.toString(), hdfs.getAclStatus(file)); + + file = new Path("/withSeveralAcls"); + o = hdfs.create(file); + o.write(23); + o.close(); + hdfs.setAcl(file, Lists.newArrayList( + aclEntry(ACCESS, USER, READ_WRITE), + aclEntry(ACCESS, USER, "foo", READ_WRITE), + aclEntry(ACCESS, USER, "bar", READ), + aclEntry(ACCESS, GROUP, READ), + aclEntry(ACCESS, GROUP, "group", READ), + aclEntry(ACCESS, OTHER, NONE))); + writtenAcls.put(file.toString(), hdfs.getAclStatus(file)); + + // Write results to the fsimage file + hdfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER, false); + hdfs.saveNamespace(); + + // Determine the location of the fsimage file + originalFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil + .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0)); + if (originalFsimage == null) { + throw new RuntimeException("Didn't generate or can't find fsimage"); + } + LOG.debug("original FS image file is " + originalFsimage); + } finally { + if (cluster != null) + cluster.shutdown(); + } + } + + @AfterClass + public static void deleteOriginalFSImage() throws IOException { + if (originalFsimage != null && originalFsimage.exists()) { + originalFsimage.delete(); + } + } + + @Test + public void testWebImageViewerForAcl() throws IOException, + InterruptedException, URISyntaxException { + WebImageViewer viewer = new WebImageViewer( + NetUtils.createSocketAddr("localhost:0")); + try { + viewer.initServer(originalFsimage.getAbsolutePath()); + int port = viewer.getPort(); + + // create a WebHdfsFileSystem instance + URI uri = new URI("webhdfs://localhost:" + String.valueOf(port)); + Configuration conf = new Configuration(); + WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)FileSystem.get(uri, conf); + + // GETACLSTATUS operation to a directory without ACL + AclStatus acl = webhdfs.getAclStatus(new Path("/dirWithNoAcl")); + assertEquals(writtenAcls.get("/dirWithNoAcl"), acl); + + // GETACLSTATUS operation to a directory with a default ACL + acl = webhdfs.getAclStatus(new Path("/dirWithDefaultAcl")); + assertEquals(writtenAcls.get("/dirWithDefaultAcl"), acl); + + // GETACLSTATUS operation to a file without ACL + acl = webhdfs.getAclStatus(new Path("/noAcl")); + assertEquals(writtenAcls.get("/noAcl"), acl); + + // GETACLSTATUS operation to a file with a ACL + acl = webhdfs.getAclStatus(new Path("/withAcl")); + assertEquals(writtenAcls.get("/withAcl"), acl); + + // GETACLSTATUS operation to a file with several ACL entries + acl = webhdfs.getAclStatus(new Path("/withSeveralAcls")); + assertEquals(writtenAcls.get("/withSeveralAcls"), acl); + + // GETACLSTATUS operation to a invalid path + URL url = new URL("http://localhost:" + port + + "/webhdfs/v1/invalid/?op=GETACLSTATUS"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.connect(); + assertEquals(HttpURLConnection.HTTP_NOT_FOUND, + connection.getResponseCode()); + } finally { + // shutdown the viewer + viewer.shutdown(); + } + } +}