From 9ca2f34c8fb03227e7364ced5183562e9f805400 Mon Sep 17 00:00:00 2001 From: Charles Lamb Date: Wed, 30 Jul 2014 01:35:04 +0000 Subject: [PATCH] HDFS-6730. Create a .RAW extended attribute namespace. (clamb) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/fs-encryption@1614535 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-hdfs/CHANGES-fs-encryption.txt | 2 + .../main/java/org/apache/hadoop/fs/XAttr.java | 13 +- .../org/apache/hadoop/hdfs/XAttrHelper.java | 8 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 6 +- .../server/common/HdfsServerConstants.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 14 +- .../namenode/XAttrPermissionFilter.java | 30 ++- .../hadoop-hdfs/src/main/proto/xattr.proto | 1 + .../src/site/apt/ExtendedAttributes.apt.vm | 4 +- .../java/org/apache/hadoop/fs/TestXAttr.java | 11 +- .../hdfs/server/namenode/FSXAttrBaseTest.java | 179 +++++++++++++++++- .../hdfs/server/namenode/TestFSDirectory.java | 15 +- .../src/test/resources/testXAttrConf.xml | 58 +++++- 13 files changed, 313 insertions(+), 32 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt index 2531a4e4049..34a86e39db7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-fs-encryption.txt @@ -68,6 +68,8 @@ fs-encryption (Unreleased) HDFS-6771. Require specification of an encryption key when creating an encryption zone. (wang) + HDFS-6730. Create a .RAW extended attribute namespace. (clamb) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java index 99f629afdfe..968ee00ce76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java @@ -26,8 +26,8 @@ import org.apache.hadoop.classification.InterfaceAudience; /** * XAttr is the POSIX Extended Attribute model similar to that found in * traditional Operating Systems. Extended Attributes consist of one - * or more name/value pairs associated with a file or directory. Four - * namespaces are defined: user, trusted, security and system. + * or more name/value pairs associated with a file or directory. Five + * namespaces are defined: user, trusted, security, system and raw. * 1) USER namespace attributes may be used by any user to store * arbitrary information. Access permissions in this namespace are * defined by a file directory's permission bits. For sticky directories, @@ -43,6 +43,12 @@ import org.apache.hadoop.classification.InterfaceAudience; *
* 4) SECURITY namespace attributes are used by the fs kernel for * security features. It is not visible to users. + *
+ * 5) RAW namespace attributes are used for internal system attributes that + * sometimes need to be exposed. Like SYSTEM namespace attributes they are + * not visible to the user except when getXAttr/getXAttrs is called on a file + * or directory in the /.reserved/raw HDFS directory hierarchy. These + * attributes can only be accessed by the superuser. *

* @see * http://en.wikipedia.org/wiki/Extended_file_attributes @@ -55,7 +61,8 @@ public class XAttr { USER, TRUSTED, SECURITY, - SYSTEM; + SYSTEM, + RAW; } private final NameSpace ns; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java index abcd47ae16a..04364ccf7e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/XAttrHelper.java @@ -49,9 +49,9 @@ public class XAttrHelper { Preconditions.checkNotNull(name, "XAttr name cannot be null."); final int prefixIndex = name.indexOf("."); - if (prefixIndex < 4) {// Prefix length is at least 4. + if (prefixIndex < 3) {// Prefix length is at least 3. throw new HadoopIllegalArgumentException("An XAttr name must be " + - "prefixed with user/trusted/security/system, followed by a '.'"); + "prefixed with user/trusted/security/system/raw, followed by a '.'"); } else if (prefixIndex == name.length() - 1) { throw new HadoopIllegalArgumentException("XAttr name cannot be empty."); } @@ -66,9 +66,11 @@ public class XAttrHelper { ns = NameSpace.SYSTEM; } else if (prefix.equals(NameSpace.SECURITY.toString().toLowerCase())) { ns = NameSpace.SECURITY; + } else if (prefix.equals(NameSpace.RAW.toString().toLowerCase())) { + ns = NameSpace.RAW; } else { throw new HadoopIllegalArgumentException("An XAttr name must be " + - "prefixed with user/trusted/security/system, followed by a '.'"); + "prefixed with user/trusted/security/system/raw, followed by a '.'"); } XAttr xAttr = (new XAttr.Builder()).setNameSpace(ns).setName(name. substring(prefixIndex + 1)).setValue(value).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 734240f06fa..6ff4ffbbaa1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1335,7 +1335,6 @@ public interface ClientProtocol { * @see * http://en.wikipedia.org/wiki/Extended_file_attributes * @param src file or directory - * @param xAttrs xAttrs to get * @return List XAttr list * @throws IOException */ @@ -1345,12 +1344,15 @@ public interface ClientProtocol { /** * Remove xattr of a file or directory.Value in xAttr parameter is ignored. - * Name must be prefixed with user/trusted/security/system. + * Name must be prefixed with user/trusted/security/system/raw. *

* A regular user only can remove xattr of "user" namespace. * A super user can remove xattr of "user" and "trusted" namespace. * XAttr of "security" and "system" namespace is only used/exposed * internally to the FS impl. + * The xattrs of the "raw" namespace are only used/exposed when accessed in + * the /.reserved/raw HDFS directory hierarchy. These attributes can only be + * accessed by the superuser. *

* @see * http://en.wikipedia.org/wiki/Extended_file_attributes diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java index 5e4d1f0e5ba..98c6398c2cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java @@ -296,8 +296,8 @@ public final class HdfsServerConstants { public static final long NAMENODE_LEASE_RECHECK_INTERVAL = 2000; public static final String CRYPTO_XATTR_ENCRYPTION_ZONE = - "system.hdfs.crypto.encryption.zone"; + "raw.hdfs.crypto.encryption.zone"; public static final String CRYPTO_XATTR_FILE_ENCRYPTION_INFO = - "system.hdfs.crypto.file.encryption.info"; + "raw.hdfs.crypto.file.encryption.info"; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index dde19b9bd21..8350225d8ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -8594,7 +8594,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, checkXAttrSize(xAttr); HdfsFileStatus resultingStat = null; FSPermissionChecker pc = getPermissionChecker(); - XAttrPermissionFilter.checkPermissionForApi(pc, xAttr); + XAttrPermissionFilter.checkPermissionForApi(pc, xAttr, + FSDirectory.isReservedRawName(src)); checkOperation(OperationCategory.WRITE); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); writeLock(); @@ -8640,10 +8641,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, String src = srcArg; nnConf.checkXAttrsConfigFlag(); FSPermissionChecker pc = getPermissionChecker(); + final boolean isRawPath = FSDirectory.isReservedRawName(src); boolean getAll = xAttrs == null || xAttrs.isEmpty(); if (!getAll) { try { - XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs); + XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs, isRawPath); } catch (AccessControlException e) { logAuditEvent(false, "getXAttrs", srcArg); throw e; @@ -8660,7 +8662,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } List all = dir.getXAttrs(src); List filteredAll = XAttrPermissionFilter. - filterXAttrsForApi(pc, all); + filterXAttrsForApi(pc, all, isRawPath); if (getAll) { return filteredAll; } else { @@ -8696,6 +8698,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, List listXAttrs(String src) throws IOException { nnConf.checkXAttrsConfigFlag(); final FSPermissionChecker pc = getPermissionChecker(); + final boolean isRawPath = FSDirectory.isReservedRawName(src); checkOperation(OperationCategory.READ); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); @@ -8708,7 +8711,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } final List all = dir.getXAttrs(src); final List filteredAll = XAttrPermissionFilter. - filterXAttrsForApi(pc, all); + filterXAttrsForApi(pc, all, isRawPath); return filteredAll; } catch (AccessControlException e) { logAuditEvent(false, "listXAttrs", src); @@ -8753,7 +8756,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, nnConf.checkXAttrsConfigFlag(); HdfsFileStatus resultingStat = null; FSPermissionChecker pc = getPermissionChecker(); - XAttrPermissionFilter.checkPermissionForApi(pc, xAttr); + XAttrPermissionFilter.checkPermissionForApi(pc, xAttr, + FSDirectory.isReservedRawName(src)); checkOperation(OperationCategory.WRITE); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); writeLock(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java index 98730142fbd..237f9d3d5ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java @@ -47,15 +47,27 @@ import com.google.common.base.Preconditions; *
* SYSTEM - extended system attributes: these are used by the HDFS * core and are not available through admin/user API. + *
+ * RAW - extended system attributes: these are used for internal system + * attributes that sometimes need to be exposed. Like SYSTEM namespace + * attributes they are not visible to the user except when getXAttr/getXAttrs + * is called on a file or directory in the /.reserved/raw HDFS directory + * hierarchy. These attributes can only be accessed by the superuser. + *
*/ @InterfaceAudience.Private public class XAttrPermissionFilter { - static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr) + static void checkPermissionForApi(FSPermissionChecker pc, XAttr xAttr, + boolean isRawPath) throws AccessControlException { + final boolean isSuperUser = pc.isSuperUser(); if (xAttr.getNameSpace() == XAttr.NameSpace.USER || - (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && - pc.isSuperUser())) { + (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && isSuperUser)) { + return; + } + if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && + isRawPath && isSuperUser) { return; } throw new AccessControlException("User doesn't have permission for xattr: " @@ -63,30 +75,34 @@ public class XAttrPermissionFilter { } static void checkPermissionForApi(FSPermissionChecker pc, - List xAttrs) throws AccessControlException { + List xAttrs, boolean isRawPath) throws AccessControlException { Preconditions.checkArgument(xAttrs != null); if (xAttrs.isEmpty()) { return; } for (XAttr xAttr : xAttrs) { - checkPermissionForApi(pc, xAttr); + checkPermissionForApi(pc, xAttr, isRawPath); } } static List filterXAttrsForApi(FSPermissionChecker pc, - List xAttrs) { + List xAttrs, boolean isRawPath) { assert xAttrs != null : "xAttrs can not be null"; if (xAttrs == null || xAttrs.isEmpty()) { return xAttrs; } List filteredXAttrs = Lists.newArrayListWithCapacity(xAttrs.size()); + final boolean isSuperUser = pc.isSuperUser(); for (XAttr xAttr : xAttrs) { if (xAttr.getNameSpace() == XAttr.NameSpace.USER) { filteredXAttrs.add(xAttr); } else if (xAttr.getNameSpace() == XAttr.NameSpace.TRUSTED && - pc.isSuperUser()) { + isSuperUser) { + filteredXAttrs.add(xAttr); + } else if (xAttr.getNameSpace() == XAttr.NameSpace.RAW && + isSuperUser && isRawPath) { filteredXAttrs.add(xAttr); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto index cb86ff27731..acdc28ebb88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto @@ -27,6 +27,7 @@ message XAttrProto { TRUSTED = 1; SECURITY = 2; SYSTEM = 3; + RAW = 4; } required XAttrNamespaceProto namespace = 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm index 56aec0ca288..0a99fe50ee6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ExtendedAttributes.apt.vm @@ -30,7 +30,7 @@ Extended Attributes in HDFS ** {Namespaces and Permissions} - In HDFS, as in Linux, there are four valid namespaces: <<>>, <<>>, <<>>, and <<>>. Each of these namespaces have different access restrictions. + In HDFS, there are five valid namespaces: <<>>, <<>>, <<>>, <<>>, and <<>>. Each of these namespaces have different access restrictions. The <<>> namespace is the namespace that will commonly be used by client applications. Access to extended attributes in the user namespace is controlled by the corresponding file permissions. @@ -40,6 +40,8 @@ Extended Attributes in HDFS The <<>> namespace is reserved for internal HDFS use. This namespace is not accessible through userspace methods. It is currently unused. + The <<>> namespace is reserved for internal system attributes that sometimes need to be exposed. Like <<>> namespace attributes they are not visible to the user except when <<>>/<<>> is called on a file or directory in the <<>> HDFS directory hierarchy. These attributes can only be accessed by the superuser. An example of where <<>> namespace extended attributes are used is the <<>> utility. Encryption zone meta data is stored in <<>> extended attributes, so as long as the administrator uses <<>> pathnames in source and target, the encrypted files in the encryption zones are transparently copied. + * {Interacting with extended attributes} The Hadoop shell has support for interacting with extended attributes via <<>> and <<>>. These commands are styled after the Linux {{{http://www.bestbits.at/acl/man/man1/getfattr.txt}getfattr(1)}} and {{{http://www.bestbits.at/acl/man/man1/setfattr.txt}setfattr(1)}} commands. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java index 032a8dfead0..e47658dd6a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestXAttr.java @@ -29,7 +29,7 @@ import org.junit.Test; * Tests for XAttr objects. */ public class TestXAttr { - private static XAttr XATTR, XATTR1, XATTR2, XATTR3, XATTR4; + private static XAttr XATTR, XATTR1, XATTR2, XATTR3, XATTR4, XATTR5; @BeforeClass public static void setUp() throws Exception { @@ -58,6 +58,11 @@ public class TestXAttr { .setName("name") .setValue(value) .build(); + XATTR5 = new XAttr.Builder() + .setNameSpace(XAttr.NameSpace.RAW) + .setName("name") + .setValue(value) + .build(); } @Test @@ -65,14 +70,17 @@ public class TestXAttr { assertNotSame(XATTR1, XATTR2); assertNotSame(XATTR2, XATTR3); assertNotSame(XATTR3, XATTR4); + assertNotSame(XATTR4, XATTR5); assertEquals(XATTR, XATTR1); assertEquals(XATTR1, XATTR1); assertEquals(XATTR2, XATTR2); assertEquals(XATTR3, XATTR3); assertEquals(XATTR4, XATTR4); + assertEquals(XATTR5, XATTR5); assertFalse(XATTR1.equals(XATTR2)); assertFalse(XATTR2.equals(XATTR3)); assertFalse(XATTR3.equals(XATTR4)); + assertFalse(XATTR4.equals(XATTR5)); } @Test @@ -81,5 +89,6 @@ public class TestXAttr { assertFalse(XATTR1.hashCode() == XATTR2.hashCode()); assertFalse(XATTR2.hashCode() == XATTR3.hashCode()); assertFalse(XATTR3.hashCode() == XATTR4.hashCode()); + assertFalse(XATTR4.hashCode() == XATTR5.hashCode()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java index 636ecc2417f..0c7b8070b44 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java @@ -69,6 +69,7 @@ public class FSXAttrBaseTest { protected static Configuration conf; private static int pathCount = 0; protected static Path path; + protected static Path rawPath; // XAttrs protected static final String name1 = "user.a1"; @@ -78,6 +79,8 @@ public class FSXAttrBaseTest { protected static final byte[] value2 = {0x37, 0x38, 0x39}; protected static final String name3 = "user.a3"; protected static final String name4 = "user.a4"; + protected static final String raw1 = "raw.a1"; + protected static final String raw2 = "raw.a2"; protected FileSystem fs; @@ -107,6 +110,7 @@ public class FSXAttrBaseTest { public void setUp() throws Exception { pathCount += 1; path = new Path("/p" + pathCount); + rawPath = new Path("/.reserved/raw/p" + pathCount); initFileSystem(); } @@ -395,7 +399,8 @@ public class FSXAttrBaseTest { Assert.fail("expected IOException"); } catch (Exception e) { GenericTestUtils.assertExceptionContains - ("An XAttr name must be prefixed with user/trusted/security/system, " + + ("An XAttr name must be prefixed with " + + "user/trusted/security/system/raw, " + "followed by a '.'", e); } @@ -582,7 +587,7 @@ public class FSXAttrBaseTest { /* Unknown namespace should throw an exception. */ final String expectedExceptionString = "An XAttr name must be prefixed " + - "with user/trusted/security/system, followed by a '.'"; + "with user/trusted/security/system/raw, followed by a '.'"; try { fs.removeXAttr(path, "wackynamespace.foo"); Assert.fail("expected IOException"); @@ -918,6 +923,176 @@ public class FSXAttrBaseTest { fsAsDiana.removeXAttr(path, name2); } + @Test(timeout = 120000) + public void testRawXAttrs() throws Exception { + final UserGroupInformation user = UserGroupInformation. + createUserForTesting("user", new String[] {"mygroup"}); + + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 0750)); + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE, + XAttrSetFlag.REPLACE)); + + { + // getXAttr + final byte[] value = fs.getXAttr(rawPath, raw1); + Assert.assertArrayEquals(value, value1); + } + + { + // getXAttrs + final Map xattrs = fs.getXAttrs(rawPath); + Assert.assertEquals(xattrs.size(), 1); + Assert.assertArrayEquals(value1, xattrs.get(raw1)); + fs.removeXAttr(rawPath, raw1); + } + + { + // replace and re-get + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(rawPath, raw1, newValue1, EnumSet.of(XAttrSetFlag.CREATE, + XAttrSetFlag.REPLACE)); + + final Map xattrs = fs.getXAttrs(rawPath); + Assert.assertEquals(xattrs.size(), 1); + Assert.assertArrayEquals(newValue1, xattrs.get(raw1)); + + fs.removeXAttr(rawPath, raw1); + } + + { + // listXAttrs on rawPath ensuring raw.* xattrs are returned + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(rawPath, raw2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + + final List xattrNames = fs.listXAttrs(rawPath); + assertTrue(xattrNames.contains(raw1)); + assertTrue(xattrNames.contains(raw2)); + assertTrue(xattrNames.size() == 2); + fs.removeXAttr(rawPath, raw1); + fs.removeXAttr(rawPath, raw2); + } + + { + // listXAttrs on non-rawPath ensuring no raw.* xattrs returned + fs.setXAttr(rawPath, raw1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(rawPath, raw2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + + final List xattrNames = fs.listXAttrs(path); + assertTrue(xattrNames.size() == 0); + fs.removeXAttr(rawPath, raw1); + fs.removeXAttr(rawPath, raw2); + } + + { + /* + * Test non-root user operations in the "raw.*" namespace. + */ + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + // Test that non-root can not set xattrs in the "raw.*" namespace + try { + // non-raw path + userFs.setXAttr(path, raw1, value1); + fail("setXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.setXAttr(rawPath, raw1, value1); + fail("setXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + // Test that non-root can not do getXAttrs in the "raw.*" namespace + try { + // non-raw path + userFs.getXAttrs(rawPath); + fail("getXAttrs should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.getXAttrs(path); + fail("getXAttrs should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + // Test that non-root can not do getXAttr in the "raw.*" namespace + try { + // non-raw path + userFs.getXAttr(rawPath, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.getXAttr(path, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + return null; + } + }); + } + + { + /* + * Test that non-root can not do getXAttr in the "raw.*" namespace + */ + fs.setXAttr(rawPath, raw1, value1); + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + try { + // non-raw path + userFs.getXAttr(rawPath, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + try { + // raw path + userFs.getXAttr(path, raw1); + fail("getXAttr should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + /* + * Test that only root can see raw.* xattrs returned from listXAttr + * and non-root can't do listXAttrs on /.reserved/raw. + */ + // non-raw path + final List xattrNames = userFs.listXAttrs(path); + assertTrue(xattrNames.size() == 0); + try { + // raw path + userFs.listXAttrs(rawPath); + fail("listXAttrs on raw path should have thrown"); + } catch (AccessControlException e) { + // ignore + } + + return null; + } + }); + fs.removeXAttr(rawPath, raw1); + } + } + /** * Creates a FileSystem for the super-user. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java index 011901ddfa7..ad067cfa197 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java @@ -191,14 +191,19 @@ public class TestFSDirectory { existingXAttrs.add(xAttr1); existingXAttrs.add(xAttr2); - // Adding a system namespace xAttr, isn't affected by inode xAttrs limit. - XAttr newXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.SYSTEM). + // Adding system and raw namespace xAttrs aren't affected by inode + // xAttrs limit. + XAttr newSystemXAttr = (new XAttr.Builder()). + setNameSpace(XAttr.NameSpace.SYSTEM).setName("a3"). + setValue(new byte[]{0x33, 0x33, 0x33}).build(); + XAttr newRawXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.RAW). setName("a3").setValue(new byte[]{0x33, 0x33, 0x33}).build(); - List newXAttrs = Lists.newArrayListWithCapacity(1); - newXAttrs.add(newXAttr); + List newXAttrs = Lists.newArrayListWithCapacity(2); + newXAttrs.add(newSystemXAttr); + newXAttrs.add(newRawXAttr); List xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); - assertEquals(xAttrs.size(), 3); + assertEquals(xAttrs.size(), 4); // Adding a trusted namespace xAttr, is affected by inode xAttrs limit. XAttr newXAttr1 = (new XAttr.Builder()).setNameSpace( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml index 7b7f866ac38..3414f5719dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testXAttrConf.xml @@ -64,7 +64,7 @@ SubstringComparator - name must be prefixed with user/trusted/security/system, followed by a '.' + name must be prefixed with user/trusted/security/system/raw, followed by a '.' @@ -125,6 +125,42 @@ + + setfattr : Add an xattr of raw namespace + + -fs NAMENODE -touchz /file1 + -fs NAMENODE -setfattr -n raw.a1 -v 123456 /file1 + + + -fs NAMENODE -rm /file1 + + + + SubstringComparator + setfattr: User doesn't have permission for xattr: raw.a1 + + + + + + + setfattr : Add an xattr of raw namespace + + -fs NAMENODE -touchz /file1 + -fs NAMENODE -setfattr -n raw.a1 -v 123456 /.reserved/raw/file1 + -fs NAMENODE -getfattr -n raw.a1 /.reserved/raw/file1 + + + -fs NAMENODE -rm /file1 + + + + SubstringComparator + raw.a1="123456" + + + + setfattr : Add an xattr, and encode is text @@ -256,6 +292,26 @@ + + + setfattr : Remove an xattr of raw namespace + + -fs NAMENODE -touchz /file1 + -fs NAMENODE -setfattr -n raw.a1 -v 123456 /.reserved/raw/file1 + -fs NAMENODE -setfattr -n raw.a2 -v 123456 /.reserved/raw/file1 + -fs NAMENODE -setfattr -x raw.a2 /.reserved/raw/file1 + -fs NAMENODE -getfattr -d /.reserved/raw/file1 + + + -fs NAMENODE -rm /file1 + + + + SubstringComparator + # file: /.reserved/raw/file1#LF#raw.a1="123456"#LF# + + + getfattr : Get an xattr