From fa08b92f46365980ac9405c544c69b8d4fafa0ef Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 23 Jul 2014 17:49:47 +0000 Subject: [PATCH 01/37] HADOOP-10872. TestPathData fails intermittently with 'Mkdirs failed to create d1'. (Contributed by Yongjun Zhang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612895 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/fs/ChecksumFileSystem.java | 4 +++- .../org/apache/hadoop/fs/shell/TestPathData.java | 12 ++++++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 18758515b4a..de285739202 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -805,6 +805,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10864. Tool documentenation is broken. (Akira Ajisaka via Arpit Agarwal) + HADOOP-10872. TestPathData fails intermittently with "Mkdirs failed + to create d1". (Yongjun Zhang via Arpit Agarwal) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index b67eadd94fc..511ca7f7549 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -437,7 +437,9 @@ public abstract class ChecksumFileSystem extends FilterFileSystem { throw new FileNotFoundException("Parent directory doesn't exist: " + parent); } else if (!mkdirs(parent)) { - throw new IOException("Mkdirs failed to create " + parent); + throw new IOException("Mkdirs failed to create " + parent + + " (exists=" + exists(parent) + ", cwd=" + getWorkingDirectory() + + ")"); } } final FSDataOutputStream out; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java index 3ddd68da23f..ea9e9847fd3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java @@ -35,19 +35,22 @@ import org.junit.Before; import org.junit.Test; public class TestPathData { + private static final String TEST_ROOT_DIR = + System.getProperty("test.build.data","build/test/data") + "/testPD"; protected Configuration conf; protected FileSystem fs; protected Path testDir; - + @Before public void initialize() throws Exception { conf = new Configuration(); fs = FileSystem.getLocal(conf); - testDir = new Path( - System.getProperty("test.build.data", "build/test/data") + "/testPD" - ); + testDir = new Path(TEST_ROOT_DIR); + // don't want scheme on the path, just an absolute path testDir = new Path(fs.makeQualified(testDir).toUri().getPath()); + fs.mkdirs(testDir); + FileSystem.setDefaultUri(conf, fs.getUri()); fs.setWorkingDirectory(testDir); fs.mkdirs(new Path("d1")); @@ -60,6 +63,7 @@ public class TestPathData { @After public void cleanup() throws Exception { + fs.delete(testDir, true); fs.close(); } From 61156a8ae3fd345c5ad0960091f117bfc607e04a Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Wed, 23 Jul 2014 18:12:55 +0000 Subject: [PATCH 02/37] HADOOP-10830. Missing lock in JavaKeyStoreProvider.createCredentialEntry. Contributed by Benoy Antony. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612904 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/security/alias/JavaKeyStoreProvider.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index de285739202..035afff58c1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -769,6 +769,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10801 dead link in site.xml (Akira AJISAKA via stevel) + HADOOP-10830. Missing lock in JavaKeyStoreProvider.createCredentialEntry. + (Benoy Antony via umamahesh) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java index 551c4ca14ff..e9269e66ae5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java @@ -194,15 +194,18 @@ public class JavaKeyStoreProvider extends CredentialProvider { @Override public CredentialEntry createCredentialEntry(String alias, char[] credential) throws IOException { + writeLock.lock(); try { if (keyStore.containsAlias(alias) || cache.containsKey(alias)) { throw new IOException("Credential " + alias + " already exists in " + this); } + return innerSetCredential(alias, credential); } catch (KeyStoreException e) { throw new IOException("Problem looking up credential " + alias + " in " + this, e); + } finally { + writeLock.unlock(); } - return innerSetCredential(alias, credential); } @Override From d15ea6c2db4dbe3cd4f5fff4cd296191f433c690 Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Wed, 23 Jul 2014 18:40:09 +0000 Subject: [PATCH 03/37] Moved CHANGES.txt entry for HADOOP-10830 to 2.6 section git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612912 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 035afff58c1..49134e12efc 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -480,6 +480,9 @@ Release 2.6.0 - UNRELEASED command when the format of the stat command uses non-curly quotes (yzhang via cmccabe) + HADOOP-10830. Missing lock in JavaKeyStoreProvider.createCredentialEntry. + (Benoy Antony via umamahesh) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -769,9 +772,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10801 dead link in site.xml (Akira AJISAKA via stevel) - HADOOP-10830. Missing lock in JavaKeyStoreProvider.createCredentialEntry. - (Benoy Antony via umamahesh) - BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS HADOOP-10520. Extended attributes definition and FileSystem APIs for From b9370fdcf601e6f01bcee171ac7e23cb936bd0ac Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 23 Jul 2014 18:53:38 +0000 Subject: [PATCH 04/37] HADOOP-10890. TestDFVariations.testMount fails intermittently. (Contributed by Yongjun Zhang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612916 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/fs/TestDFVariations.java | 21 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 49134e12efc..2463fb065da 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -811,6 +811,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10872. TestPathData fails intermittently with "Mkdirs failed to create d1". (Yongjun Zhang via Arpit Agarwal) + HADOOP-10890. TestDFVariations.testMount fails intermittently. (Yongjun + Zhang via Arpit Agarwal) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java index d457c0e8a46..97dbe5e6069 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java @@ -29,14 +29,33 @@ import java.util.Random; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; +import org.junit.After; +import org.junit.Before; import org.junit.Test; + import static org.junit.Assert.*; public class TestDFVariations { + private static final String TEST_ROOT_DIR = + System.getProperty("test.build.data","build/test/data") + "/TestDFVariations"; + private static File test_root = null; + @Before + public void setup() throws IOException { + test_root = new File(TEST_ROOT_DIR); + test_root.mkdirs(); + } + + @After + public void after() throws IOException { + FileUtil.setWritable(test_root, true); + FileUtil.fullyDelete(test_root); + assertTrue(!test_root.exists()); + } + public static class XXDF extends DF { public XXDF() throws IOException { - super(new File(System.getProperty("test.build.data","/tmp")), 0L); + super(test_root, 0L); } @Override From 5343b43fd989ec596afed807ddce29ad96c23e2d Mon Sep 17 00:00:00 2001 From: Uma Maheswara Rao G Date: Wed, 23 Jul 2014 19:05:11 +0000 Subject: [PATCH 05/37] HDFS-6422. getfattr in CLI doesn't throw exception or return non-0 return code when xattr doesn't exist. (Charles Lamb via umamahesh) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612922 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/protocol/ClientProtocol.java | 2 +- .../hdfs/server/namenode/FSEditLog.java | 3 +- .../hdfs/server/namenode/FSEditLogLoader.java | 4 + .../hdfs/server/namenode/FSEditLogOp.java | 5 + .../hdfs/server/namenode/FSNamesystem.java | 62 +++- .../namenode/XAttrPermissionFilter.java | 17 +- .../hdfs/web/resources/XAttrNameParam.java | 4 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 69 ++++ .../hdfs/server/namenode/FSXAttrBaseTest.java | 351 ++++++++++++++++-- .../namenode/TestNamenodeRetryCache.java | 4 +- .../namenode/ha/TestRetryCacheWithHA.java | 54 ++- .../hadoop/hdfs/web/resources/TestParam.java | 6 - .../src/test/resources/editsStored | Bin 4970 -> 4992 bytes .../src/test/resources/editsStored.xml | 2 + 15 files changed, 534 insertions(+), 52 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8c172bf03cf..4fbe4cfc283 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -892,6 +892,9 @@ Release 2.5.0 - UNRELEASED HDFS-6703. NFS: Files can be deleted from a read-only mount (Srikanth Upputuri via brandonli) + HDFS-6422. getfattr in CLI doesn't throw exception or return non-0 return code + when xattr doesn't exist. (Charles Lamb via umamahesh) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index ad331d1e755..9398c721a68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1337,6 +1337,6 @@ public interface ClientProtocol { * @param xAttr XAttr to remove * @throws IOException */ - @Idempotent + @AtMostOnce public void removeXAttr(String src, XAttr xAttr) throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 85cfc1c7746..b2adcd455fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -1074,10 +1074,11 @@ public class FSEditLog implements LogsPurgeable { logEdit(op); } - void logRemoveXAttrs(String src, List xAttrs) { + void logRemoveXAttrs(String src, List xAttrs, boolean toLogRpcIds) { final RemoveXAttrOp op = RemoveXAttrOp.getInstance(); op.src = src; op.xAttrs = xAttrs; + logRpcIds(op, toLogRpcIds); logEdit(op); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index 858cd57b23f..a721491948d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -821,6 +821,10 @@ public class FSEditLogLoader { RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op; fsDir.unprotectedRemoveXAttrs(removeXAttrOp.src, removeXAttrOp.xAttrs); + if (toAddRetryCache) { + fsNamesys.addCacheEntry(removeXAttrOp.rpcClientId, + removeXAttrOp.rpcCallId); + } break; } default: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index e972799b335..5543e0cb86e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -3551,6 +3551,7 @@ public abstract class FSEditLogOp { XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in); src = p.getSrc(); xAttrs = PBHelper.convertXAttrs(p.getXAttrsList()); + readRpcIds(in, logVersion); } @Override @@ -3561,18 +3562,22 @@ public abstract class FSEditLogOp { } b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs)); b.build().writeDelimitedTo(out); + // clientId and callId + writeRpcIds(rpcClientId, rpcCallId, out); } @Override protected void toXml(ContentHandler contentHandler) throws SAXException { XMLUtils.addSaxString(contentHandler, "SRC", src); appendXAttrsToXml(contentHandler, xAttrs); + appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId); } @Override void fromXml(Stanza st) throws InvalidXmlException { src = st.getValue("SRC"); xAttrs = readXAttrsFromXml(st); + readRpcIdsFromXml(st); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3e922ba023c..6ce6a70ce2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -8279,11 +8279,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, nnConf.checkXAttrsConfigFlag(); FSPermissionChecker pc = getPermissionChecker(); boolean getAll = xAttrs == null || xAttrs.isEmpty(); - List filteredXAttrs = null; if (!getAll) { - filteredXAttrs = XAttrPermissionFilter.filterXAttrsForApi(pc, xAttrs); - if (filteredXAttrs.isEmpty()) { - return filteredXAttrs; + try { + XAttrPermissionFilter.checkPermissionForApi(pc, xAttrs); + } catch (AccessControlException e) { + logAuditEvent(false, "getXAttrs", src); + throw e; } } checkOperation(OperationCategory.READ); @@ -8302,15 +8303,21 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if (filteredAll == null || filteredAll.isEmpty()) { return null; } - List toGet = Lists.newArrayListWithCapacity(filteredXAttrs.size()); - for (XAttr xAttr : filteredXAttrs) { + List toGet = Lists.newArrayListWithCapacity(xAttrs.size()); + for (XAttr xAttr : xAttrs) { + boolean foundIt = false; for (XAttr a : filteredAll) { if (xAttr.getNameSpace() == a.getNameSpace() && xAttr.getName().equals(a.getName())) { toGet.add(a); + foundIt = true; break; } } + if (!foundIt) { + throw new IOException( + "At least one of the attributes provided was not found."); + } } return toGet; } @@ -8344,17 +8351,42 @@ public class FSNamesystem implements Namesystem, FSClusterStats, readUnlock(); } } - + + /** + * Remove an xattr for a file or directory. + * + * @param src + * - path to remove the xattr from + * @param xAttr + * - xAttr to remove + * @throws AccessControlException + * @throws SafeModeException + * @throws UnresolvedLinkException + * @throws IOException + */ void removeXAttr(String src, XAttr xAttr) throws IOException { - nnConf.checkXAttrsConfigFlag(); - HdfsFileStatus resultingStat = null; - FSPermissionChecker pc = getPermissionChecker(); + CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); + if (cacheEntry != null && cacheEntry.isSuccess()) { + return; // Return previous response + } + boolean success = false; try { - XAttrPermissionFilter.checkPermissionForApi(pc, xAttr); + removeXAttrInt(src, xAttr, cacheEntry != null); + success = true; } catch (AccessControlException e) { logAuditEvent(false, "removeXAttr", src); throw e; + } finally { + RetryCache.setState(cacheEntry, success); } + } + + void removeXAttrInt(String src, XAttr xAttr, boolean logRetryCache) + throws IOException { + nnConf.checkXAttrsConfigFlag(); + HdfsFileStatus resultingStat = null; + FSPermissionChecker pc = getPermissionChecker(); + XAttrPermissionFilter.checkPermissionForApi(pc, xAttr); checkOperation(OperationCategory.WRITE); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); writeLock(); @@ -8368,12 +8400,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, xAttrs.add(xAttr); List removedXAttrs = dir.removeXAttrs(src, xAttrs); if (removedXAttrs != null && !removedXAttrs.isEmpty()) { - getEditLog().logRemoveXAttrs(src, removedXAttrs); + getEditLog().logRemoveXAttrs(src, removedXAttrs, logRetryCache); + } else { + throw new IOException( + "No matching attributes found for remove operation"); } resultingStat = getAuditFileInfo(src, false); - } catch (AccessControlException e) { - logAuditEvent(false, "removeXAttr", src); - throw e; } finally { writeUnlock(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java index 47f29399e5a..98730142fbd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrPermissionFilter.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.security.AccessControlException; import com.google.common.collect.Lists; +import com.google.common.base.Preconditions; /** * There are four types of extended attributes defined by the @@ -60,8 +61,20 @@ public class XAttrPermissionFilter { throw new AccessControlException("User doesn't have permission for xattr: " + XAttrHelper.getPrefixName(xAttr)); } - - static List filterXAttrsForApi(FSPermissionChecker pc, + + static void checkPermissionForApi(FSPermissionChecker pc, + List xAttrs) throws AccessControlException { + Preconditions.checkArgument(xAttrs != null); + if (xAttrs.isEmpty()) { + return; + } + + for (XAttr xAttr : xAttrs) { + checkPermissionForApi(pc, xAttr); + } + } + + static List filterXAttrsForApi(FSPermissionChecker pc, List xAttrs) { assert xAttrs != null : "xAttrs can not be null"; if (xAttrs == null || xAttrs.isEmpty()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java index 3860f916e48..8137b4494f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/XAttrNameParam.java @@ -25,8 +25,8 @@ public class XAttrNameParam extends StringParam { /** Default parameter value. **/ public static final String DEFAULT = ""; - private static Domain DOMAIN = new Domain(NAME, - Pattern.compile("^(user\\.|trusted\\.|system\\.|security\\.).+")); + private static Domain DOMAIN = new Domain(NAME, + Pattern.compile(".*")); public XAttrNameParam(final String str) { super(DOMAIN, str == null || str.equals(DEFAULT) ? null : str); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 8eb1c41e052..aac16f4e5e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -2653,6 +2653,75 @@ public class TestDFSShell { } } + /* + * 1. Test that CLI throws an exception and returns non-0 when user does + * not have permission to read an xattr. + * 2. Test that CLI throws an exception and returns non-0 when a non-existent + * xattr is requested. + */ + @Test (timeout = 120000) + public void testGetFAttrErrors() throws Exception { + final UserGroupInformation user = UserGroupInformation. + createUserForTesting("user", new String[] {"mygroup"}); + MiniDFSCluster cluster = null; + PrintStream bakErr = null; + try { + final Configuration conf = new HdfsConfiguration(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + + final FileSystem fs = cluster.getFileSystem(); + final Path p = new Path("/foo"); + fs.mkdirs(p); + bakErr = System.err; + + final FsShell fshell = new FsShell(conf); + final ByteArrayOutputStream out = new ByteArrayOutputStream(); + System.setErr(new PrintStream(out)); + + // No permission for "other". + fs.setPermission(p, new FsPermission((short) 0700)); + + { + final int ret = ToolRunner.run(fshell, new String[] { + "-setfattr", "-n", "user.a1", "-v", "1234", "/foo"}); + assertEquals("Returned should be 0", 0, ret); + out.reset(); + } + + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + int ret = ToolRunner.run(fshell, new String[] { + "-getfattr", "-n", "user.a1", "/foo"}); + String str = out.toString(); + assertTrue("xattr value was incorrectly returned", + str.indexOf("1234") == -1); + out.reset(); + return null; + } + }); + + { + final int ret = ToolRunner.run(fshell, new String[]{ + "-getfattr", "-n", "user.nonexistent", "/foo"}); + String str = out.toString(); + assertTrue("xattr value was incorrectly returned", + str.indexOf( + "getfattr: At least one of the attributes provided was not found") + >= 0); + out.reset(); + } + } finally { + if (bakErr != null) { + System.setErr(bakErr); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + /** * Test that the server trash configuration is respected when * the client configuration is not set. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java index 86f1ec90b8b..636ecc2417f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.FileNotFoundException; import java.security.PrivilegedExceptionAction; import java.util.EnumSet; import java.util.List; @@ -46,6 +45,7 @@ import static org.apache.hadoop.fs.permission.AclEntryType.USER; import static org.apache.hadoop.fs.permission.FsAction.ALL; import static org.apache.hadoop.fs.permission.FsAction.READ; import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.junit.After; @@ -261,11 +261,12 @@ public class FSXAttrBaseTest { fs.setXAttr(path, "user.", value1, EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE)); Assert.fail("Setting xattr with empty name should fail."); + } catch (RemoteException e) { + assertEquals("Unexpected RemoteException: " + e, e.getClassName(), + HadoopIllegalArgumentException.class.getCanonicalName()); + GenericTestUtils.assertExceptionContains("XAttr name cannot be empty", e); } catch (HadoopIllegalArgumentException e) { GenericTestUtils.assertExceptionContains("XAttr name cannot be empty", e); - } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains("Invalid value: \"user.\" does " + - "not belong to the domain ^(user\\.|trusted\\.|system\\.|security\\.).+", e); } // Set xattr with invalid name: "a1" @@ -274,11 +275,12 @@ public class FSXAttrBaseTest { XAttrSetFlag.REPLACE)); Assert.fail("Setting xattr with invalid name prefix or without " + "name prefix should fail."); + } catch (RemoteException e) { + assertEquals("Unexpected RemoteException: " + e, e.getClassName(), + HadoopIllegalArgumentException.class.getCanonicalName()); + GenericTestUtils.assertExceptionContains("XAttr name must be prefixed", e); } catch (HadoopIllegalArgumentException e) { GenericTestUtils.assertExceptionContains("XAttr name must be prefixed", e); - } catch (IllegalArgumentException e) { - GenericTestUtils.assertExceptionContains("Invalid value: \"a1\" does " + - "not belong to the domain ^(user\\.|trusted\\.|system\\.|security\\.).+", e); } // Set xattr without XAttrSetFlag @@ -341,9 +343,18 @@ public class FSXAttrBaseTest { } /** - * Tests for getting xattr - * 1. To get xattr which does not exist. - * 2. To get multiple xattrs. + * getxattr tests. Test that getxattr throws an exception if any of + * the following are true: + * an xattr that was requested doesn't exist + * the caller specifies an unknown namespace + * the caller doesn't have access to the namespace + * the caller doesn't have permission to get the value of the xattr + * the caller does not have search access to the parent directory + * the caller has only read access to the owning directory + * the caller has only search access to the owning directory and + * execute/search access to the actual entity + * the caller does not have search access to the owning directory and read + * access to the actual entity */ @Test(timeout = 120000) public void testGetXAttrs() throws Exception { @@ -351,21 +362,159 @@ public class FSXAttrBaseTest { fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); - // XAttr does not exist. - byte[] value = fs.getXAttr(path, name3); - Assert.assertEquals(value, null); + /* An XAttr that was requested does not exist. */ + try { + final byte[] value = fs.getXAttr(path, name3); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains( + "At least one of the attributes provided was not found.", e); + } - List names = Lists.newArrayList(); - names.add(name1); - names.add(name2); - names.add(name3); - Map xattrs = fs.getXAttrs(path, names); - Assert.assertEquals(xattrs.size(), 2); - Assert.assertArrayEquals(value1, xattrs.get(name1)); - Assert.assertArrayEquals(value2, xattrs.get(name2)); + /* Throw an exception if an xattr that was requested does not exist. */ + { + final List names = Lists.newArrayList(); + names.add(name1); + names.add(name2); + names.add(name3); + try { + final Map xattrs = fs.getXAttrs(path, names); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains( + "At least one of the attributes provided was not found.", e); + } + } fs.removeXAttr(path, name1); fs.removeXAttr(path, name2); + + /* Unknown namespace should throw an exception. */ + try { + final byte[] xattr = fs.getXAttr(path, "wackynamespace.foo"); + Assert.fail("expected IOException"); + } catch (Exception e) { + GenericTestUtils.assertExceptionContains + ("An XAttr name must be prefixed with user/trusted/security/system, " + + "followed by a '.'", + e); + } + + /* + * The 'trusted' namespace should not be accessible and should throw an + * exception. + */ + final UserGroupInformation user = UserGroupInformation. + createUserForTesting("user", new String[] {"mygroup"}); + fs.setXAttr(path, "trusted.foo", "1234".getBytes()); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + final byte[] xattr = userFs.getXAttr(path, "trusted.foo"); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("User doesn't have permission", e); + } + + fs.setXAttr(path, name1, "1234".getBytes()); + + /* + * Test that an exception is thrown if the caller doesn't have permission to + * get the value of the xattr. + */ + + /* Set access so that only the owner has access. */ + fs.setPermission(path, new FsPermission((short) 0700)); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + final byte[] xattr = userFs.getXAttr(path, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* + * The caller must have search access to the parent directory. + */ + final Path childDir = new Path(path, "child" + pathCount); + /* Set access to parent so that only the owner has access. */ + FileSystem.mkdirs(fs, childDir, FsPermission.createImmutable((short)0700)); + fs.setXAttr(childDir, name1, "1234".getBytes()); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + final byte[] xattr = userFs.getXAttr(childDir, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* Check that read access to the owning directory is not good enough. */ + fs.setPermission(path, new FsPermission((short) 0704)); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + final byte[] xattr = userFs.getXAttr(childDir, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* + * Check that search access to the owning directory and search/execute + * access to the actual entity with extended attributes is not good enough. + */ + fs.setPermission(path, new FsPermission((short) 0701)); + fs.setPermission(childDir, new FsPermission((short) 0701)); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + final byte[] xattr = userFs.getXAttr(childDir, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* + * Check that search access to the owning directory and read access to + * the actual entity with the extended attribute is good enough. + */ + fs.setPermission(path, new FsPermission((short) 0701)); + fs.setPermission(childDir, new FsPermission((short) 0704)); + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + final byte[] xattr = userFs.getXAttr(childDir, name1); + return null; + } + }); } /** @@ -402,6 +551,166 @@ public class FSXAttrBaseTest { fs.removeXAttr(path, name3); } + /** + * removexattr tests. Test that removexattr throws an exception if any of + * the following are true: + * an xattr that was requested doesn't exist + * the caller specifies an unknown namespace + * the caller doesn't have access to the namespace + * the caller doesn't have permission to get the value of the xattr + * the caller does not have "execute" (scan) access to the parent directory + * the caller has only read access to the owning directory + * the caller has only execute access to the owning directory and execute + * access to the actual entity + * the caller does not have execute access to the owning directory and write + * access to the actual entity + */ + @Test(timeout = 120000) + public void testRemoveXAttrPermissions() throws Exception { + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750)); + fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(path, name3, null, EnumSet.of(XAttrSetFlag.CREATE)); + + try { + fs.removeXAttr(path, name2); + fs.removeXAttr(path, name2); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("No matching attributes found", e); + } + + /* Unknown namespace should throw an exception. */ + final String expectedExceptionString = "An XAttr name must be prefixed " + + "with user/trusted/security/system, followed by a '.'"; + try { + fs.removeXAttr(path, "wackynamespace.foo"); + Assert.fail("expected IOException"); + } catch (RemoteException e) { + assertEquals("Unexpected RemoteException: " + e, e.getClassName(), + HadoopIllegalArgumentException.class.getCanonicalName()); + GenericTestUtils.assertExceptionContains(expectedExceptionString, e); + } catch (HadoopIllegalArgumentException e) { + GenericTestUtils.assertExceptionContains(expectedExceptionString, e); + } + + /* + * The 'trusted' namespace should not be accessible and should throw an + * exception. + */ + final UserGroupInformation user = UserGroupInformation. + createUserForTesting("user", new String[] {"mygroup"}); + fs.setXAttr(path, "trusted.foo", "1234".getBytes()); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + userFs.removeXAttr(path, "trusted.foo"); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("User doesn't have permission", e); + } finally { + fs.removeXAttr(path, "trusted.foo"); + } + + /* + * Test that an exception is thrown if the caller doesn't have permission to + * get the value of the xattr. + */ + + /* Set access so that only the owner has access. */ + fs.setPermission(path, new FsPermission((short) 0700)); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + userFs.removeXAttr(path, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* + * The caller must have "execute" (scan) access to the parent directory. + */ + final Path childDir = new Path(path, "child" + pathCount); + /* Set access to parent so that only the owner has access. */ + FileSystem.mkdirs(fs, childDir, FsPermission.createImmutable((short)0700)); + fs.setXAttr(childDir, name1, "1234".getBytes()); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + userFs.removeXAttr(childDir, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* Check that read access to the owning directory is not good enough. */ + fs.setPermission(path, new FsPermission((short) 0704)); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + userFs.removeXAttr(childDir, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* + * Check that execute access to the owning directory and scan access to + * the actual entity with extended attributes is not good enough. + */ + fs.setPermission(path, new FsPermission((short) 0701)); + fs.setPermission(childDir, new FsPermission((short) 0701)); + try { + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + userFs.removeXAttr(childDir, name1); + return null; + } + }); + Assert.fail("expected IOException"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Permission denied", e); + } + + /* + * Check that execute access to the owning directory and write access to + * the actual entity with extended attributes is good enough. + */ + fs.setPermission(path, new FsPermission((short) 0701)); + fs.setPermission(childDir, new FsPermission((short) 0706)); + user.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + final FileSystem userFs = dfsCluster.getFileSystem(); + userFs.removeXAttr(childDir, name1); + return null; + } + }); + } + @Test(timeout = 120000) public void testRenameFileWithXAttr() throws Exception { FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java index d6f38853474..a0ae43b57fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeRetryCache.java @@ -415,7 +415,7 @@ public class TestNamenodeRetryCache { LightWeightCache cacheSet = (LightWeightCache) namesystem.getRetryCache().getCacheSet(); - assertEquals(22, cacheSet.size()); + assertEquals(23, cacheSet.size()); Map oldEntries = new HashMap(); @@ -434,7 +434,7 @@ public class TestNamenodeRetryCache { assertTrue(namesystem.hasRetryCache()); cacheSet = (LightWeightCache) namesystem .getRetryCache().getCacheSet(); - assertEquals(22, cacheSet.size()); + assertEquals(23, cacheSet.size()); iter = cacheSet.iterator(); while (iter.hasNext()) { CacheEntry entry = iter.next(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java index a34a0365f99..77f7090e8f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java @@ -160,7 +160,7 @@ public class TestRetryCacheWithHA { FSNamesystem fsn0 = cluster.getNamesystem(0); LightWeightCache cacheSet = (LightWeightCache) fsn0.getRetryCache().getCacheSet(); - assertEquals(22, cacheSet.size()); + assertEquals(23, cacheSet.size()); Map oldEntries = new HashMap(); @@ -181,7 +181,7 @@ public class TestRetryCacheWithHA { FSNamesystem fsn1 = cluster.getNamesystem(1); cacheSet = (LightWeightCache) fsn1 .getRetryCache().getCacheSet(); - assertEquals(22, cacheSet.size()); + assertEquals(23, cacheSet.size()); iter = cacheSet.iterator(); while (iter.hasNext()) { CacheEntry entry = iter.next(); @@ -1047,6 +1047,49 @@ public class TestRetryCacheWithHA { } } + /** removeXAttr */ + class RemoveXAttrOp extends AtMostOnceOp { + private final String src; + + RemoveXAttrOp(DFSClient client, String src) { + super("removeXAttr", client); + this.src = src; + } + + @Override + void prepare() throws Exception { + Path p = new Path(src); + if (!dfs.exists(p)) { + DFSTestUtil.createFile(dfs, p, BlockSize, DataNodes, 0); + client.setXAttr(src, "user.key", "value".getBytes(), + EnumSet.of(XAttrSetFlag.CREATE)); + } + } + + @Override + void invoke() throws Exception { + client.removeXAttr(src, "user.key"); + } + + @Override + boolean checkNamenodeBeforeReturn() throws Exception { + for (int i = 0; i < CHECKTIMES; i++) { + Map iter = dfs.getXAttrs(new Path(src)); + Set keySet = iter.keySet(); + if (!keySet.contains("user.key")) { + return true; + } + Thread.sleep(1000); + } + return false; + } + + @Override + Object getResult() { + return null; + } + } + @Test (timeout=60000) public void testCreateSnapshot() throws Exception { final DFSClient client = genClientWithDummyHandler(); @@ -1183,6 +1226,13 @@ public class TestRetryCacheWithHA { testClientRetryWithFailover(op); } + @Test (timeout=60000) + public void testRemoveXAttr() throws Exception { + DFSClient client = genClientWithDummyHandler(); + AtMostOnceOp op = new RemoveXAttrOp(client, "/removexattr"); + testClientRetryWithFailover(op); + } + /** * When NN failover happens, if the client did not receive the response and * send a retry request to the other NN, the same response should be recieved diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index 1a20739a712..a84243e7d4a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -355,12 +355,6 @@ public class TestParam { public void testXAttrNameParam() { final XAttrNameParam p = new XAttrNameParam("user.a1"); Assert.assertEquals(p.getXAttrName(), "user.a1"); - try { - new XAttrNameParam("a1"); - Assert.fail(); - } catch (IllegalArgumentException e) { - LOG.info("EXPECTED: " + e); - } } @Test diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index a3561cda528a9bae7b6fb6e16180ca9a9d344d7a..a134969ca79136819a5f1e2d5e16771d54aed04a 100644 GIT binary patch delta 27 jcmaE*)}X%Oj}W87USER a2 + e03f4a52-3d85-4e05-8942-286185e639bd + 82 From a6b1d2a85cad227182c28633886670a077aadb4c Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Wed, 23 Jul 2014 20:59:39 +0000 Subject: [PATCH 06/37] HDFS-6114. Block Scan log rolling will never happen if blocks written continuously leading to huge size of dncp_block_verification.log.curr (vinayakumarb via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612943 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++ .../datanode/BlockPoolSliceScanner.java | 54 ++++++++++++++----- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4fbe4cfc283..63168e0ca21 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -357,6 +357,10 @@ Release 2.6.0 - UNRELEASED HDFS-6731. Run "hdfs zkfc-formatZK" on a server in a non-namenode will cause a null pointer exception. (Masatake Iwasaki via brandonli) + HDFS-6114. Block Scan log rolling will never happen if blocks written + continuously leading to huge size of dncp_block_verification.log.curr + (vinayakumarb via cmccabe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java index 1039b4fe922..bbb67fc4739 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java @@ -84,6 +84,10 @@ class BlockPoolSliceScanner { private final SortedSet blockInfoSet = new TreeSet(BlockScanInfo.LAST_SCAN_TIME_COMPARATOR); + + private final SortedSet newBlockInfoSet = + new TreeSet(BlockScanInfo.LAST_SCAN_TIME_COMPARATOR); + private final GSet blockMap = new LightWeightGSet( LightWeightGSet.computeCapacity(0.5, "BlockMap")); @@ -195,7 +199,7 @@ class BlockPoolSliceScanner { BlockScanInfo info = new BlockScanInfo( block ); info.lastScanTime = scanTime--; //still keep 'info.lastScanType' to NONE. - addBlockInfo(info); + addBlockInfo(info, false); } RollingLogs rollingLogs = null; @@ -221,25 +225,42 @@ class BlockPoolSliceScanner { // Should we change throttler bandwidth every time bytesLeft changes? // not really required. } - - private synchronized void addBlockInfo(BlockScanInfo info) { - boolean added = blockInfoSet.add(info); + + /** + * Add the BlockScanInfo to sorted set of blockScanInfo + * @param info BlockScanInfo to be added + * @param isNewBlock true if the block is the new Block, false if + * BlockScanInfo is being updated with new scanTime + */ + private synchronized void addBlockInfo(BlockScanInfo info, + boolean isNewBlock) { + boolean added = false; + if (isNewBlock) { + // check whether the block already present + boolean exists = blockInfoSet.contains(info); + added = !exists && newBlockInfoSet.add(info); + } else { + added = blockInfoSet.add(info); + } blockMap.put(info); if (added) { updateBytesToScan(info.getNumBytes(), info.lastScanTime); } } - + private synchronized void delBlockInfo(BlockScanInfo info) { boolean exists = blockInfoSet.remove(info); + if (!exists){ + exists = newBlockInfoSet.remove(info); + } blockMap.remove(info); if (exists) { updateBytesToScan(-info.getNumBytes(), info.lastScanTime); } } - + /** Update blockMap by the given LogEntry */ private synchronized void updateBlockInfo(LogEntry e) { BlockScanInfo info = blockMap.get(new Block(e.blockId, 0, e.genStamp)); @@ -249,7 +270,7 @@ class BlockPoolSliceScanner { delBlockInfo(info); info.lastScanTime = e.verificationTime; info.lastScanType = ScanType.VERIFICATION_SCAN; - addBlockInfo(info); + addBlockInfo(info, false); } } @@ -275,14 +296,14 @@ class BlockPoolSliceScanner { info = new BlockScanInfo(block.getLocalBlock()); info.lastScanTime = getNewBlockScanTime(); - addBlockInfo(info); + addBlockInfo(info, true); adjustThrottler(); } /** Deletes the block from internal structures */ synchronized void deleteBlock(Block block) { BlockScanInfo info = blockMap.get(block); - if ( info != null ) { + if (info != null) { delBlockInfo(info); } } @@ -319,7 +340,7 @@ class BlockPoolSliceScanner { info.lastScanType = type; info.lastScanTime = now; info.lastScanOk = scanOk; - addBlockInfo(info); + addBlockInfo(info, false); // Don't update meta data if the verification failed. if (!scanOk) { @@ -578,7 +599,7 @@ class BlockPoolSliceScanner { delBlockInfo(info); info.lastScanTime = lastScanTime; lastScanTime += verifyInterval; - addBlockInfo(info); + addBlockInfo(info, false); } } } @@ -674,12 +695,21 @@ class BlockPoolSliceScanner { throw e; } finally { rollVerificationLogs(); + rollNewBlocksInfo(); if (LOG.isDebugEnabled()) { LOG.debug("Done scanning block pool: " + blockPoolId); } } } - + + // add new blocks to scan in next iteration + private synchronized void rollNewBlocksInfo() { + for (BlockScanInfo newBlock : newBlockInfoSet) { + blockInfoSet.add(newBlock); + } + newBlockInfoSet.clear(); + } + private synchronized void rollVerificationLogs() { if (verificationLog != null) { try { From 2a5f1029a5221c42ab61b22f99d79251ed069ca4 Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Wed, 23 Jul 2014 21:22:50 +0000 Subject: [PATCH 07/37] HDFS-6455. NFS: Exception should be added in NFS log for invalid separator in nfs.exports.allowed.hosts. Contributed by Abhiraj Butala git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612947 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/nfs/NfsExports.java | 7 ++++++- .../hdfs/nfs/mount/RpcProgramMountd.java | 19 +++++++++++++++---- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 5 ++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java index 96286865385..b617ae5088d 100644 --- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java +++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/NfsExports.java @@ -53,7 +53,12 @@ public class NfsExports { long expirationPeriodNano = conf.getLong( Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY, Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_DEFAULT) * 1000 * 1000; - exports = new NfsExports(cacheSize, expirationPeriodNano, matchHosts); + try { + exports = new NfsExports(cacheSize, expirationPeriodNano, matchHosts); + } catch (IllegalArgumentException e) { + LOG.error("Invalid NFS Exports provided: ", e); + return exports; + } } return exports; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java index 9fbab240f6e..2814cb007e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java @@ -104,6 +104,10 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface { @Override public XDR mnt(XDR xdr, XDR out, int xid, InetAddress client) { + if (hostsMatcher == null) { + return MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_ACCES, out, xid, + null); + } AccessPrivilege accessPrivilege = hostsMatcher.getAccessPrivilege(client); if (accessPrivilege == AccessPrivilege.NONE) { return MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_ACCES, out, xid, @@ -208,16 +212,23 @@ public class RpcProgramMountd extends RpcProgram implements MountInterface { } else if (mntproc == MNTPROC.UMNTALL) { umntall(out, xid, client); } else if (mntproc == MNTPROC.EXPORT) { - // Currently only support one NFS export + // Currently only support one NFS export List hostsMatchers = new ArrayList(); - hostsMatchers.add(hostsMatcher); - out = MountResponse.writeExportList(out, xid, exports, hostsMatchers); + if (hostsMatcher != null) { + hostsMatchers.add(hostsMatcher); + out = MountResponse.writeExportList(out, xid, exports, hostsMatchers); + } else { + // This means there are no valid exports provided. + RpcAcceptedReply.getInstance(xid, + RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()).write( + out); + } } else { // Invalid procedure RpcAcceptedReply.getInstance(xid, RpcAcceptedReply.AcceptState.PROC_UNAVAIL, new VerifierNone()).write( out); - } + } ChannelBuffer buf = ChannelBuffers.wrappedBuffer(out.asReadOnlyWrap().buffer()); RpcResponse rsp = new RpcResponse(buf, info.remoteAddress()); RpcUtil.sendRpcResponse(ctx, rsp); diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java index f254f50709d..1650b14724d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java @@ -2123,8 +2123,11 @@ public class RpcProgramNfs3 extends RpcProgram implements Nfs3Interface { if (!doPortMonitoring(remoteAddress)) { return false; } - + // Check export table + if (exports == null) { + return false; + } InetAddress client = ((InetSocketAddress) remoteAddress).getAddress(); AccessPrivilege access = exports.getAccessPrivilege(client); if (access == AccessPrivilege.NONE) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 63168e0ca21..bbeebd19843 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -361,6 +361,9 @@ Release 2.6.0 - UNRELEASED continuously leading to huge size of dncp_block_verification.log.curr (vinayakumarb via cmccabe) + HDFS-6455. NFS: Exception should be added in NFS log for invalid separator in + nfs.exports.allowed.hosts. (Abhiraj Butala via brandonli) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES From 28fca92521b04e9d9b6f4d095c593282a06e0a36 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Wed, 23 Jul 2014 21:40:57 +0000 Subject: [PATCH 08/37] YARN-2147. client lacks delegation token exception details when application submit fails. Contributed by Chen He git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612950 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../security/DelegationTokenRenewer.java | 6 +++- .../security/TestDelegationTokenRenewer.java | 36 ++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4c42fd7cdce..6fbfbab5423 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -94,6 +94,9 @@ Release 2.6.0 - UNRELEASED YARN-2313. Livelock can occur in FairScheduler when there are lots of running apps (Tsuyoshi Ozawa via Sandy Ryza) + YARN-2147. client lacks delegation token exception details when + application submit fails (Chen He via jlowe) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java index 38e908926d9..bdcfd0460ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java @@ -388,7 +388,11 @@ public class DelegationTokenRenewer extends AbstractService { // If user provides incorrect token then it should not be added for // renewal. for (DelegationTokenToRenew dtr : tokenList) { - renewToken(dtr); + try { + renewToken(dtr); + } catch (IOException ioe) { + throw new IOException("Failed to renew token: " + dtr.token, ioe); + } } for (DelegationTokenToRenew dtr : tokenList) { addTokenToList(dtr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java index 0c1ded3a149..f65fcdcb3b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java @@ -24,6 +24,7 @@ import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -673,7 +674,40 @@ public class TestDelegationTokenRenewer { Thread.sleep(200); } } - + + @Test(timeout=20000) + public void testDTRonAppSubmission() + throws IOException, InterruptedException, BrokenBarrierException { + final Credentials credsx = new Credentials(); + final Token tokenx = mock(Token.class); + credsx.addToken(new Text("token"), tokenx); + doReturn(true).when(tokenx).isManaged(); + doThrow(new IOException("boom")) + .when(tokenx).renew(any(Configuration.class)); + // fire up the renewer + final DelegationTokenRenewer dtr = + createNewDelegationTokenRenewer(conf, counter); + RMContext mockContext = mock(RMContext.class); + ClientRMService mockClientRMService = mock(ClientRMService.class); + when(mockContext.getClientRMService()).thenReturn(mockClientRMService); + InetSocketAddress sockAddr = + InetSocketAddress.createUnresolved("localhost", 1234); + when(mockClientRMService.getBindAddress()).thenReturn(sockAddr); + dtr.setRMContext(mockContext); + when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr); + dtr.init(conf); + dtr.start(); + + try { + dtr.addApplicationSync(mock(ApplicationId.class), credsx, false); + fail("Catch IOException on app submission"); + } catch (IOException e){ + Assert.assertTrue(e.getMessage().contains(tokenx.toString())); + Assert.assertTrue(e.getCause().toString().contains("boom")); + } + + } + @Test(timeout=20000) public void testConcurrentAddApplication() throws IOException, InterruptedException, BrokenBarrierException { From 9bfae42538048f25596d688d27be9f21956e0870 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Wed, 23 Jul 2014 21:42:01 +0000 Subject: [PATCH 09/37] HADOOP-10887. Add XAttrs to ViewFs and make XAttrs + ViewFileSystem internal dir behavior consistent. Contributed by Stephen Chu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612951 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/fs/FileContext.java | 29 +++++++ .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../apache/hadoop/fs/viewfs/ChRootedFs.java | 34 ++++++++ .../hadoop/fs/viewfs/ViewFileSystem.java | 34 ++++++++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 80 +++++++++++++++++++ .../fs/viewfs/ViewFileSystemBaseTest.java | 30 +++++++ .../hadoop/fs/viewfs/ViewFsBaseTest.java | 30 +++++++ 8 files changed, 241 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2463fb065da..08bc9012e9e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -452,6 +452,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10855. Allow Text to be read with a known Length. (todd) + HADOOP-10887. Add XAttrs to ViewFs and make XAttrs + ViewFileSystem + internal dir behavior consistent. (Stephen Chu via wang) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 8bb797eb93f..2bfcbdccebd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -2484,4 +2484,33 @@ public final class FileContext { } }.resolve(this, absF); } + + /** + * Get all of the xattr names for a file or directory. + * Only those xattr names which the logged-in user has permissions to view + * are returned. + *

+ * A regular user can only get xattr names for the "user" namespace. + * The super user can only get xattr names for "user" and "trusted" + * namespaces. + * The xattrs of the "security" and "system" namespaces are only + * used/exposed internally by/to the FS impl. + *

+ * @see + * http://en.wikipedia.org/wiki/Extended_file_attributes + * + * @param path Path to get extended attributes + * @return List of the XAttr names of the file or directory + * @throws IOException + */ + public List listXAttrs(Path path) throws IOException { + final Path absF = fixRelativePart(path); + return new FSLinkResolver>() { + @Override + public List next(final AbstractFileSystem fs, final Path p) + throws IOException { + return fs.listXAttrs(p); + } + }.resolve(this, absF); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index deaceb33424..cb921c88424 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -2509,7 +2509,7 @@ public abstract class FileSystem extends Configured implements Closeable { * http://en.wikipedia.org/wiki/Extended_file_attributes * * @param path Path to get extended attributes - * @return Map describing the XAttrs of the file or directory + * @return List of the XAttr names of the file or directory * @throws IOException */ public List listXAttrs(Path path) throws IOException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index f1975eae1b2..5d53eb79d0a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -22,6 +22,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.EnumSet; import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -37,6 +38,7 @@ import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; @@ -313,6 +315,38 @@ class ChRootedFs extends AbstractFileSystem { return myFs.getAclStatus(fullPath(path)); } + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + myFs.setXAttr(fullPath(path), name, value, flag); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + return myFs.getXAttr(fullPath(path), name); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + return myFs.getXAttrs(fullPath(path)); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + return myFs.getXAttrs(fullPath(path), names); + } + + @Override + public List listXAttrs(Path path) throws IOException { + return myFs.listXAttrs(fullPath(path)); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + myFs.removeXAttr(fullPath(path), name); + } + @Override public void setVerifyChecksum(final boolean verifyChecksum) throws IOException, UnresolvedLinkException { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 34a9afc5499..b4ac18eb1af 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -913,5 +913,39 @@ public class ViewFileSystem extends FileSystem { .addEntries(AclUtil.getMinimalAcl(PERMISSION_555)) .stickyBit(false).build(); } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("setXAttr", path); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + throw new NotInMountpointException(path, "getXAttr"); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + throw new NotInMountpointException(path, "getXAttrs"); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + throw new NotInMountpointException(path, "getXAttrs"); + } + + @Override + public List listXAttrs(Path path) throws IOException { + throw new NotInMountpointException(path, "listXAttrs"); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("removeXAttr", path); + } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index 232fcbbb409..5cdccd29975 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -26,6 +26,7 @@ import java.net.URISyntaxException; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; +import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.classification.InterfaceAudience; @@ -48,6 +49,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.local.LocalConfigKeys; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclUtil; @@ -651,6 +653,50 @@ public class ViewFs extends AbstractFileSystem { fsState.resolve(getUriPath(path), true); return res.targetFileSystem.getAclStatus(res.remainingPath); } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + res.targetFileSystem.setXAttr(res.remainingPath, name, value, flag); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + return res.targetFileSystem.getXAttr(res.remainingPath, name); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + return res.targetFileSystem.getXAttrs(res.remainingPath); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + return res.targetFileSystem.getXAttrs(res.remainingPath, names); + } + + @Override + public List listXAttrs(Path path) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + return res.targetFileSystem.listXAttrs(res.remainingPath); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + InodeTree.ResolveResult res = + fsState.resolve(getUriPath(path), true); + res.targetFileSystem.removeXAttr(res.remainingPath, name); + } /* @@ -921,5 +967,39 @@ public class ViewFs extends AbstractFileSystem { .addEntries(AclUtil.getMinimalAcl(PERMISSION_555)) .stickyBit(false).build(); } + + @Override + public void setXAttr(Path path, String name, byte[] value, + EnumSet flag) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("setXAttr", path); + } + + @Override + public byte[] getXAttr(Path path, String name) throws IOException { + throw new NotInMountpointException(path, "getXAttr"); + } + + @Override + public Map getXAttrs(Path path) throws IOException { + throw new NotInMountpointException(path, "getXAttrs"); + } + + @Override + public Map getXAttrs(Path path, List names) + throws IOException { + throw new NotInMountpointException(path, "getXAttrs"); + } + + @Override + public List listXAttrs(Path path) throws IOException { + throw new NotInMountpointException(path, "listXAttrs"); + } + + @Override + public void removeXAttr(Path path, String name) throws IOException { + checkPathIsSlash(path); + throw readOnlyMountTable("removeXAttr", path); + } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index e1a440d0614..a32455604c9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -773,4 +773,34 @@ public class ViewFileSystemBaseTest { assertFalse(aclStatus.isStickyBit()); } + @Test(expected=AccessControlException.class) + public void testInternalSetXAttr() throws IOException { + fsView.setXAttr(new Path("/internalDir"), "xattrName", null); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalGetXAttr() throws IOException { + fsView.getXAttr(new Path("/internalDir"), "xattrName"); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalGetXAttrs() throws IOException { + fsView.getXAttrs(new Path("/internalDir")); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalGetXAttrsWithNames() throws IOException { + fsView.getXAttrs(new Path("/internalDir"), new ArrayList()); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalListXAttr() throws IOException { + fsView.listXAttrs(new Path("/internalDir")); + } + + @Test(expected=AccessControlException.class) + public void testInternalRemoveXAttr() throws IOException { + fsView.removeXAttr(new Path("/internalDir"), "xattrName"); + } + } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java index 2813c34bef4..035b280249d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFsBaseTest.java @@ -747,4 +747,34 @@ public class ViewFsBaseTest { AclUtil.getMinimalAcl(PERMISSION_555)); assertFalse(aclStatus.isStickyBit()); } + + @Test(expected=AccessControlException.class) + public void testInternalSetXAttr() throws IOException { + fcView.setXAttr(new Path("/internalDir"), "xattrName", null); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalGetXAttr() throws IOException { + fcView.getXAttr(new Path("/internalDir"), "xattrName"); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalGetXAttrs() throws IOException { + fcView.getXAttrs(new Path("/internalDir")); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalGetXAttrsWithNames() throws IOException { + fcView.getXAttrs(new Path("/internalDir"), new ArrayList()); + } + + @Test(expected=NotInMountpointException.class) + public void testInternalListXAttr() throws IOException { + fcView.listXAttrs(new Path("/internalDir")); + } + + @Test(expected=AccessControlException.class) + public void testInternalRemoveXAttr() throws IOException { + fcView.removeXAttr(new Path("/internalDir"), "xattrName"); + } } From 06e5c5cb2de0148014a7c0433c885b099757b775 Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Thu, 24 Jul 2014 03:12:45 +0000 Subject: [PATCH 10/37] YARN-2300. Improved the documentation of the sample requests for RM REST API - submitting an app. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612981 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../src/site/apt/ResourceManagerRest.apt.vm | 109 ++++++++---------- 2 files changed, 50 insertions(+), 62 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6fbfbab5423..6857d6c3747 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -303,6 +303,9 @@ Release 2.5.0 - UNRELEASED YARN-1408 Preemption caused Invalid State Event: ACQUIRED at KILLED and caused a task timeout for 30mins. (Sunil G via mayank) + YARN-2300. Improved the documentation of the sample requests for RM REST API - + submitting an app. (Varun Vasudev via zjshen) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 6359e2b7f98..1952e11176d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -2228,68 +2228,48 @@ _01_000001 { "application-id":"application_1404203615263_0001", "application-name":"test", - "queue":"testqueue", - "priority":"3", "am-container-spec": { "local-resources": { "entry": - { - "key":"example", - "value": + [ { - "resource":"http://www.test.com/file.txt", - "type":"FILE", - "visibility":"APPLICATION", - "size":"100", - "timestamp":"1404203616003" + "key":"AppMaster.jar", + "value": + { + "resource":"hdfs://hdfs-namenode:9000/user/testuser/DistributedShell/demo-app/AppMaster.jar", + "type":"FILE", + "visibility":"APPLICATION", + "size": "43004", + "timestamp": "1405452071209" + } } - } + ] + }, + "commands": + { + "command":"{{JAVA_HOME}}/bin/java -Xmx10m org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster --container_memory 10 --container_vcores 1 --num_containers 1 --priority 0 1>/AppMaster.stdout 2>/AppMaster.stderr" }, "environment": - { - "entry": - { - "key":"APP_VAR", - "value":"ENV_SETTING" - } - }, - "commands": - { - "command":"/bin/sleep 5" - }, - "service-data": - { - "entry": - { - "key":"test", - "value":"dmFsdWUxMg" - } - }, - "credentials": - { - "tokens":null, - "secrets": - { - "entry": - { - "key":"secret1", - "value":"c2VjcmV0MQ" - } - } - }, - "application-acls": { "entry": [ { - "key":"VIEW_APP", - "value":"testuser3, testuser4" + "key": "DISTRIBUTEDSHELLSCRIPTTIMESTAMP", + "value": "1405459400754" }, { - "key":"MODIFY_APP", - "value":"testuser1, testuser2" + "key": "CLASSPATH", + "value": "{{CLASSPATH}}./*{{HADOOP_CONF_DIR}}{{HADOOP_COMMON_HOME}}/share/hadoop/common/*{{HADOOP_COMMON_HOME}}/share/hadoop/common/lib/*{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/*{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/lib/*{{HADOOP_YARN_HOME}}/share/hadoop/yarn/*{{HADOOP_YARN_HOME}}/share/hadoop/yarn/lib/*./log4j.properties" + }, + { + "key": "DISTRIBUTEDSHELLSCRIPTLEN", + "value": "6" + }, + { + "key": "DISTRIBUTEDSHELLSCRIPTLOCATION", + "value": "hdfs://hdfs-namenode:9000/user/testuser/demo-app/shellCommands" } ] } @@ -2302,16 +2282,9 @@ _01_000001 "vCores":"1" }, "application-type":"YARN", - "keep-containers-across-application-attempts":"false", - "application-tags": - { - "tag": - [ - "tag 2", - "tag1" - ] - } + "keep-containers-across-application-attempts":"false" } + +---+ Response Header: @@ -2349,22 +2322,34 @@ _01_000001 example - http://www.test.com/file.txt + hdfs://hdfs-namenode:9000/user/testuser/DistributedShell/demo-app/AppMaster.jar FILE APPLICATION - 100 - 1404204892877 + 43004 + 1405452071209 - APP_VAR - ENV_SETTING + DISTRIBUTEDSHELLSCRIPTTIMESTAMP + 1405459400754 + + + CLASSPATH + {{CLASSPATH}}<CPS>./*<CPS>{{HADOOP_CONF_DIR}}<CPS>{{HADOOP_COMMON_HOME}}/share/hadoop/common/*<CPS>{{HADOOP_COMMON_HOME}}/share/hadoop/common/lib/*<CPS>{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/*<CPS>{{HADOOP_HDFS_HOME}}/share/hadoop/hdfs/lib/*<CPS>{{HADOOP_YARN_HOME}}/share/hadoop/yarn/*<CPS>{{HADOOP_YARN_HOME}}/share/hadoop/yarn/lib/*<CPS>./log4j.properties + + + DISTRIBUTEDSHELLSCRIPTLEN + 6 + + + DISTRIBUTEDSHELLSCRIPTLOCATION + hdfs://hdfs-namenode:9000/user/testuser/demo-app/shellCommands - /bin/sleep 5 + {{JAVA_HOME}}/bin/java -Xmx10m org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster --container_memory 10 --container_vcores 1 --num_containers 1 --priority 0 1><LOG_DIR>/AppMaster.stdout 2><LOG_DIR>/AppMaster.stderr From 2050e0dad661bade3e140d7a5692cfe1999badc3 Mon Sep 17 00:00:00 2001 From: Devarajulu K Date: Thu, 24 Jul 2014 05:02:00 +0000 Subject: [PATCH 11/37] YARN-1342. Recover container tokens upon nodemanager restart. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1612995 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../BaseContainerTokenSecretManager.java | 2 +- .../yarn/server/nodemanager/NodeManager.java | 6 +- .../recovery/NMLeveldbStateStoreService.java | 117 ++++++++++++-- .../recovery/NMNullStateStoreService.java | 30 +++- .../recovery/NMStateStoreService.java | 40 ++++- .../NMContainerTokenSecretManager.java | 105 ++++++++++--- .../security/NMTokenSecretManagerInNM.java | 5 +- .../recovery/NMMemoryStateStoreService.java | 55 ++++++- .../TestNMLeveldbStateStoreService.java | 96 +++++++++++- .../TestNMContainerTokenSecretManager.java | 144 ++++++++++++++++++ .../TestNMTokenSecretManagerInNM.java | 8 +- 12 files changed, 557 insertions(+), 54 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6857d6c3747..1712abeb9e4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -62,6 +62,9 @@ Release 2.6.0 - UNRELEASED YARN-2295. Refactored DistributedShell to use public APIs of protocol records. (Li Lu via jianhe) + YARN-1342. Recover container tokens upon nodemanager restart. (Jason Lowe via + devaraj) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java index ccfe8f59fd1..e73d07c26c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java @@ -43,7 +43,7 @@ public class BaseContainerTokenSecretManager extends private static Log LOG = LogFactory .getLog(BaseContainerTokenSecretManager.class); - private int serialNo = new SecureRandom().nextInt(); + protected int serialNo = new SecureRandom().nextInt(); protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); protected final Lock readLock = readWriteLock.readLock(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 65988a211d6..a479be29f73 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -173,8 +173,8 @@ public class NodeManager extends CompositeService NMContainerTokenSecretManager containerTokenSecretManager) throws IOException { if (nmStore.canRecover()) { - nmTokenSecretManager.recover(nmStore.loadNMTokenState()); - // TODO: recover containerTokenSecretManager + nmTokenSecretManager.recover(); + containerTokenSecretManager.recover(); } } @@ -190,7 +190,7 @@ public class NodeManager extends CompositeService initAndStartRecoveryStore(conf); NMContainerTokenSecretManager containerTokenSecretManager = - new NMContainerTokenSecretManager(conf); + new NMContainerTokenSecretManager(conf, nmStore); NMTokenSecretManagerInNM nmTokenSecretManager = new NMTokenSecretManagerInNM(nmStore); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java index b905c1e5ad8..008da7a2b8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; @@ -90,6 +91,12 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { NM_TOKENS_KEY_PREFIX + CURRENT_MASTER_KEY_SUFFIX; private static final String NM_TOKENS_PREV_MASTER_KEY = NM_TOKENS_KEY_PREFIX + PREV_MASTER_KEY_SUFFIX; + private static final String CONTAINER_TOKENS_KEY_PREFIX = + "ContainerTokens/"; + private static final String CONTAINER_TOKENS_CURRENT_MASTER_KEY = + CONTAINER_TOKENS_KEY_PREFIX + CURRENT_MASTER_KEY_SUFFIX; + private static final String CONTAINER_TOKENS_PREV_MASTER_KEY = + CONTAINER_TOKENS_KEY_PREFIX + PREV_MASTER_KEY_SUFFIX; private DB db; @@ -141,7 +148,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { key.substring(0, userEndPos+1))); } } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } finally { if (iter != null) { iter.close(); @@ -260,7 +267,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { try { db.put(bytes(key), proto.toByteArray()); } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } @@ -283,7 +290,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { batch.close(); } } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } @@ -306,7 +313,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { batch.close(); } } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } @@ -355,7 +362,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { DeletionServiceDeleteTaskProto.parseFrom(entry.getValue())); } } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } finally { if (iter != null) { iter.close(); @@ -371,7 +378,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { try { db.put(bytes(key), taskProto.toByteArray()); } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } @@ -381,14 +388,14 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { try { db.delete(bytes(key)); } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } @Override - public RecoveredNMTokenState loadNMTokenState() throws IOException { - RecoveredNMTokenState state = new RecoveredNMTokenState(); + public RecoveredNMTokensState loadNMTokensState() throws IOException { + RecoveredNMTokensState state = new RecoveredNMTokensState(); state.applicationMasterKeys = new HashMap(); LeveldbIterator iter = null; @@ -420,7 +427,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { } } } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } finally { if (iter != null) { iter.close(); @@ -454,7 +461,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { try { db.delete(bytes(key)); } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } @@ -468,7 +475,91 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { try { db.put(bytes(dbKey), pb.getProto().toByteArray()); } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); + } + } + + + @Override + public RecoveredContainerTokensState loadContainerTokensState() + throws IOException { + RecoveredContainerTokensState state = new RecoveredContainerTokensState(); + state.activeTokens = new HashMap(); + LeveldbIterator iter = null; + try { + iter = new LeveldbIterator(db); + iter.seek(bytes(CONTAINER_TOKENS_KEY_PREFIX)); + final int containerTokensKeyPrefixLength = + CONTAINER_TOKENS_KEY_PREFIX.length(); + while (iter.hasNext()) { + Entry entry = iter.next(); + String fullKey = asString(entry.getKey()); + if (!fullKey.startsWith(CONTAINER_TOKENS_KEY_PREFIX)) { + break; + } + String key = fullKey.substring(containerTokensKeyPrefixLength); + if (key.equals(CURRENT_MASTER_KEY_SUFFIX)) { + state.currentMasterKey = parseMasterKey(entry.getValue()); + } else if (key.equals(PREV_MASTER_KEY_SUFFIX)) { + state.previousMasterKey = parseMasterKey(entry.getValue()); + } else if (key.startsWith(ConverterUtils.CONTAINER_PREFIX)) { + loadContainerToken(state, fullKey, key, entry.getValue()); + } + } + } catch (DBException e) { + throw new IOException(e); + } finally { + if (iter != null) { + iter.close(); + } + } + return state; + } + + private static void loadContainerToken(RecoveredContainerTokensState state, + String key, String containerIdStr, byte[] value) throws IOException { + ContainerId containerId; + Long expTime; + try { + containerId = ConverterUtils.toContainerId(containerIdStr); + expTime = Long.parseLong(asString(value)); + } catch (IllegalArgumentException e) { + throw new IOException("Bad container token state for " + key, e); + } + state.activeTokens.put(containerId, expTime); + } + + @Override + public void storeContainerTokenCurrentMasterKey(MasterKey key) + throws IOException { + storeMasterKey(CONTAINER_TOKENS_CURRENT_MASTER_KEY, key); + } + + @Override + public void storeContainerTokenPreviousMasterKey(MasterKey key) + throws IOException { + storeMasterKey(CONTAINER_TOKENS_PREV_MASTER_KEY, key); + } + + @Override + public void storeContainerToken(ContainerId containerId, Long expTime) + throws IOException { + String key = CONTAINER_TOKENS_KEY_PREFIX + containerId; + try { + db.put(bytes(key), bytes(expTime.toString())); + } catch (DBException e) { + throw new IOException(e); + } + } + + @Override + public void removeContainerToken(ContainerId containerId) + throws IOException { + String key = CONTAINER_TOKENS_KEY_PREFIX + containerId; + try { + db.delete(bytes(key)); + } catch (DBException e) { + throw new IOException(e); } } @@ -554,7 +645,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService { try { db.put(bytes(key), data); } catch (DBException e) { - throw new IOException(e.getMessage(), e); + throw new IOException(e); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java index 5d9e0ea15a8..89205b1f637 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; @@ -80,7 +81,7 @@ public class NMNullStateStoreService extends NMStateStoreService { } @Override - public RecoveredNMTokenState loadNMTokenState() throws IOException { + public RecoveredNMTokensState loadNMTokensState() throws IOException { throw new UnsupportedOperationException( "Recovery not supported by this state store"); } @@ -105,6 +106,33 @@ public class NMNullStateStoreService extends NMStateStoreService { throws IOException { } + @Override + public RecoveredContainerTokensState loadContainerTokensState() + throws IOException { + throw new UnsupportedOperationException( + "Recovery not supported by this state store"); + } + + @Override + public void storeContainerTokenCurrentMasterKey(MasterKey key) + throws IOException { + } + + @Override + public void storeContainerTokenPreviousMasterKey(MasterKey key) + throws IOException { + } + + @Override + public void storeContainerToken(ContainerId containerId, + Long expirationTime) throws IOException { + } + + @Override + public void removeContainerToken(ContainerId containerId) + throws IOException { + } + @Override protected void initStorage(Configuration conf) throws IOException { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java index 8a5944dbd14..87c438b59bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; @@ -102,7 +103,7 @@ public abstract class NMStateStoreService extends AbstractService { } } - public static class RecoveredNMTokenState { + public static class RecoveredNMTokensState { MasterKey currentMasterKey; MasterKey previousMasterKey; Map applicationMasterKeys; @@ -120,6 +121,24 @@ public abstract class NMStateStoreService extends AbstractService { } } + public static class RecoveredContainerTokensState { + MasterKey currentMasterKey; + MasterKey previousMasterKey; + Map activeTokens; + + public MasterKey getCurrentMasterKey() { + return currentMasterKey; + } + + public MasterKey getPreviousMasterKey() { + return previousMasterKey; + } + + public Map getActiveTokens() { + return activeTokens; + } + } + /** Initialize the state storage */ @Override public void serviceInit(Configuration conf) throws IOException { @@ -193,7 +212,8 @@ public abstract class NMStateStoreService extends AbstractService { public abstract void removeDeletionTask(int taskId) throws IOException; - public abstract RecoveredNMTokenState loadNMTokenState() throws IOException; + public abstract RecoveredNMTokensState loadNMTokensState() + throws IOException; public abstract void storeNMTokenCurrentMasterKey(MasterKey key) throws IOException; @@ -208,6 +228,22 @@ public abstract class NMStateStoreService extends AbstractService { ApplicationAttemptId attempt) throws IOException; + public abstract RecoveredContainerTokensState loadContainerTokensState() + throws IOException; + + public abstract void storeContainerTokenCurrentMasterKey(MasterKey key) + throws IOException; + + public abstract void storeContainerTokenPreviousMasterKey(MasterKey key) + throws IOException; + + public abstract void storeContainerToken(ContainerId containerId, + Long expirationTime) throws IOException; + + public abstract void removeContainerToken(ContainerId containerId) + throws IOException; + + protected abstract void initStorage(Configuration conf) throws IOException; protected abstract void startStorage() throws IOException; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java index 8860a952521..2a92d40ad5c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager.security; +import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; @@ -33,6 +34,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerTokensState; import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.MasterKeyData; @@ -49,14 +53,74 @@ public class NMContainerTokenSecretManager extends private MasterKeyData previousMasterKey; private final TreeMap> recentlyStartedContainerTracker; - + private final NMStateStoreService stateStore; private String nodeHostAddr; public NMContainerTokenSecretManager(Configuration conf) { + this(conf, new NMNullStateStoreService()); + } + + public NMContainerTokenSecretManager(Configuration conf, + NMStateStoreService stateStore) { super(conf); recentlyStartedContainerTracker = new TreeMap>(); + this.stateStore = stateStore; + } + + public synchronized void recover() + throws IOException { + RecoveredContainerTokensState state = + stateStore.loadContainerTokensState(); + MasterKey key = state.getCurrentMasterKey(); + if (key != null) { + super.currentMasterKey = + new MasterKeyData(key, createSecretKey(key.getBytes().array())); + } + + key = state.getPreviousMasterKey(); + if (key != null) { + previousMasterKey = + new MasterKeyData(key, createSecretKey(key.getBytes().array())); + } + + // restore the serial number from the current master key + if (super.currentMasterKey != null) { + super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1; + } + + for (Entry entry : state.getActiveTokens().entrySet()) { + ContainerId containerId = entry.getKey(); + Long expTime = entry.getValue(); + List containerList = + recentlyStartedContainerTracker.get(expTime); + if (containerList == null) { + containerList = new ArrayList(); + recentlyStartedContainerTracker.put(expTime, containerList); + } + if (!containerList.contains(containerId)) { + containerList.add(containerId); + } + } + } + + private void updateCurrentMasterKey(MasterKeyData key) { + super.currentMasterKey = key; + try { + stateStore.storeContainerTokenCurrentMasterKey(key.getMasterKey()); + } catch (IOException e) { + LOG.error("Unable to update current master key in state store", e); + } + } + + private void updatePreviousMasterKey(MasterKeyData key) { + previousMasterKey = key; + try { + stateStore.storeContainerTokenPreviousMasterKey(key.getMasterKey()); + } catch (IOException e) { + LOG.error("Unable to update previous master key in state store", e); + } } /** @@ -68,21 +132,16 @@ public class NMContainerTokenSecretManager extends */ @Private public synchronized void setMasterKey(MasterKey masterKeyRecord) { - LOG.info("Rolling master-key for container-tokens, got key with id " - + masterKeyRecord.getKeyId()); - if (super.currentMasterKey == null) { - super.currentMasterKey = - new MasterKeyData(masterKeyRecord, createSecretKey(masterKeyRecord - .getBytes().array())); - } else { - if (super.currentMasterKey.getMasterKey().getKeyId() != masterKeyRecord - .getKeyId()) { - // Update keys only if the key has changed. - this.previousMasterKey = super.currentMasterKey; - super.currentMasterKey = - new MasterKeyData(masterKeyRecord, createSecretKey(masterKeyRecord - .getBytes().array())); + // Update keys only if the key has changed. + if (super.currentMasterKey == null || super.currentMasterKey.getMasterKey() + .getKeyId() != masterKeyRecord.getKeyId()) { + LOG.info("Rolling master-key for container-tokens, got key with id " + + masterKeyRecord.getKeyId()); + if (super.currentMasterKey != null) { + updatePreviousMasterKey(super.currentMasterKey); } + updateCurrentMasterKey(new MasterKeyData(masterKeyRecord, + createSecretKey(masterKeyRecord.getBytes().array()))); } } @@ -137,14 +196,19 @@ public class NMContainerTokenSecretManager extends removeAnyContainerTokenIfExpired(); + ContainerId containerId = tokenId.getContainerID(); Long expTime = tokenId.getExpiryTimeStamp(); // We might have multiple containers with same expiration time. if (!recentlyStartedContainerTracker.containsKey(expTime)) { recentlyStartedContainerTracker .put(expTime, new ArrayList()); } - recentlyStartedContainerTracker.get(expTime).add(tokenId.getContainerID()); - + recentlyStartedContainerTracker.get(expTime).add(containerId); + try { + stateStore.storeContainerToken(containerId, expTime); + } catch (IOException e) { + LOG.error("Unable to store token for container " + containerId, e); + } } protected synchronized void removeAnyContainerTokenIfExpired() { @@ -155,6 +219,13 @@ public class NMContainerTokenSecretManager extends while (containersI.hasNext()) { Entry> containerEntry = containersI.next(); if (containerEntry.getKey() < currTime) { + for (ContainerId container : containerEntry.getValue()) { + try { + stateStore.removeContainerToken(container); + } catch (IOException e) { + LOG.error("Unable to remove token for container " + container, e); + } + } containersI.remove(); } else { break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java index a9b9b994add..f6169e70265 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java @@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.security.NMTokenIdentifier; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; -import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokenState; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokensState; import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager; import org.apache.hadoop.yarn.server.security.MasterKeyData; @@ -64,8 +64,9 @@ public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager { this.stateStore = stateStore; } - public synchronized void recover(RecoveredNMTokenState state) + public synchronized void recover() throws IOException { + RecoveredNMTokensState state = stateStore.loadNMTokensState(); MasterKey key = state.getCurrentMasterKey(); if (key != null) { super.currentMasterKey = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java index 9909d9db9e5..fef2b12221f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; @@ -36,7 +37,8 @@ import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; public class NMMemoryStateStoreService extends NMStateStoreService { private Map trackerStates; private Map deleteTasks; - private RecoveredNMTokenState nmTokenState; + private RecoveredNMTokensState nmTokenState; + private RecoveredContainerTokensState containerTokenState; public NMMemoryStateStoreService() { super(NMMemoryStateStoreService.class.getName()); @@ -117,12 +119,13 @@ public class NMMemoryStateStoreService extends NMStateStoreService { @Override protected void initStorage(Configuration conf) { - nmTokenState = new RecoveredNMTokenState(); + nmTokenState = new RecoveredNMTokensState(); nmTokenState.applicationMasterKeys = new HashMap(); + containerTokenState = new RecoveredContainerTokensState(); + containerTokenState.activeTokens = new HashMap(); trackerStates = new HashMap(); deleteTasks = new HashMap(); - } @Override @@ -157,9 +160,9 @@ public class NMMemoryStateStoreService extends NMStateStoreService { @Override - public RecoveredNMTokenState loadNMTokenState() throws IOException { + public RecoveredNMTokensState loadNMTokensState() throws IOException { // return a copy so caller can't modify our state - RecoveredNMTokenState result = new RecoveredNMTokenState(); + RecoveredNMTokensState result = new RecoveredNMTokensState(); result.currentMasterKey = nmTokenState.currentMasterKey; result.previousMasterKey = nmTokenState.previousMasterKey; result.applicationMasterKeys = @@ -197,6 +200,48 @@ public class NMMemoryStateStoreService extends NMStateStoreService { } + @Override + public RecoveredContainerTokensState loadContainerTokensState() + throws IOException { + // return a copy so caller can't modify our state + RecoveredContainerTokensState result = + new RecoveredContainerTokensState(); + result.currentMasterKey = containerTokenState.currentMasterKey; + result.previousMasterKey = containerTokenState.previousMasterKey; + result.activeTokens = + new HashMap(containerTokenState.activeTokens); + return result; + } + + @Override + public void storeContainerTokenCurrentMasterKey(MasterKey key) + throws IOException { + MasterKeyPBImpl keypb = (MasterKeyPBImpl) key; + containerTokenState.currentMasterKey = + new MasterKeyPBImpl(keypb.getProto()); + } + + @Override + public void storeContainerTokenPreviousMasterKey(MasterKey key) + throws IOException { + MasterKeyPBImpl keypb = (MasterKeyPBImpl) key; + containerTokenState.previousMasterKey = + new MasterKeyPBImpl(keypb.getProto()); + } + + @Override + public void storeContainerToken(ContainerId containerId, + Long expirationTime) throws IOException { + containerTokenState.activeTokens.put(containerId, expirationTime); + } + + @Override + public void removeContainerToken(ContainerId containerId) + throws IOException { + containerTokenState.activeTokens.remove(containerId); + } + + private static class TrackerState { Map inProgressMap = new HashMap(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java index ca17a4e6e8a..833a062d3b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java @@ -27,11 +27,13 @@ import java.io.File; import java.io.IOException; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.ServiceStateException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -42,12 +44,15 @@ import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.Deletion import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LocalizedResourceProto; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerTokensState; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredDeletionServiceState; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredLocalizationState; -import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokenState; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredNMTokensState; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources; import org.apache.hadoop.yarn.server.nodemanager.recovery.records.NMDBSchemaVersion; +import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.After; import org.junit.Assert; @@ -502,7 +507,7 @@ public class TestNMLeveldbStateStoreService { @Test public void testNMTokenStorage() throws IOException { // test empty when no state - RecoveredNMTokenState state = stateStore.loadNMTokenState(); + RecoveredNMTokensState state = stateStore.loadNMTokensState(); assertNull(state.getCurrentMasterKey()); assertNull(state.getPreviousMasterKey()); assertTrue(state.getApplicationMasterKeys().isEmpty()); @@ -512,7 +517,7 @@ public class TestNMLeveldbStateStoreService { MasterKey currentKey = secretMgr.generateKey(); stateStore.storeNMTokenCurrentMasterKey(currentKey); restartStateStore(); - state = stateStore.loadNMTokenState(); + state = stateStore.loadNMTokensState(); assertEquals(currentKey, state.getCurrentMasterKey()); assertNull(state.getPreviousMasterKey()); assertTrue(state.getApplicationMasterKeys().isEmpty()); @@ -521,7 +526,7 @@ public class TestNMLeveldbStateStoreService { MasterKey prevKey = secretMgr.generateKey(); stateStore.storeNMTokenPreviousMasterKey(prevKey); restartStateStore(); - state = stateStore.loadNMTokenState(); + state = stateStore.loadNMTokensState(); assertEquals(currentKey, state.getCurrentMasterKey()); assertEquals(prevKey, state.getPreviousMasterKey()); assertTrue(state.getApplicationMasterKeys().isEmpty()); @@ -536,7 +541,7 @@ public class TestNMLeveldbStateStoreService { MasterKey attemptKey2 = secretMgr.generateKey(); stateStore.storeNMTokenApplicationMasterKey(attempt2, attemptKey2); restartStateStore(); - state = stateStore.loadNMTokenState(); + state = stateStore.loadNMTokensState(); assertEquals(currentKey, state.getCurrentMasterKey()); assertEquals(prevKey, state.getPreviousMasterKey()); Map loadedAppKeys = @@ -558,7 +563,7 @@ public class TestNMLeveldbStateStoreService { currentKey = secretMgr.generateKey(); stateStore.storeNMTokenCurrentMasterKey(currentKey); restartStateStore(); - state = stateStore.loadNMTokenState(); + state = stateStore.loadNMTokensState(); assertEquals(currentKey, state.getCurrentMasterKey()); assertEquals(prevKey, state.getPreviousMasterKey()); loadedAppKeys = state.getApplicationMasterKeys(); @@ -568,10 +573,89 @@ public class TestNMLeveldbStateStoreService { assertEquals(attemptKey3, loadedAppKeys.get(attempt3)); } + @Test + public void testContainerTokenStorage() throws IOException { + // test empty when no state + RecoveredContainerTokensState state = + stateStore.loadContainerTokensState(); + assertNull(state.getCurrentMasterKey()); + assertNull(state.getPreviousMasterKey()); + assertTrue(state.getActiveTokens().isEmpty()); + + // store a master key and verify recovered + ContainerTokenKeyGeneratorForTest keygen = + new ContainerTokenKeyGeneratorForTest(new YarnConfiguration()); + MasterKey currentKey = keygen.generateKey(); + stateStore.storeContainerTokenCurrentMasterKey(currentKey); + restartStateStore(); + state = stateStore.loadContainerTokensState(); + assertEquals(currentKey, state.getCurrentMasterKey()); + assertNull(state.getPreviousMasterKey()); + assertTrue(state.getActiveTokens().isEmpty()); + + // store a previous key and verify recovered + MasterKey prevKey = keygen.generateKey(); + stateStore.storeContainerTokenPreviousMasterKey(prevKey); + restartStateStore(); + state = stateStore.loadContainerTokensState(); + assertEquals(currentKey, state.getCurrentMasterKey()); + assertEquals(prevKey, state.getPreviousMasterKey()); + assertTrue(state.getActiveTokens().isEmpty()); + + // store a few container tokens and verify recovered + ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1); + Long expTime1 = 1234567890L; + ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2); + Long expTime2 = 9876543210L; + stateStore.storeContainerToken(cid1, expTime1); + stateStore.storeContainerToken(cid2, expTime2); + restartStateStore(); + state = stateStore.loadContainerTokensState(); + assertEquals(currentKey, state.getCurrentMasterKey()); + assertEquals(prevKey, state.getPreviousMasterKey()); + Map loadedActiveTokens = + state.getActiveTokens(); + assertEquals(2, loadedActiveTokens.size()); + assertEquals(expTime1, loadedActiveTokens.get(cid1)); + assertEquals(expTime2, loadedActiveTokens.get(cid2)); + + // add/update/remove tokens and verify recovered + ContainerId cid3 = BuilderUtils.newContainerId(3, 3, 3, 3); + Long expTime3 = 135798642L; + stateStore.storeContainerToken(cid3, expTime3); + stateStore.removeContainerToken(cid1); + expTime2 += 246897531L; + stateStore.storeContainerToken(cid2, expTime2); + prevKey = currentKey; + stateStore.storeContainerTokenPreviousMasterKey(prevKey); + currentKey = keygen.generateKey(); + stateStore.storeContainerTokenCurrentMasterKey(currentKey); + restartStateStore(); + state = stateStore.loadContainerTokensState(); + assertEquals(currentKey, state.getCurrentMasterKey()); + assertEquals(prevKey, state.getPreviousMasterKey()); + loadedActiveTokens = state.getActiveTokens(); + assertEquals(2, loadedActiveTokens.size()); + assertNull(loadedActiveTokens.get(cid1)); + assertEquals(expTime2, loadedActiveTokens.get(cid2)); + assertEquals(expTime3, loadedActiveTokens.get(cid3)); + } + private static class NMTokenSecretManagerForTest extends BaseNMTokenSecretManager { public MasterKey generateKey() { return createNewMasterKey().getMasterKey(); } } + + private static class ContainerTokenKeyGeneratorForTest extends + BaseContainerTokenSecretManager { + public ContainerTokenKeyGeneratorForTest(Configuration conf) { + super(conf); + } + + public MasterKey generateKey() { + return createNewMasterKey().getMasterKey(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java new file mode 100644 index 00000000000..f2a46adaf8a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMContainerTokenSecretManager.java @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.security; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.SecretManager.InvalidToken; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService; +import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.junit.Test; + +public class TestNMContainerTokenSecretManager { + + @Test + public void testRecovery() throws IOException { + YarnConfiguration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true); + final NodeId nodeId = NodeId.newInstance("somehost", 1234); + final ContainerId cid1 = BuilderUtils.newContainerId(1, 1, 1, 1); + final ContainerId cid2 = BuilderUtils.newContainerId(2, 2, 2, 2); + ContainerTokenKeyGeneratorForTest keygen = + new ContainerTokenKeyGeneratorForTest(conf); + NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService(); + stateStore.init(conf); + stateStore.start(); + NMContainerTokenSecretManager secretMgr = + new NMContainerTokenSecretManager(conf, stateStore); + secretMgr.setNodeId(nodeId); + MasterKey currentKey = keygen.generateKey(); + secretMgr.setMasterKey(currentKey); + ContainerTokenIdentifier tokenId1 = + createContainerTokenId(cid1, nodeId, "user1", secretMgr); + ContainerTokenIdentifier tokenId2 = + createContainerTokenId(cid2, nodeId, "user2", secretMgr); + assertNotNull(secretMgr.retrievePassword(tokenId1)); + assertNotNull(secretMgr.retrievePassword(tokenId2)); + + // restart and verify tokens still valid + secretMgr = new NMContainerTokenSecretManager(conf, stateStore); + secretMgr.setNodeId(nodeId); + secretMgr.recover(); + assertEquals(currentKey, secretMgr.getCurrentKey()); + assertTrue(secretMgr.isValidStartContainerRequest(tokenId1)); + assertTrue(secretMgr.isValidStartContainerRequest(tokenId2)); + assertNotNull(secretMgr.retrievePassword(tokenId1)); + assertNotNull(secretMgr.retrievePassword(tokenId2)); + + // roll master key and start a container + secretMgr.startContainerSuccessful(tokenId2); + currentKey = keygen.generateKey(); + secretMgr.setMasterKey(currentKey); + + // restart and verify tokens still valid due to prev key persist + secretMgr = new NMContainerTokenSecretManager(conf, stateStore); + secretMgr.setNodeId(nodeId); + secretMgr.recover(); + assertEquals(currentKey, secretMgr.getCurrentKey()); + assertTrue(secretMgr.isValidStartContainerRequest(tokenId1)); + assertFalse(secretMgr.isValidStartContainerRequest(tokenId2)); + assertNotNull(secretMgr.retrievePassword(tokenId1)); + assertNotNull(secretMgr.retrievePassword(tokenId2)); + + // roll master key again, restart, and verify keys no longer valid + currentKey = keygen.generateKey(); + secretMgr.setMasterKey(currentKey); + secretMgr = new NMContainerTokenSecretManager(conf, stateStore); + secretMgr.setNodeId(nodeId); + secretMgr.recover(); + assertEquals(currentKey, secretMgr.getCurrentKey()); + assertTrue(secretMgr.isValidStartContainerRequest(tokenId1)); + assertFalse(secretMgr.isValidStartContainerRequest(tokenId2)); + try { + secretMgr.retrievePassword(tokenId1); + fail("token should not be valid"); + } catch (InvalidToken e) { + // expected + } + try { + secretMgr.retrievePassword(tokenId2); + fail("token should not be valid"); + } catch (InvalidToken e) { + // expected + } + + stateStore.close(); + } + + private static ContainerTokenIdentifier createContainerTokenId( + ContainerId cid, NodeId nodeId, String user, + NMContainerTokenSecretManager secretMgr) throws IOException { + long rmid = cid.getApplicationAttemptId().getApplicationId() + .getClusterTimestamp(); + ContainerTokenIdentifier ctid = new ContainerTokenIdentifier(cid, + nodeId.toString(), user, BuilderUtils.newResource(1024, 1), + System.currentTimeMillis() + 100000L, + secretMgr.getCurrentKey().getKeyId(), rmid, + Priority.newInstance(0), 0); + Token token = BuilderUtils.newContainerToken(nodeId, + secretMgr.createPassword(ctid), ctid); + return BuilderUtils.newContainerTokenIdentifier(token); + } + + private static class ContainerTokenKeyGeneratorForTest extends + BaseContainerTokenSecretManager { + public ContainerTokenKeyGeneratorForTest(Configuration conf) { + super(conf); + } + + public MasterKey generateKey() { + return createNewMasterKey().getMasterKey(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java index 1f1fc51e568..5c1f3a1c310 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/TestNMTokenSecretManagerInNM.java @@ -73,7 +73,7 @@ public class TestNMTokenSecretManagerInNM { // restart and verify key is still there and token still valid secretMgr = new NMTokenSecretManagerInNM(stateStore); - secretMgr.recover(stateStore.loadNMTokenState()); + secretMgr.recover(); secretMgr.setNodeId(nodeId); assertEquals(currentKey, secretMgr.getCurrentKey()); assertTrue(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1)); @@ -88,7 +88,7 @@ public class TestNMTokenSecretManagerInNM { // restart and verify attempt1 key is still valid due to prev key persist secretMgr = new NMTokenSecretManagerInNM(stateStore); - secretMgr.recover(stateStore.loadNMTokenState()); + secretMgr.recover(); secretMgr.setNodeId(nodeId); assertEquals(currentKey, secretMgr.getCurrentKey()); assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1)); @@ -101,7 +101,7 @@ public class TestNMTokenSecretManagerInNM { currentKey = keygen.generateKey(); secretMgr.setMasterKey(currentKey); secretMgr = new NMTokenSecretManagerInNM(stateStore); - secretMgr.recover(stateStore.loadNMTokenState()); + secretMgr.recover(); secretMgr.setNodeId(nodeId); assertEquals(currentKey, secretMgr.getCurrentKey()); assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1)); @@ -117,7 +117,7 @@ public class TestNMTokenSecretManagerInNM { // remove last attempt, restart, verify both tokens are now bad secretMgr.appFinished(attempt2.getApplicationId()); secretMgr = new NMTokenSecretManagerInNM(stateStore); - secretMgr.recover(stateStore.loadNMTokenState()); + secretMgr.recover(); secretMgr.setNodeId(nodeId); assertEquals(currentKey, secretMgr.getCurrentKey()); assertFalse(secretMgr.isAppAttemptNMTokenKeyPresent(attempt1)); From 2054453a39efeca86361e26033a65f2715f4785c Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 24 Jul 2014 06:22:02 +0000 Subject: [PATCH 12/37] HADOOP-10882. Move DirectBufferPool into common util. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613006 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../main/java/org/apache/hadoop}/util/DirectBufferPool.java | 6 ++++-- .../java/org/apache/hadoop}/util/TestDirectBufferPool.java | 4 ++-- .../main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java | 2 +- .../java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java | 2 +- .../hadoop/hdfs/protocol/datatransfer/PacketReceiver.java | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) rename {hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs => hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop}/util/DirectBufferPool.java (95%) rename {hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs => hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop}/util/TestDirectBufferPool.java (95%) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 08bc9012e9e..491daf04e9f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -455,6 +455,8 @@ Release 2.6.0 - UNRELEASED HADOOP-10887. Add XAttrs to ViewFs and make XAttrs + ViewFileSystem internal dir behavior consistent. (Stephen Chu via wang) + HADOOP-10882. Move DirectBufferPool into common util. (todd) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DirectBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java similarity index 95% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DirectBufferPool.java rename to hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java index 7332d34594e..510938b7fff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/DirectBufferPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DirectBufferPool.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.util; +package org.apache.hadoop.util; import java.lang.ref.WeakReference; import java.nio.ByteBuffer; @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.classification.InterfaceAudience; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.classification.InterfaceStability; /** * A simple class for pooling direct ByteBuffers. This is necessary @@ -40,7 +41,8 @@ import com.google.common.annotations.VisibleForTesting; * allocated at the same size. There is no attempt to reuse larger * buffers to satisfy smaller allocations. */ -@InterfaceAudience.Private +@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) +@InterfaceStability.Evolving public class DirectBufferPool { // Essentially implement a multimap with weak values. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestDirectBufferPool.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java similarity index 95% rename from hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestDirectBufferPool.java rename to hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java index 31a18fb8815..c8fd754666c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestDirectBufferPool.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDirectBufferPool.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.hdfs.util; +package org.apache.hadoop.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotSame; @@ -29,7 +29,7 @@ import org.junit.Test; import com.google.common.collect.Lists; public class TestDirectBufferPool { - final DirectBufferPool pool = new DirectBufferPool(); + final org.apache.hadoop.util.DirectBufferPool pool = new org.apache.hadoop.util.DirectBufferPool(); @Test public void testBasics() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java index bb9612a9956..cd75e53b273 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.shortcircuit.ClientMmap; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitReplica; -import org.apache.hadoop.hdfs.util.DirectBufferPool; +import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.util.DataChecksum; import com.google.common.annotations.VisibleForTesting; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java index c68e548099b..47455754d72 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.server.datanode.BlockMetadataHeader; import org.apache.hadoop.hdfs.shortcircuit.ClientMmap; -import org.apache.hadoop.hdfs.util.DirectBufferPool; +import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.security.UserGroupInformation; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java index 3503554636a..0de445c222d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/PacketReceiver.java @@ -27,7 +27,7 @@ import java.nio.channels.ReadableByteChannel; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.util.DirectBufferPool; +import org.apache.hadoop.util.DirectBufferPool; import org.apache.hadoop.io.IOUtils; import com.google.common.base.Preconditions; From ef9b6a45c437a56f9ebf198cba902e06e875f27c Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Thu, 24 Jul 2014 17:28:31 +0000 Subject: [PATCH 13/37] HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI. Contributed by Vinayakumar B. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613195 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/main/webapps/hdfs/dfshealth.html | 1 - .../hadoop-hdfs/src/main/webapps/hdfs/index.html | 13 +------------ .../src/main/webapps/secondary/index.html | 11 ----------- 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bbeebd19843..217bf4df094 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -252,6 +252,9 @@ Trunk (Unreleased) HDFS-5794. Fix the inconsistency of layout version number of ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9) + HDFS-6657. Remove link to 'Legacy UI' in trunk's Namenode UI. + (Vinayakumar B via wheat 9) + Release 2.6.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index fadba070721..8fdf73ba19c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -66,7 +66,6 @@

Hadoop, 2014.

-
Legacy UI
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html index 99bb13b326c..aa62a372396 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/index.html @@ -18,18 +18,7 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> - + Hadoop Administration - - -

Hadoop Administration

- - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html index 97e0207e06f..f7ef858b9e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/index.html @@ -21,15 +21,4 @@ Hadoop Administration - - -

Hadoop Administration

- - \ No newline at end of file From f2137d7c0e19176d5ad7e28c6abcfc03eac49ec3 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Thu, 24 Jul 2014 17:47:08 +0000 Subject: [PATCH 14/37] HADOOP-10894. Fix dead link in ToolRunner documentation. (Contributed by Akira Ajisaka) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613200 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/util/ToolRunner.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 491daf04e9f..5ea931c649e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -819,6 +819,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10890. TestDFVariations.testMount fails intermittently. (Yongjun Zhang via Arpit Agarwal) + HADOOP-10894. Fix dead link in ToolRunner documentation. (Akira Ajisaka + via Arpit Agarwal) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java index 49581000ca3..16872d0891e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ToolRunner.java @@ -30,7 +30,7 @@ import org.apache.hadoop.conf.Configuration; *

ToolRunner can be used to run classes implementing * Tool interface. It works in conjunction with * {@link GenericOptionsParser} to parse the - * + * * generic hadoop command line arguments and modifies the * Configuration of the Tool. The * application-specific options are passed along without being modified. From a7855e1c3376fee23eb2ed61f9ae4ad3c9754722 Mon Sep 17 00:00:00 2001 From: Haohui Mai Date: Thu, 24 Jul 2014 17:59:45 +0000 Subject: [PATCH 15/37] HDFS-6723. New NN webUI no longer displays decommissioned state for dead node. Contributed by Ming Ma. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613220 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 217bf4df094..45109896397 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -983,6 +983,9 @@ Release 2.5.0 - UNRELEASED HDFS-6622. Rename and AddBlock may race and produce invalid edits (kihwal via cmccabe) + HDFS-6723. New NN webUI no longer displays decommissioned state for dead node. + (Ming Ma via wheat9) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 8fdf73ba19c..25895261982 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -282,7 +282,7 @@ {name} ({xferaddr}) {lastContact} - Dead{?decomissioned}, Decomissioned{/decomissioned} + Dead{?decommissioned}, Decommissioned{/decommissioned} - - - From 8c6e172a0ad8f06a4f9b70d61d9f3f7789405815 Mon Sep 17 00:00:00 2001 From: Jing Zhao Date: Thu, 24 Jul 2014 18:28:00 +0000 Subject: [PATCH 16/37] HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException: Namenode is in startup mode. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613237 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../web/resources/NamenodeWebHdfsMethods.java | 3 +- .../apache/hadoop/hdfs/web/TestWebHDFS.java | 39 +++++++++- .../hadoop/hdfs/web/TestWebHDFSForHA.java | 76 +++++++++++++++++-- 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 45109896397..376c272a576 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -367,6 +367,9 @@ Release 2.6.0 - UNRELEASED HDFS-6455. NFS: Exception should be added in NFS log for invalid separator in nfs.exports.allowed.hosts. (Abhiraj Butala via brandonli) + HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException: Namenode + is in startup mode. (jing9) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 92a58f9822e..d7235b38727 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -113,6 +113,7 @@ import org.apache.hadoop.hdfs.web.resources.XAttrNameParam; import org.apache.hadoop.hdfs.web.resources.XAttrSetFlagParam; import org.apache.hadoop.hdfs.web.resources.XAttrValueParam; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException; import org.apache.hadoop.net.Node; @@ -190,7 +191,7 @@ public class NamenodeWebHdfsMethods { throws IOException { final NamenodeProtocols np = namenode.getRpcServer(); if (np == null) { - throw new IOException("Namenode is in startup mode"); + throw new RetriableException("Namenode is in startup mode"); } return np; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java index e9c74c6de30..14312110aa6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java @@ -39,14 +39,18 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.TestDFSClientRetries; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; -import org.apache.hadoop.hdfs.TestDFSClientRetries; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; +import org.apache.hadoop.ipc.RetriableException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; import org.junit.Assert; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; /** Test WebHDFS */ public class TestWebHDFS { @@ -445,4 +449,37 @@ public class TestWebHDFS { } } } + + /** + * Make sure a RetriableException is thrown when rpcServer is null in + * NamenodeWebHdfsMethods. + */ + @Test + public void testRaceWhileNNStartup() throws Exception { + MiniDFSCluster cluster = null; + final Configuration conf = WebHdfsTestUtil.createConf(); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); + cluster.waitActive(); + final NameNode namenode = cluster.getNameNode(); + final NamenodeProtocols rpcServer = namenode.getRpcServer(); + Whitebox.setInternalState(namenode, "rpcServer", null); + + final Path foo = new Path("/foo"); + final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, + WebHdfsFileSystem.SCHEME); + try { + webHdfs.mkdirs(foo); + fail("Expected RetriableException"); + } catch (RetriableException e) { + GenericTestUtils.assertExceptionContains("Namenode is in startup mode", + e); + } + Whitebox.setInternalState(namenode, "rpcServer", rpcServer); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java index 772e367f93c..0340b952259 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFSForHA.java @@ -18,6 +18,15 @@ package org.apache.hadoop.hdfs.web; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.net.URI; +import java.util.HashMap; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -29,18 +38,14 @@ import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; +import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.token.Token; import org.junit.Assert; import org.junit.Test; - -import java.io.IOException; -import java.net.URI; - -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; +import org.mockito.internal.util.reflection.Whitebox; public class TestWebHDFSForHA { private static final String LOGICAL_NAME = "minidfs"; @@ -182,4 +187,61 @@ public class TestWebHDFSForHA { } } } + + /** + * Make sure the WebHdfsFileSystem will retry based on RetriableException when + * rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up. + */ + @Test (timeout=120000) + public void testRetryWhileNNStartup() throws Exception { + final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME); + MiniDFSCluster cluster = null; + final Map resultMap = new HashMap(); + + try { + cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo) + .numDataNodes(0).build(); + HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME); + cluster.waitActive(); + cluster.transitionToActive(0); + + final NameNode namenode = cluster.getNameNode(0); + final NamenodeProtocols rpcServer = namenode.getRpcServer(); + Whitebox.setInternalState(namenode, "rpcServer", null); + + new Thread() { + @Override + public void run() { + boolean result = false; + FileSystem fs = null; + try { + fs = FileSystem.get(WEBHDFS_URI, conf); + final Path dir = new Path("/test"); + result = fs.mkdirs(dir); + } catch (IOException e) { + result = false; + } finally { + IOUtils.cleanup(null, fs); + } + synchronized (TestWebHDFSForHA.this) { + resultMap.put("mkdirs", result); + TestWebHDFSForHA.this.notifyAll(); + } + } + }.start(); + + Thread.sleep(1000); + Whitebox.setInternalState(namenode, "rpcServer", rpcServer); + synchronized (this) { + while (!resultMap.containsKey("mkdirs")) { + this.wait(); + } + Assert.assertTrue(resultMap.get("mkdirs")); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } From e171254d56bfff467a67a6cf9160595c941f50c0 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 24 Jul 2014 23:42:06 +0000 Subject: [PATCH 17/37] Name node cannot start if the path of a file under construction contains .snapshot. (wang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613329 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hdfs/server/namenode/FSImageFormat.java | 11 ++ .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 137 ++++++++++++++++++ .../test/resources/hadoop-0.23-reserved.tgz | Bin 0 -> 4558 bytes .../src/test/resources/hadoop-1-reserved.tgz | Bin 0 -> 2572 bytes 5 files changed, 151 insertions(+) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-0.23-reserved.tgz create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-1-reserved.tgz diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 376c272a576..498454916d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -908,6 +908,9 @@ Release 2.5.0 - UNRELEASED HDFS-6422. getfattr in CLI doesn't throw exception or return non-0 return code when xattr doesn't exist. (Charles Lamb via umamahesh) + HDFS-6696. Name node cannot start if the path of a file under + construction contains ".snapshot". (wang) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java index 49a035cfff0..5b6d269546b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java @@ -614,6 +614,16 @@ public class FSImageFormat { INodeDirectory parentINode = fsDir.rootDir; for (long i = 0; i < numFiles; i++) { pathComponents = FSImageSerialization.readPathComponents(in); + for (int j=0; j < pathComponents.length; j++) { + byte[] newComponent = renameReservedComponentOnUpgrade + (pathComponents[j], getLayoutVersion()); + if (!Arrays.equals(newComponent, pathComponents[j])) { + String oldPath = DFSUtil.byteArray2PathString(pathComponents); + pathComponents[j] = newComponent; + String newPath = DFSUtil.byteArray2PathString(pathComponents); + LOG.info("Renaming reserved path " + oldPath + " to " + newPath); + } + } final INode newNode = loadINode( pathComponents[pathComponents.length-1], false, in, counter); @@ -926,6 +936,7 @@ public class FSImageFormat { oldnode = namesystem.dir.getInode(cons.getId()).asFile(); inSnapshot = true; } else { + path = renameReservedPathsOnUpgrade(path, getLayoutVersion()); final INodesInPath iip = fsDir.getLastINodeInPath(path); oldnode = INodeFile.valueOf(iip.getINode(0), path); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java index 1e1f668f210..f5dbdceaa17 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java @@ -70,6 +70,9 @@ public class TestDFSUpgradeFromImage { private static final String HADOOP_DFS_DIR_TXT = "hadoop-dfs-dir.txt"; private static final String HADOOP22_IMAGE = "hadoop-22-dfs-dir.tgz"; private static final String HADOOP1_BBW_IMAGE = "hadoop1-bbw.tgz"; + private static final String HADOOP1_RESERVED_IMAGE = "hadoop-1-reserved.tgz"; + private static final String HADOOP023_RESERVED_IMAGE = + "hadoop-0.23-reserved.tgz"; private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz"; private static class ReferenceFileInfo { @@ -325,6 +328,140 @@ public class TestDFSUpgradeFromImage { } } + /** + * Test upgrade from a branch-1.2 image with reserved paths + */ + @Test + public void testUpgradeFromRel1ReservedImage() throws Exception { + unpackStorage(HADOOP1_RESERVED_IMAGE); + MiniDFSCluster cluster = null; + // Try it once without setting the upgrade flag to ensure it fails + final Configuration conf = new Configuration(); + // Try it again with a custom rename string + try { + FSImageFormat.setRenameReservedPairs( + ".snapshot=.user-snapshot," + + ".reserved=.my-reserved"); + cluster = + new MiniDFSCluster.Builder(conf) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + // Make sure the paths were renamed as expected + // Also check that paths are present after a restart, checks that the + // upgraded fsimage has the same state. + final String[] expected = new String[] { + "/.my-reserved", + "/.user-snapshot", + "/.user-snapshot/.user-snapshot", + "/.user-snapshot/open", + "/dir1", + "/dir1/.user-snapshot", + "/dir2", + "/dir2/.user-snapshot", + "/user", + "/user/andrew", + "/user/andrew/.user-snapshot", + }; + for (int i=0; i<2; i++) { + // Restart the second time through this loop + if (i==1) { + cluster.finalizeCluster(conf); + cluster.restartNameNode(true); + } + ArrayList toList = new ArrayList(); + toList.add(new Path("/")); + ArrayList found = new ArrayList(); + while (!toList.isEmpty()) { + Path p = toList.remove(0); + FileStatus[] statuses = dfs.listStatus(p); + for (FileStatus status: statuses) { + final String path = status.getPath().toUri().getPath(); + System.out.println("Found path " + path); + found.add(path); + if (status.isDirectory()) { + toList.add(status.getPath()); + } + } + } + for (String s: expected) { + assertTrue("Did not find expected path " + s, found.contains(s)); + } + assertEquals("Found an unexpected path while listing filesystem", + found.size(), expected.length); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + + /** + * Test upgrade from a 0.23.11 image with reserved paths + */ + @Test + public void testUpgradeFromRel023ReservedImage() throws Exception { + unpackStorage(HADOOP023_RESERVED_IMAGE); + MiniDFSCluster cluster = null; + // Try it once without setting the upgrade flag to ensure it fails + final Configuration conf = new Configuration(); + // Try it again with a custom rename string + try { + FSImageFormat.setRenameReservedPairs( + ".snapshot=.user-snapshot," + + ".reserved=.my-reserved"); + cluster = + new MiniDFSCluster.Builder(conf) + .format(false) + .startupOption(StartupOption.UPGRADE) + .numDataNodes(0).build(); + DistributedFileSystem dfs = cluster.getFileSystem(); + // Make sure the paths were renamed as expected + // Also check that paths are present after a restart, checks that the + // upgraded fsimage has the same state. + final String[] expected = new String[] { + "/.user-snapshot", + "/dir1", + "/dir1/.user-snapshot", + "/dir2", + "/dir2/.user-snapshot" + }; + for (int i=0; i<2; i++) { + // Restart the second time through this loop + if (i==1) { + cluster.finalizeCluster(conf); + cluster.restartNameNode(true); + } + ArrayList toList = new ArrayList(); + toList.add(new Path("/")); + ArrayList found = new ArrayList(); + while (!toList.isEmpty()) { + Path p = toList.remove(0); + FileStatus[] statuses = dfs.listStatus(p); + for (FileStatus status: statuses) { + final String path = status.getPath().toUri().getPath(); + System.out.println("Found path " + path); + found.add(path); + if (status.isDirectory()) { + toList.add(status.getPath()); + } + } + } + for (String s: expected) { + assertTrue("Did not find expected path " + s, found.contains(s)); + } + assertEquals("Found an unexpected path while listing filesystem", + found.size(), expected.length); + } + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } + /** * Test upgrade from 2.0 image with a variety of .snapshot and .reserved * paths to test renaming on upgrade diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-0.23-reserved.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-0.23-reserved.tgz new file mode 100644 index 0000000000000000000000000000000000000000..0f53f2adb27e1a8379e4bc0d74a9c98700228559 GIT binary patch literal 4558 zcmeIzjZ=~b90zc^)~u~`xy*u;?bI}N;Zm9<$Zbsxbv2fyXv%4pQuBq2R1|%h;P<~ zeptpvWNe1L8OaWZ@A>5u*ZIeWy?kc0q`jGDp7_P~#A;tABY)SB3$pWoDe^z~n z@UbVq?de_xxdKiu$QgD2)*Q;zc(1z{x|-t`jrbZ}Ji4-`dV^-LC2`$uhT0F%^$dID z*<)_RPXUidWHW4v+ZG&T_G7+;yk~W|#m1N7@P|P40EE9&0~yjoq)r-1oAbGPC^Y69 z8*UBLHtxT8w^o18V|(T;gT>RwS5d_GrsdiZBgM9ybh2w)ue`X!X`xyt-rQ$n?F*Pj zOV^?=_{DSz@iURwnUG0LbjJ-8CLJ4BH~-3ZbK?xeVx1T!iP~+uER+OLLZFgBUsli| z$oIv4?Tdy*&cHr_=RuB&Nmxu9ZAnQ6S`$(H@2}jdNaHDFq-60XB?4c4RrVKg+Hh$+ zG29<_xad@|PgWUuL;pZDXLxP=g%B=}DkQfLiO%=YzRG?;<|j>etHx!hpD(9jwp_kD zaKdQNx_^aAR%hm<5XW0a|J|H7{tV%qFWpKK3SE6DoVv2sADd>W=)F~X!D#QcSngA(hYnuF&RDD?uM=ITa*`plqUukOJuK}kg|b^0H}}d(-S~nC(zD~ zB4Fd*=CbIYch=R4dGsZQaXhgON%sm(o-s$6KT9ao%7A;PPU7mW6tBOW)?UT#i0U0s z5BXN{>Wax?n0VL64BigXXDW8i_ap%KB|%3S?<|{en!a6?l^L(sQ>%;_m zLCNwwX1mBVRc@3xjna$Y07YK9J9w$lzb_qy%wqq1QPi}*v?-CMJkZmnOnW>MknJq# zG2N7P%GE5$hz)IESxXOy?0(g3Ek`FuhEy(jgjOm9v^_CQO_#zjWhA7PpqQ;9+R-^v6L)y0ibQ&z8N~|R#}8p9>HR|8l?ZOCO&F|uJD~&=HUH92 z;nvdzI5FVIt*zd~Sr(6}nT6OwB(fda-?>x<;=Y@GJKNnLK;BphoqAfs?ttbZ1PKt0 zhuiMlVpmC5)O%cKmn(rCa^z z!TydX=5Slq?Y>Kko7or1p$FZ{as$T(^VH7ntp>}`#KN`CWB45`g4a4DE0`5{_BCar z1P@nf8Zcp^d+_jYNRpk~`fCXSIQ2sj!9K{~h0QKk#Z^8?ZFcLc&Hndwge=7vnaJ*F zY*XmbMMY9*hIDb zTGLCRZ3uk#3v%j32*$+I<)L6c>gErz7Sj_;X6Q&#A?i+>-DDQjmUKOORujJyMG_Wf z9RbY(R-UaKwH+`WrEon~=^6` z*{t(|7K}iJSb$>(_t?T%U&Kzm{UGD4^4O^^r`T)f1^15)MPJx2)4Iq7a?CVt$f**Rs>nq|P3ENNKBsw56s7fr+-Q%qJLTq+)*8nTJK# z%EyJu5}OhlW+5gbxkV%j&QLTJuoxLWp`bWY`D}OlX?MH*1MB@KynlJY>5g_fg*@8M zerm9b_z8Bi-6+bVw?-thjbEaA;P}gh4_x4|3 z5-@|ZOsf(*j6Kl$`x~uDr$ws^RJ-@Pb^Y-pF%n386994p%`AQe zFRR5%y*m1Rg7$+@d_Xa%-wc~ThV)~$=-he$7t71_DUqebHsn5jHKbryxA9w>n_4Lo zig88{5}-583K2DaBrXQvS{;XN1?fUZggJBQ|GLXKt&jLI=Xy*9U{B9ex3AG5Y z$mN*?qfS#wos|%?UKi02A+hDAZte-|JolOlMOr%or`PsU%=W1L< zeNg~)zPBS2{qrveZ_%xnNBSyyrQ9yKR{TW0v$+$^y@VlYTd1V7Ryx!B=7W6H_L>r( zlo)vaOWIb>u4j^g%+|{Cs?X-*0v@zyS1|T>qslSrSx%08{L9-LV<{DTccQtv2FlzL z)R=C}ot=iRU+iuL`I9e}d7Glf0Inrns4NU|BcekKzSL#oh3<*HYCYd;M^mJ`{6?2yq0OYXuy<;W2c{YI5wiVx^ zZD7I%?ME$u)=1ec^; zTx^l{^ezi?E*;RoRPz+VVLB?>^_2J9aD2cLxoVZ_rAMFAzYkJAO)5KX_n{95J{q~NDI-Wd8mh9#HC>elkZavDdlYeMnI zmG!01lXf`wM;J!Rjuv&x|S(ec%FR_UvwgfT#vXJbTQ#6BE0p0;qA4p zz6{|bf1Kw9%Q!o4w(zU`tj}TQ#THHg`zE#w*$!P0mIxefVvO~gcJhKoW`Z5Sy2m*& z!j5|5RCOV1?ztR}iY=h?W;25J1pAE((@$TX_jgSzXY6ih)kRzH^yhncH&6N}#S~(s zu~WaBL5EDVD44ES;G{RqNN1dW#>%SUn1RpfC+Z|L%UQtoIC7U9+(3v(`P-tGQU75P zJR@5&@%o<@379>Bzjoxc4UjGl{RL8Fh%r399%lr3pDWF(Jt@mDbl2;77DfbEx8A5n z^ Date: Thu, 24 Jul 2014 23:49:22 +0000 Subject: [PATCH 18/37] HADOOP-10891. Add EncryptedKeyVersion factory method to KeyProviderCryptoExtension. (wang) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613332 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../key/KeyProviderCryptoExtension.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 5ea931c649e..55914f83197 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -189,6 +189,9 @@ Trunk (Unreleased) HADOOP-10720. KMS: Implement generateEncryptedKey and decryptEncryptedKey in the REST API. (asuresh via tucu) + HADOOP-10891. Add EncryptedKeyVersion factory method to + KeyProviderCryptoExtension. (wang) + BUG FIXES HADOOP-9451. Fault single-layer config if node group topology is enabled. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java index 0ba73f1519d..227e19b4841 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java @@ -79,6 +79,30 @@ public class KeyProviderCryptoExtension extends this.encryptedKeyVersion = encryptedKeyVersion; } + /** + * Factory method to create a new EncryptedKeyVersion that can then be + * passed into {@link #decryptEncryptedKey}. Note that the fields of the + * returned EncryptedKeyVersion will only partially be populated; it is not + * necessarily suitable for operations besides decryption. + * + * @param encryptionKeyVersionName Version name of the encryption key used + * to encrypt the encrypted key. + * @param encryptedKeyIv Initialization vector of the encrypted + * key. The IV of the encryption key used to + * encrypt the encrypted key is derived from + * this IV. + * @param encryptedKeyMaterial Key material of the encrypted key. + * @return EncryptedKeyVersion suitable for decryption. + */ + public static EncryptedKeyVersion createForDecryption(String + encryptionKeyVersionName, byte[] encryptedKeyIv, + byte[] encryptedKeyMaterial) { + KeyVersion encryptedKeyVersion = new KeyVersion(null, null, + encryptedKeyMaterial); + return new EncryptedKeyVersion(null, encryptionKeyVersionName, + encryptedKeyIv, encryptedKeyVersion); + } + /** * @return Name of the encryption key used to encrypt the encrypted key. */ From 2bb650146ddb36830ea9c0d248fd3df1f6aa7534 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Fri, 25 Jul 2014 07:02:53 +0000 Subject: [PATCH 19/37] HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress edits files (vinayakumarb) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613355 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/namenode/FileJournalManager.java | 31 +++++++++++++++++-- .../TestNNStorageRetentionManager.java | 11 +++++-- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 498454916d8..9fb5ba132e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -370,6 +370,9 @@ Release 2.6.0 - UNRELEASED HDFS-6715. Webhdfs wont fail over when it gets java.io.IOException: Namenode is in startup mode. (jing9) + HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress edits + files (vinayakumarb) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java index a41ff1390c5..362c316cc2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java @@ -71,6 +71,8 @@ public class FileJournalManager implements JournalManager { NameNodeFile.EDITS.getName() + "_(\\d+)-(\\d+)"); private static final Pattern EDITS_INPROGRESS_REGEX = Pattern.compile( NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+)"); + private static final Pattern EDITS_INPROGRESS_STALE_REGEX = Pattern.compile( + NameNodeFile.EDITS_INPROGRESS.getName() + "_(\\d+).*(\\S+)"); private File currentInProgress = null; @@ -162,8 +164,7 @@ public class FileJournalManager implements JournalManager { throws IOException { LOG.info("Purging logs older than " + minTxIdToKeep); File[] files = FileUtil.listFiles(sd.getCurrentDir()); - List editLogs = - FileJournalManager.matchEditLogs(files); + List editLogs = matchEditLogs(files, true); for (EditLogFile log : editLogs) { if (log.getFirstTxId() < minTxIdToKeep && log.getLastTxId() < minTxIdToKeep) { @@ -244,8 +245,13 @@ public class FileJournalManager implements JournalManager { public static List matchEditLogs(File logDir) throws IOException { return matchEditLogs(FileUtil.listFiles(logDir)); } - + static List matchEditLogs(File[] filesInStorage) { + return matchEditLogs(filesInStorage, false); + } + + private static List matchEditLogs(File[] filesInStorage, + boolean forPurging) { List ret = Lists.newArrayList(); for (File f : filesInStorage) { String name = f.getName(); @@ -256,6 +262,7 @@ public class FileJournalManager implements JournalManager { long startTxId = Long.parseLong(editsMatch.group(1)); long endTxId = Long.parseLong(editsMatch.group(2)); ret.add(new EditLogFile(f, startTxId, endTxId)); + continue; } catch (NumberFormatException nfe) { LOG.error("Edits file " + f + " has improperly formatted " + "transaction ID"); @@ -270,12 +277,30 @@ public class FileJournalManager implements JournalManager { long startTxId = Long.parseLong(inProgressEditsMatch.group(1)); ret.add( new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID, true)); + continue; } catch (NumberFormatException nfe) { LOG.error("In-progress edits file " + f + " has improperly " + "formatted transaction ID"); // skip } } + if (forPurging) { + // Check for in-progress stale edits + Matcher staleInprogressEditsMatch = EDITS_INPROGRESS_STALE_REGEX + .matcher(name); + if (staleInprogressEditsMatch.matches()) { + try { + long startTxId = Long.valueOf(staleInprogressEditsMatch.group(1)); + ret.add(new EditLogFile(f, startTxId, HdfsConstants.INVALID_TXID, + true)); + continue; + } catch (NumberFormatException nfe) { + LOG.error("In-progress stale edits file " + f + " has improperly " + + "formatted transaction ID"); + // skip + } + } + } } return ret; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java index 8f08ef39f87..346d94962bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNNStorageRetentionManager.java @@ -212,18 +212,25 @@ public class TestNNStorageRetentionManager { tc.addImage("/foo1/current/" + getImageFileName(300), false); tc.addImage("/foo1/current/" + getImageFileName(400), false); + // Segments containing txns upto txId 250 are extra and should be purged. tc.addLog("/foo2/current/" + getFinalizedEditsFileName(1, 100), true); - // Without lowering the max segments to retain, we'd retain all segments - // going back to txid 150 (300 - 150). tc.addLog("/foo2/current/" + getFinalizedEditsFileName(101, 175), true); + tc.addLog("/foo2/current/" + getInProgressEditsFileName(176) + ".empty", + true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(176, 200), true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(201, 225), true); + tc.addLog("/foo2/current/" + getInProgressEditsFileName(226) + ".corrupt", + true); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(226, 240), true); // Only retain 2 extra segments. The 301-350 and 351-400 segments are // considered required, not extra. tc.addLog("/foo2/current/" + getFinalizedEditsFileName(241, 275), false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(276, 300), false); + tc.addLog("/foo2/current/" + getInProgressEditsFileName(301) + ".empty", + false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(301, 350), false); + tc.addLog("/foo2/current/" + getInProgressEditsFileName(351) + ".corrupt", + false); tc.addLog("/foo2/current/" + getFinalizedEditsFileName(351, 400), false); tc.addLog("/foo2/current/" + getInProgressEditsFileName(401), false); runTest(tc); From 1e553858f930e43fac62986549a178cdcf39384c Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Fri, 25 Jul 2014 16:13:07 +0000 Subject: [PATCH 20/37] YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue delays convergence towards fairness. (Ashwin Shankar via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613459 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/fair/FSLeafQueue.java | 19 ++++- .../scheduler/fair/FSParentQueue.java | 5 -- .../scheduler/fair/FSQueue.java | 13 ---- .../scheduler/fair/TestFairScheduler.java | 73 +++++++++++++++++++ 5 files changed, 91 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1712abeb9e4..99b60b1aeda 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -65,6 +65,9 @@ Release 2.6.0 - UNRELEASED YARN-1342. Recover container tokens upon nodemanager restart. (Jason Lowe via devaraj) + YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue delays + convergence towards fairness. (Ashwin Shankar via kasha) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index 8f957382e6a..3b3f6ce2296 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -224,16 +224,17 @@ public class FSLeafQueue extends FSQueue { @Override public RMContainer preemptContainer() { RMContainer toBePreempted = null; - if (LOG.isDebugEnabled()) { - LOG.debug("Queue " + getName() + " is going to preempt a container " + - "from its applications."); - } // If this queue is not over its fair share, reject if (!preemptContainerPreCheck()) { return toBePreempted; } + if (LOG.isDebugEnabled()) { + LOG.debug("Queue " + getName() + " is going to preempt a container " + + "from its applications."); + } + // Choose the app that is most over fair share Comparator comparator = policy.getComparator(); AppSchedulable candidateSched = null; @@ -328,4 +329,14 @@ public class FSLeafQueue extends FSQueue { SchedulerApplicationAttempt schedulerAttempt, RMContainer rmContainer) { // TODO Auto-generated method stub } + + /** + * Helper method to check if the queue should preempt containers + * + * @return true if check passes (can preempt) or false otherwise + */ + private boolean preemptContainerPreCheck() { + return parent.getPolicy().checkIfUsageOverFairShare(getResourceUsage(), + getFairShare()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index 5ab60afbca2..9af72a511e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -164,11 +164,6 @@ public class FSParentQueue extends FSQueue { public RMContainer preemptContainer() { RMContainer toBePreempted = null; - // If this queue is not over its fair share, reject - if (!preemptContainerPreCheck()) { - return toBePreempted; - } - // Find the childQueue which is most over fair share FSQueue candidateQueue = null; Comparator comparator = policy.getComparator(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java index 716e1ee6874..1e94046100a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java @@ -187,17 +187,4 @@ public abstract class FSQueue extends Schedulable implements Queue { } return true; } - - /** - * Helper method to check if the queue should preempt containers - * - * @return true if check passes (can preempt) or false otherwise - */ - protected boolean preemptContainerPreCheck() { - if (this == scheduler.getQueueManager().getRootQueue()) { - return true; - } - return parent.getPolicy() - .checkIfUsageOverFairShare(getResourceUsage(), getFairShare()); - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index df157e75001..33ec3184a91 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -1221,6 +1221,79 @@ public class TestFairScheduler extends FairSchedulerTestBase { scheduler.getSchedulerApp(app4).getPreemptionContainers().isEmpty()); } + @Test + public void testPreemptionIsNotDelayedToNextRound() throws Exception { + conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000); + conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000); + conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); + conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false"); + + MockClock clock = new MockClock(); + scheduler.setClock(clock); + + PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); + out.println(""); + out.println(""); + out.println(""); + out.println("8"); + out.println(""); + out.println(""); + out.println(""); + out.println(""); + out.println("2"); + out.println(""); + out.print("10"); + out.println(""); + out.close(); + + scheduler.init(conf); + scheduler.start(); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + + // Add a node of 8G + RMNode node1 = MockNodes.newNodeInfo(1, + Resources.createResource(8 * 1024, 8), 1, "127.0.0.1"); + NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); + scheduler.handle(nodeEvent1); + + // Run apps in queueA.A1 and queueB + ApplicationAttemptId app1 = createSchedulingRequest(1 * 1024, 1, + "queueA.queueA1", "user1", 7, 1); + // createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app1); + ApplicationAttemptId app2 = createSchedulingRequest(1 * 1024, 1, "queueB", + "user2", 1, 1); + + scheduler.update(); + + NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); + for (int i = 0; i < 8; i++) { + scheduler.handle(nodeUpdate1); + } + + // verify if the apps got the containers they requested + assertEquals(7, scheduler.getSchedulerApp(app1).getLiveContainers().size()); + assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size()); + + // Now submit an app in queueA.queueA2 + ApplicationAttemptId app3 = createSchedulingRequest(1 * 1024, 1, + "queueA.queueA2", "user3", 7, 1); + scheduler.update(); + + // Let 11 sec pass + clock.tick(11); + + scheduler.update(); + Resource toPreempt = scheduler.resToPreempt(scheduler.getQueueManager() + .getLeafQueue("queueA.queueA2", false), clock.getTime()); + assertEquals(2980, toPreempt.getMemory()); + + // verify if the 3 containers required by queueA2 are preempted in the same + // round + scheduler.preemptResources(toPreempt); + assertEquals(3, scheduler.getSchedulerApp(app1).getPreemptionContainers() + .size()); + } + @Test (timeout = 5000) /** * Tests the timing of decision to preempt tasks. From 77363b9d839e47bef2325b8682eabe00d4c83354 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Fri, 25 Jul 2014 17:12:22 +0000 Subject: [PATCH 21/37] YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613478 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/yarn/sls/RumenToSLSConverter.java | 4 ++++ .../main/java/org/apache/hadoop/yarn/sls/SLSRunner.java | 4 ++++ .../org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java | 4 ++++ .../apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java | 4 ++++ .../org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java | 5 +++++ .../apache/hadoop/yarn/sls/nodemanager/NMSimulator.java | 4 ++++ .../org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java | 8 +++++++- .../yarn/sls/scheduler/CapacitySchedulerMetrics.java | 5 +++++ .../hadoop/yarn/sls/scheduler/ContainerSimulator.java | 4 ++++ .../hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java | 4 ++++ .../hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java | 4 ++++ .../sls/scheduler/NodeUpdateSchedulerEventWrapper.java | 4 ++++ .../apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java | 4 ++++ .../yarn/sls/scheduler/ResourceSchedulerWrapper.java | 3 +++ .../hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java | 4 ++++ .../hadoop/yarn/sls/scheduler/SchedulerMetrics.java | 4 ++++ .../hadoop/yarn/sls/scheduler/SchedulerWrapper.java | 4 ++++ .../org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java | 6 ++++++ .../java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java | 4 ++++ .../java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java | 4 ++++ hadoop-yarn-project/CHANGES.txt | 2 ++ 21 files changed, 88 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java index 567963412b1..2d4b4ae5264 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/RumenToSLSConverter.java @@ -21,6 +21,8 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.Options; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.map.ObjectWriter; @@ -42,6 +44,8 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +@Private +@Unstable public class RumenToSLSConverter { private static final String EOL = System.getProperty("line.separator"); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java index 501d11e0ccd..9baa73626d5 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java @@ -32,6 +32,8 @@ import java.util.Iterator; import java.util.Random; import java.util.Arrays; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.fs.Path; import org.apache.hadoop.tools.rumen.JobTraceReader; import org.apache.hadoop.tools.rumen.LoggedJob; @@ -66,6 +68,8 @@ import org.apache.log4j.Logger; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.map.ObjectMapper; +@Private +@Unstable public class SLSRunner { // RM, Runner private ResourceManager rm; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 67c09940120..5af4eaa2de4 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -29,6 +29,8 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -70,6 +72,8 @@ import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.scheduler.TaskRunner; import org.apache.hadoop.yarn.sls.utils.SLSUtils; +@Private +@Unstable public abstract class AMSimulator extends TaskRunner.Task { // resource manager protected ResourceManager rm; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java index d789e1ca2aa..d24510ba6fd 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java @@ -27,6 +27,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -45,6 +47,8 @@ import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.log4j.Logger; +@Private +@Unstable public class MRAMSimulator extends AMSimulator { /* Vocabulary Used: diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java index 21785cc67d9..8fd5b3f770a 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/conf/SLSConfiguration.java @@ -18,6 +18,11 @@ package org.apache.hadoop.yarn.sls.conf; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Private +@Unstable public class SLSConfiguration { // sls public static final String PREFIX = "yarn.sls."; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java index b0e2e37fa47..4112685e152 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java @@ -27,6 +27,8 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.DelayQueue; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; @@ -54,6 +56,8 @@ import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; import org.apache.hadoop.yarn.sls.scheduler.TaskRunner; import org.apache.hadoop.yarn.sls.utils.SLSUtils; +@Private +@Unstable public class NMSimulator extends TaskRunner.Task { // node resource private RMNode node; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java index 76671572ce8..1d573822d9b 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.sls.nodemanager; import java.util.ArrayList; import java.util.List; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.net.Node; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; @@ -36,6 +38,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode .UpdatedContainerInfo; +@Private +@Unstable public class NodeInfo { private static int NODE_ID = 0; @@ -43,6 +47,8 @@ public class NodeInfo { return NodeId.newInstance(host, port); } + @Private + @Unstable private static class FakeRMNodeImpl implements RMNode { private NodeId nodeId; private String hostName; @@ -164,7 +170,7 @@ public class NodeInfo { perNode = resourceOption; } } - + public static RMNode newNodeInfo(String rackName, String hostName, final ResourceOption resourceOption, int port) { final NodeId nodeId = newNodeID(hostName, port); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java index 1d878537061..a73f48c4d7e 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/CapacitySchedulerMetrics.java @@ -18,6 +18,11 @@ package org.apache.hadoop.yarn.sls.scheduler; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Private +@Unstable public class CapacitySchedulerMetrics extends SchedulerMetrics { public CapacitySchedulerMetrics() { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java index 27a50d3b00f..86229763e15 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ContainerSimulator.java @@ -21,9 +21,13 @@ package org.apache.hadoop.yarn.sls.scheduler; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; +@Private +@Unstable public class ContainerSimulator implements Delayed { // id private ContainerId id; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java index b706b3ee6d8..f427dcd557a 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FairSchedulerMetrics.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.sls.scheduler; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair .AppSchedulable; @@ -28,6 +30,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair import com.codahale.metrics.Gauge; import org.apache.hadoop.yarn.sls.SLSRunner; +@Private +@Unstable public class FairSchedulerMetrics extends SchedulerMetrics { private int totalMemoryMB = Integer.MAX_VALUE; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java index 882b3e1215b..6ab2e1d0107 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/FifoSchedulerMetrics.java @@ -18,12 +18,16 @@ package org.apache.hadoop.yarn.sls.scheduler; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo .FifoScheduler; import com.codahale.metrics.Gauge; +@Private +@Unstable public class FifoSchedulerMetrics extends SchedulerMetrics { public FifoSchedulerMetrics() { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java index 4bf93138053..12dfe8baa16 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/NodeUpdateSchedulerEventWrapper.java @@ -18,9 +18,13 @@ package org.apache.hadoop.yarn.sls.scheduler; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .NodeUpdateSchedulerEvent; +@Private +@Unstable public class NodeUpdateSchedulerEventWrapper extends NodeUpdateSchedulerEvent { public NodeUpdateSchedulerEventWrapper(NodeUpdateSchedulerEvent event) { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java index bbe24c883c7..da9b56fd546 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.sls.scheduler; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.net.Node; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -33,6 +35,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode import java.util.Collections; import java.util.List; +@Private +@Unstable public class RMNodeWrapper implements RMNode { private RMNode node; private List updates; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 455808543b6..0bd0c87d2d0 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -36,6 +36,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configurable; @@ -92,6 +93,8 @@ import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.SlidingWindowReservoir; import com.codahale.metrics.Timer; +@Private +@Unstable final public class ResourceSchedulerWrapper extends AbstractYarnScheduler implements SchedulerWrapper, ResourceScheduler, Configurable { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 6a84e5838c4..44a872198d6 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.sls.scheduler; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.yarn.sls.SLSRunner; import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; @@ -100,6 +102,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +@Private +@Unstable public class SLSCapacityScheduler extends CapacityScheduler implements SchedulerWrapper,Configurable { private static final String EOL = System.getProperty("line.separator"); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java index 814d1334011..ecf516d7c98 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java @@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.sls.scheduler; import java.util.HashSet; import java.util.Set; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.server.resourcemanager.scheduler .ResourceScheduler; @@ -30,6 +32,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; +@Private +@Unstable public abstract class SchedulerMetrics { protected ResourceScheduler scheduler; protected Set trackedQueues; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java index 44629f5347f..524b8bf23e1 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerWrapper.java @@ -19,11 +19,15 @@ package org.apache.hadoop.yarn.sls.scheduler; import java.util.Set; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import com.codahale.metrics.MetricRegistry; +@Private +@Unstable public interface SchedulerWrapper { public MetricRegistry getMetrics(); diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java index efb573c9ec3..c936dd93180 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java @@ -25,9 +25,15 @@ import java.util.concurrent.Delayed; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.exceptions.YarnException; +@Private +@Unstable public class TaskRunner { + @Private + @Unstable public abstract static class Task implements Runnable, Delayed { private long start; private long end; diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java index 70ab96d711c..f62f02471b9 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/utils/SLSUtils.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.sls.utils; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.tools.rumen.JobTraceReader; @@ -36,6 +38,8 @@ import java.util.Map; import java.util.List; import java.util.Iterator; +@Private +@Unstable public class SLSUtils { public static String[] getRackHostName(String hostname) { diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java index e6dd8467898..45301a18a54 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/web/SLSWebApp.java @@ -30,6 +30,8 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event .SchedulerEventType; import org.mortbay.jetty.Handler; @@ -49,6 +51,8 @@ import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import org.mortbay.jetty.handler.ResourceHandler; +@Private +@Unstable public class SLSWebApp extends HttpServlet { private static final long serialVersionUID = 1905162041950251407L; private transient Server server; diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 99b60b1aeda..69ec1b25a11 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -439,6 +439,8 @@ Release 2.5.0 - UNRELEASED YARN-2319. Made the MiniKdc instance start/close before/after the class of TestRMWebServicesDelegationTokens. (Wenwu Peng via zjshen) + YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES From cc0464f4fe793d9894776e062193e6609b7c1689 Mon Sep 17 00:00:00 2001 From: Vinayakumar B Date: Fri, 25 Jul 2014 18:17:50 +0000 Subject: [PATCH 22/37] HDFS-6752. Avoid Address bind errors in TestDatanodeConfig#testMemlockLimit (vinayakumarb) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613486 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9fb5ba132e1..502771a1160 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -373,6 +373,9 @@ Release 2.6.0 - UNRELEASED HDFS-5919. FileJournalManager doesn't purge empty and corrupt inprogress edits files (vinayakumarb) + HDFS-6752. Avoid Address bind errors in TestDatanodeConfig#testMemlockLimit + (vinayakumarb) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java index cfa14cca41b..9cdb763f032 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeConfig.java @@ -53,6 +53,8 @@ public class TestDatanodeConfig { Configuration conf = new HdfsConfiguration(); conf.setInt(DFSConfigKeys.DFS_DATANODE_HTTPS_PORT_KEY, 0); conf.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "localhost:0"); + conf.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "localhost:0"); + conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "localhost:0"); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build(); cluster.waitActive(); } From 79d214121b29104c14bf9d64c444bf75568cc4d4 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 25 Jul 2014 19:02:19 +0000 Subject: [PATCH 23/37] Fix up HDFS CHANGES.txt for 2.5.0 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613494 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 43 ++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 502771a1160..65f0665653a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -398,6 +398,15 @@ Release 2.5.0 - UNRELEASED HDFS-6406. Add capability for NFS gateway to reject connections from unprivileged ports. (atm) + HDFS-2006. Ability to support storing extended attributes per file. + + HDFS-5978. Create a tool to take fsimage and expose read-only WebHDFS API. + (Akira Ajisaka via wheat9) + + HDFS-6278. Create HTML5-based UI for SNN. (wheat9) + + HDFS-6279. Create new index page for JN / DN. (wheat9) + IMPROVEMENTS HDFS-6007. Update documentation about short-circuit local reads (iwasakims @@ -415,9 +424,6 @@ Release 2.5.0 - UNRELEASED HDFS-6158. Clean up dead code for OfflineImageViewer. (wheat9) - HDFS-5978. Create a tool to take fsimage and expose read-only WebHDFS API. - (Akira Ajisaka via wheat9) - HDFS-6164. Remove lsr in OfflineImageViewer. (wheat9) HDFS-6167. Relocate the non-public API classes in the hdfs.client package. @@ -445,10 +451,6 @@ Release 2.5.0 - UNRELEASED HDFS-6265. Prepare HDFS codebase for JUnit 4.11. (cnauroth) - HDFS-6278. Create HTML5-based UI for SNN. (wheat9) - - HDFS-6279. Create new index page for JN / DN. (wheat9) - HDFS-5693. Few NN metrics data points were collected via JMX when NN is under heavy load. (Ming Ma via jing9) @@ -820,9 +822,6 @@ Release 2.5.0 - UNRELEASED HDFS-6464. Support multiple xattr.name parameters for WebHDFS getXAttrs. (Yi Liu via umamahesh) - HDFS-6375. Listing extended attributes with the search permission. - (Charles Lamb via wang) - HDFS-6539. test_native_mini_dfs is skipped in hadoop-hdfs/pom.xml (decstery via cmccabe) @@ -917,6 +916,18 @@ Release 2.5.0 - UNRELEASED HDFS-6696. Name node cannot start if the path of a file under construction contains ".snapshot". (wang) + HDFS-6312. WebHdfs HA failover is broken on secure clusters. + (daryn via tucu) + + HDFS-6618. FSNamesystem#delete drops the FSN lock between removing INodes + from the tree and deleting them from the inode map (kihwal via cmccabe) + + HDFS-6622. Rename and AddBlock may race and produce invalid edits (kihwal + via cmccabe) + + HDFS-6723. New NN webUI no longer displays decommissioned state for dead node. + (Ming Ma via wheat9) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) @@ -986,18 +997,6 @@ Release 2.5.0 - UNRELEASED HDFS-6492. Support create-time xattrs and atomically setting multiple xattrs. (wang) - HDFS-6312. WebHdfs HA failover is broken on secure clusters. - (daryn via tucu) - - HDFS-6618. FSNamesystem#delete drops the FSN lock between removing INodes - from the tree and deleting them from the inode map (kihwal via cmccabe) - - HDFS-6622. Rename and AddBlock may race and produce invalid edits (kihwal - via cmccabe) - - HDFS-6723. New NN webUI no longer displays decommissioned state for dead node. - (Ming Ma via wheat9) - Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES From 10286e98ad3e07607b5a368b8f9b75ae99db1062 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 25 Jul 2014 19:06:05 +0000 Subject: [PATCH 24/37] Fix up Common CHANGES.txt for 2.5.0 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613498 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 55914f83197..419d45c6831 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -503,6 +503,9 @@ Release 2.5.0 - UNRELEASED HADOOP-8943. Support multiple group mapping providers. (Kai Zheng via brandonli) + HADOOP-9361 Strictly define the expected behavior of filesystem APIs and + write tests to verify compliance (stevel) + IMPROVEMENTS HADOOP-10451. Remove unused field and imports from SaslRpcServer. @@ -597,9 +600,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10747. Support configurable retries on SASL connection failures in RPC client. (cnauroth) - HADOOP-10674. Improve PureJavaCrc32 performance and use java.util.zip.CRC32 - for Java 7 and above. (szetszwo) - HADOOP-10754. Reenable several HA ZooKeeper-related tests on Windows. (cnauroth) @@ -611,9 +611,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10767. Clean up unused code in Ls shell command. (cnauroth) - HADOOP-9361 Strictly define the expected behavior of filesystem APIs and - write tests to verify compliance (stevel) - HADOOP-9651 Filesystems to throw FileAlreadyExistsException in createFile(path, overwrite=false) when the file exists (stevel) @@ -626,6 +623,9 @@ Release 2.5.0 - UNRELEASED OPTIMIZATIONS + HADOOP-10674. Improve PureJavaCrc32 performance and use java.util.zip.CRC32 + for Java 7 and above. (szetszwo) + BUG FIXES HADOOP-10378. Typo in help printed by hdfs dfs -help. @@ -780,27 +780,6 @@ Release 2.5.0 - UNRELEASED HADOOP-10801 dead link in site.xml (Akira AJISAKA via stevel) - BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS - - HADOOP-10520. Extended attributes definition and FileSystem APIs for - extended attributes. (Yi Liu via wang) - - HADOOP-10546. Javadoc and other small fixes for extended attributes in - hadoop-common. (Charles Lamb via wang) - - HADOOP-10521. FsShell commands for extended attributes. (Yi Liu via wang) - - HADOOP-10548. Improve FsShell xattr error handling and other fixes. (Charles Lamb via umamahesh) - - HADOOP-10567. Shift XAttr value encoding code out for reuse. (Yi Liu via umamahesh) - - HADOOP-10621. Remove CRLF for xattr value base64 encoding for better display.(Yi Liu via umamahesh) - - HADOOP-10575. Small fixes for XAttrCommands and test. (Yi Liu via umamahesh) - - HADOOP-10561. Copy command with preserve option should handle Xattrs. - (Yi Liu via cnauroth) - HADOOP-10590. ServiceAuthorizationManager is not threadsafe. (Benoy Antony via vinayakumarb) HADOOP-10711. Cleanup some extra dependencies from hadoop-auth. (rkanter via tucu) @@ -825,6 +804,27 @@ Release 2.5.0 - UNRELEASED HADOOP-10894. Fix dead link in ToolRunner documentation. (Akira Ajisaka via Arpit Agarwal) + BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS + + HADOOP-10520. Extended attributes definition and FileSystem APIs for + extended attributes. (Yi Liu via wang) + + HADOOP-10546. Javadoc and other small fixes for extended attributes in + hadoop-common. (Charles Lamb via wang) + + HADOOP-10521. FsShell commands for extended attributes. (Yi Liu via wang) + + HADOOP-10548. Improve FsShell xattr error handling and other fixes. (Charles Lamb via umamahesh) + + HADOOP-10567. Shift XAttr value encoding code out for reuse. (Yi Liu via umamahesh) + + HADOOP-10621. Remove CRLF for xattr value base64 encoding for better display.(Yi Liu via umamahesh) + + HADOOP-10575. Small fixes for XAttrCommands and test. (Yi Liu via umamahesh) + + HADOOP-10561. Copy command with preserve option should handle Xattrs. + (Yi Liu via cnauroth) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES From d4fec3493351c619a0278929ae2d5c8cd67cbfbe Mon Sep 17 00:00:00 2001 From: Jian He Date: Fri, 25 Jul 2014 20:42:37 +0000 Subject: [PATCH 25/37] YARN-2211. Persist AMRMToken master key in RMStateStore for RM recovery. Contributed by Xuan Gong git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613515 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/client/ProtocolHATestBase.java | 1 + .../TestApplicationMasterServiceOnHA.java | 6 +- .../pom.xml | 31 +++++ .../RMSecretManagerService.java | 6 +- .../resourcemanager/ResourceManager.java | 3 + .../recovery/FileSystemRMStateStore.java | 55 +++++++- .../recovery/MemoryRMStateStore.java | 15 +++ .../recovery/NullRMStateStore.java | 7 + .../recovery/RMStateStore.java | 20 ++- .../recovery/ZKRMStateStore.java | 45 ++++++- .../records/AMRMTokenSecretManagerState.java | 76 +++++++++++ .../pb/AMRMTokenSecretManagerStatePBImpl.java | 126 ++++++++++++++++++ .../security/AMRMTokenSecretManager.java | 48 ++++++- ...yarn_server_resourcemanager_recovery.proto | 30 +++++ .../server/resourcemanager/TestRMRestart.java | 9 +- .../recovery/RMStateStoreTestBase.java | 71 +++++++++- .../recovery/TestFSRMStateStore.java | 2 +- .../recovery/TestZKRMStateStore.java | 1 + .../rmapp/TestRMAppTransitions.java | 2 +- .../attempt/TestRMAppAttemptTransitions.java | 3 +- .../scheduler/capacity/TestUtils.java | 5 +- .../security/TestAMRMTokens.java | 4 +- 23 files changed, 535 insertions(+), 34 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 69ec1b25a11..e6fbea9350e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -68,6 +68,9 @@ Release 2.6.0 - UNRELEASED YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue delays convergence towards fairness. (Ashwin Shankar via kasha) + YARN-2211. Persist AMRMToken master key in RMStateStore for RM recovery. + (Xuan Gong via jianhe) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java index 15bfa28d20d..72cb1b1684c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java @@ -267,6 +267,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes{ protected void startHACluster(int numOfNMs, boolean overrideClientRMService, boolean overrideRTS, boolean overrideApplicationMasterService) throws Exception { + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster = new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java index 4771ccba9db..0b42ac3c6b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestApplicationMasterServiceOnHA.java @@ -54,11 +54,9 @@ public class TestApplicationMasterServiceOnHA extends ProtocolHATestBase{ amClient = ClientRMProxy .createRMProxy(this.conf, ApplicationMasterProtocol.class); - AMRMTokenIdentifier id = - new AMRMTokenIdentifier(attemptId); Token appToken = - new Token(id, this.cluster.getResourceManager() - .getRMContext().getAMRMTokenSecretManager()); + this.cluster.getResourceManager().getRMContext() + .getAMRMTokenSecretManager().createAndGetAMRMToken(attemptId); appToken.setService(new Text("appToken service")); UserGroupInformation.setLoginUser(UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index c2a94ead159..0f89bbe38a2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -244,6 +244,37 @@ + + + org.apache.hadoop + hadoop-maven-plugins + + + compile-protoc + generate-sources + + protoc + + + ${protobuf.version} + ${protoc.path} + + ${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto + ${basedir}/../../hadoop-yarn-api/src/main/proto + ${basedir}/../hadoop-yarn-server-common/src/main/proto + ${basedir}/src/main/proto + + + ${basedir}/src/main/proto + + yarn_server_resourcemanager_recovery.proto + + + ${project.build.directory}/generated-sources/java + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java index 9fdde6589a3..d0d7d16a276 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java @@ -60,7 +60,7 @@ public class RMSecretManagerService extends AbstractService { clientToAMSecretManager = createClientToAMTokenSecretManager(); rmContext.setClientToAMTokenSecretManager(clientToAMSecretManager); - amRmTokenSecretManager = createAMRMTokenSecretManager(conf); + amRmTokenSecretManager = createAMRMTokenSecretManager(conf, this.rmContext); rmContext.setAMRMTokenSecretManager(amRmTokenSecretManager); rmDTSecretManager = @@ -115,8 +115,8 @@ public class RMSecretManagerService extends AbstractService { } protected AMRMTokenSecretManager createAMRMTokenSecretManager( - Configuration conf) { - return new AMRMTokenSecretManager(conf); + Configuration conf, RMContext rmContext) { + return new AMRMTokenSecretManager(conf, rmContext); } protected ClientToAMTokenSecretManagerInRM createClientToAMTokenSecretManager() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 409ba4893b9..7dceda249da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -1026,6 +1026,9 @@ public class ResourceManager extends CompositeService implements Recoverable { // recover RMdelegationTokenSecretManager rmContext.getRMDelegationTokenSecretManager().recover(state); + // recover AMRMTokenSecretManager + rmContext.getAMRMTokenSecretManager().recover(state); + // recover applications rmAppManager.recover(state); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java index 0e605a9b07d..243c7a19912 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java @@ -22,6 +22,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -43,16 +44,18 @@ import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; - import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl; @@ -76,6 +79,8 @@ public class FileSystemRMStateStore extends RMStateStore { protected static final String ROOT_DIR_NAME = "FSRMStateRoot"; protected static final RMStateVersion CURRENT_VERSION_INFO = RMStateVersion .newInstance(1, 1); + protected static final String AMRMTOKEN_SECRET_MANAGER_NODE = + "AMRMTokenSecretManagerNode"; protected FileSystem fs; @@ -89,6 +94,7 @@ public class FileSystemRMStateStore extends RMStateStore { @VisibleForTesting Path fsWorkingPath; + Path amrmTokenSecretManagerRoot; @Override public synchronized void initInternal(Configuration conf) throws Exception{ @@ -96,6 +102,8 @@ public class FileSystemRMStateStore extends RMStateStore { rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME); rmDTSecretManagerRoot = new Path(rootDirPath, RM_DT_SECRET_MANAGER_ROOT); rmAppRoot = new Path(rootDirPath, RM_APP_ROOT); + amrmTokenSecretManagerRoot = + new Path(rootDirPath, AMRMTOKEN_SECRET_MANAGER_ROOT); } @Override @@ -113,6 +121,7 @@ public class FileSystemRMStateStore extends RMStateStore { fs = fsWorkingPath.getFileSystem(conf); fs.mkdirs(rmDTSecretManagerRoot); fs.mkdirs(rmAppRoot); + fs.mkdirs(amrmTokenSecretManagerRoot); } @Override @@ -180,9 +189,32 @@ public class FileSystemRMStateStore extends RMStateStore { loadRMDTSecretManagerState(rmState); // recover RM applications loadRMAppState(rmState); + // recover AMRMTokenSecretManager + loadAMRMTokenSecretManagerState(rmState); return rmState; } + private void loadAMRMTokenSecretManagerState(RMState rmState) + throws Exception { + checkAndResumeUpdateOperation(amrmTokenSecretManagerRoot); + Path amrmTokenSecretManagerStateDataDir = + new Path(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE); + FileStatus status; + try { + status = fs.getFileStatus(amrmTokenSecretManagerStateDataDir); + assert status.isFile(); + } catch (FileNotFoundException ex) { + return; + } + byte[] data = readFile(amrmTokenSecretManagerStateDataDir, status.getLen()); + AMRMTokenSecretManagerStatePBImpl stateData = + new AMRMTokenSecretManagerStatePBImpl( + AMRMTokenSecretManagerStateProto.parseFrom(data)); + rmState.amrmTokenSecretManagerState = + AMRMTokenSecretManagerState.newInstance( + stateData.getCurrentMasterKey(), stateData.getNextMasterKey()); + } + private void loadRMAppState(RMState rmState) throws Exception { try { List attempts = @@ -597,4 +629,25 @@ public class FileSystemRMStateStore extends RMStateStore { return new Path(root, nodeName); } + @Override + public synchronized void storeOrUpdateAMRMTokenSecretManagerState( + AMRMTokenSecretManagerState amrmTokenSecretManagerState, + boolean isUpdate){ + Path nodeCreatePath = + getNodePath(amrmTokenSecretManagerRoot, AMRMTOKEN_SECRET_MANAGER_NODE); + AMRMTokenSecretManagerState data = + AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState); + byte[] stateData = data.getProto().toByteArray(); + try { + if (isUpdate) { + updateFile(nodeCreatePath, stateData); + } else { + writeFile(nodeCreatePath, stateData); + } + } catch (Exception ex) { + LOG.info("Error storing info for AMRMTokenSecretManager", ex); + notifyStoreOperationFailed(ex); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java index 05cbb09630f..369f89a545e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; @@ -72,6 +73,10 @@ public class MemoryRMStateStore extends RMStateStore { state.rmSecretManagerState.getTokenState()); returnState.rmSecretManagerState.dtSequenceNumber = state.rmSecretManagerState.dtSequenceNumber; + returnState.amrmTokenSecretManagerState = + state.amrmTokenSecretManagerState == null ? null + : AMRMTokenSecretManagerState + .newInstance(state.amrmTokenSecretManagerState); return returnState; } @@ -267,6 +272,16 @@ public class MemoryRMStateStore extends RMStateStore { return null; } + @Override + public void storeOrUpdateAMRMTokenSecretManagerState( + AMRMTokenSecretManagerState amrmTokenSecretManagerState, + boolean isUpdate) { + if (amrmTokenSecretManagerState != null) { + state.amrmTokenSecretManagerState = AMRMTokenSecretManagerState + .newInstance(amrmTokenSecretManagerState); + } + } + @Override public void deleteStore() throws Exception { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java index 690f0bef94d..ea7087176c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java @@ -25,6 +25,7 @@ import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; @@ -138,6 +139,12 @@ public class NullRMStateStore extends RMStateStore { return null; } + @Override + public void storeOrUpdateAMRMTokenSecretManagerState( + AMRMTokenSecretManagerState state, boolean isUpdate) { + //DO Nothing + } + @Override public void deleteStore() throws Exception { // Do nothing diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index 5b75b429697..e2c4e7e47fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -45,16 +45,14 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPB import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMFatalEventType; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNewSavedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -85,6 +83,8 @@ public abstract class RMStateStore extends AbstractService { protected static final String DELEGATION_TOKEN_PREFIX = "RMDelegationToken_"; protected static final String DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX = "RMDTSequenceNumber_"; + protected static final String AMRMTOKEN_SECRET_MANAGER_ROOT = + "AMRMTokenSecretManagerRoot"; protected static final String VERSION_NODE = "RMVersionNode"; protected static final String EPOCH_NODE = "EpochNode"; @@ -412,6 +412,8 @@ public abstract class RMStateStore extends AbstractService { RMDTSecretManagerState rmSecretManagerState = new RMDTSecretManagerState(); + AMRMTokenSecretManagerState amrmTokenSecretManagerState = null; + public Map getApplicationState() { return appState; } @@ -419,6 +421,10 @@ public abstract class RMStateStore extends AbstractService { public RMDTSecretManagerState getRMDTSecretManagerState() { return rmSecretManagerState; } + + public AMRMTokenSecretManagerState getAMRMTokenSecretManagerState() { + return amrmTokenSecretManagerState; + } } private Dispatcher rmDispatcher; @@ -713,6 +719,14 @@ public abstract class RMStateStore extends AbstractService { protected abstract void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception; + /** + * Blocking API Derived classes must implement this method to store or update + * the state of AMRMToken Master Key + */ + public abstract void storeOrUpdateAMRMTokenSecretManagerState( + AMRMTokenSecretManagerState amrmTokenSecretManagerState, + boolean isUpdate); + /** * Non-blocking API * ResourceManager services call this to remove an application from the state diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java index 68b4632d3b5..5644ad9e34a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java @@ -44,18 +44,19 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.HAUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationAttemptStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.EpochProto; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; - import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.Epoch; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AMRMTokenSecretManagerStatePBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl; @@ -128,6 +129,9 @@ public class ZKRMStateStore extends RMStateStore { * | |----- Key_1 * | |----- Key_2 * .... + * |--- AMRMTOKEN_SECRET_MANAGER_ROOT + * |----- currentMasterKey + * |----- nextMasterKey * */ private String zkRootNodePath; @@ -136,6 +140,7 @@ public class ZKRMStateStore extends RMStateStore { private String dtMasterKeysRootPath; private String delegationTokensRootPath; private String dtSequenceNumberPath; + private String amrmTokenSecretManagerRoot; @VisibleForTesting protected String znodeWorkingPath; @@ -255,6 +260,8 @@ public class ZKRMStateStore extends RMStateStore { RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME); dtSequenceNumberPath = getNodePath(rmDTSecretManagerRoot, RM_DT_SEQUENTIAL_NUMBER_ZNODE_NAME); + amrmTokenSecretManagerRoot = + getNodePath(zkRootNodePath, AMRMTOKEN_SECRET_MANAGER_ROOT); } @Override @@ -275,6 +282,7 @@ public class ZKRMStateStore extends RMStateStore { createRootDir(dtMasterKeysRootPath); createRootDir(delegationTokensRootPath); createRootDir(dtSequenceNumberPath); + createRootDir(amrmTokenSecretManagerRoot); } private void createRootDir(final String rootPath) throws Exception { @@ -427,9 +435,27 @@ public class ZKRMStateStore extends RMStateStore { loadRMDTSecretManagerState(rmState); // recover RM applications loadRMAppState(rmState); + // recover AMRMTokenSecretManager + loadAMRMTokenSecretManagerState(rmState); return rmState; } + private void loadAMRMTokenSecretManagerState(RMState rmState) + throws Exception { + byte[] data = getDataWithRetries(amrmTokenSecretManagerRoot, true); + if (data == null) { + LOG.warn("There is no data saved"); + return; + } + AMRMTokenSecretManagerStatePBImpl stateData = + new AMRMTokenSecretManagerStatePBImpl( + AMRMTokenSecretManagerStateProto.parseFrom(data)); + rmState.amrmTokenSecretManagerState = + AMRMTokenSecretManagerState.newInstance( + stateData.getCurrentMasterKey(), stateData.getNextMasterKey()); + + } + private synchronized void loadRMDTSecretManagerState(RMState rmState) throws Exception { loadRMDelegationKeyState(rmState); @@ -1112,4 +1138,19 @@ public class ZKRMStateStore extends RMStateStore { return zk; } + @Override + public synchronized void storeOrUpdateAMRMTokenSecretManagerState( + AMRMTokenSecretManagerState amrmTokenSecretManagerState, + boolean isUpdate) { + AMRMTokenSecretManagerState data = + AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState); + byte[] stateData = data.getProto().toByteArray(); + try { + setDataWithRetries(amrmTokenSecretManagerRoot, stateData, -1); + } catch (Exception ex) { + LOG.info("Error storing info for AMRMTokenSecretManager", ex); + notifyStoreOperationFailed(ex); + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java new file mode 100644 index 00000000000..89b4ff0fc7a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/AMRMTokenSecretManagerState.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager; +import org.apache.hadoop.yarn.util.Records; + +/** + * Contains all the state data that needs to be stored persistently + * for {@link AMRMTokenSecretManager} + */ +@Public +@Unstable +public abstract class AMRMTokenSecretManagerState { + public static AMRMTokenSecretManagerState newInstance( + MasterKey currentMasterKey, MasterKey nextMasterKey) { + AMRMTokenSecretManagerState data = + Records.newRecord(AMRMTokenSecretManagerState.class); + data.setCurrentMasterKey(currentMasterKey); + data.setNextMasterKey(nextMasterKey); + return data; + } + + public static AMRMTokenSecretManagerState newInstance( + AMRMTokenSecretManagerState state) { + AMRMTokenSecretManagerState data = + Records.newRecord(AMRMTokenSecretManagerState.class); + data.setCurrentMasterKey(state.getCurrentMasterKey()); + data.setNextMasterKey(state.getNextMasterKey()); + return data; + } + + /** + * {@link AMRMTokenSecretManager} current Master key + */ + @Public + @Unstable + public abstract MasterKey getCurrentMasterKey(); + + @Public + @Unstable + public abstract void setCurrentMasterKey(MasterKey currentMasterKey); + + /** + * {@link AMRMTokenSecretManager} next Master key + */ + @Public + @Unstable + public abstract MasterKey getNextMasterKey(); + + @Public + @Unstable + public abstract void setNextMasterKey(MasterKey nextMasterKey); + + public abstract AMRMTokenSecretManagerStateProto getProto(); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java new file mode 100644 index 00000000000..6ce0c546ada --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/AMRMTokenSecretManagerStatePBImpl.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb; + +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProto; +import org.apache.hadoop.yarn.proto.YarnServerResourceManagerRecoveryProtos.AMRMTokenSecretManagerStateProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; + +public class AMRMTokenSecretManagerStatePBImpl extends AMRMTokenSecretManagerState{ + AMRMTokenSecretManagerStateProto proto = + AMRMTokenSecretManagerStateProto.getDefaultInstance(); + AMRMTokenSecretManagerStateProto.Builder builder = null; + boolean viaProto = false; + + private MasterKey currentMasterKey = null; + private MasterKey nextMasterKey = null; + + public AMRMTokenSecretManagerStatePBImpl() { + builder = AMRMTokenSecretManagerStateProto.newBuilder(); + } + + public AMRMTokenSecretManagerStatePBImpl(AMRMTokenSecretManagerStateProto proto) { + this.proto = proto; + viaProto = true; + } + + public AMRMTokenSecretManagerStateProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToBuilder() { + if (this.currentMasterKey != null) { + builder.setCurrentMasterKey(convertToProtoFormat(this.currentMasterKey)); + } + if (this.nextMasterKey != null) { + builder.setNextMasterKey(convertToProtoFormat(this.nextMasterKey)); + } + } + + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = AMRMTokenSecretManagerStateProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public MasterKey getCurrentMasterKey() { + AMRMTokenSecretManagerStateProtoOrBuilder p = viaProto ? proto : builder; + if (this.currentMasterKey != null) { + return this.currentMasterKey; + } + if (!p.hasCurrentMasterKey()) { + return null; + } + this.currentMasterKey = convertFromProtoFormat(p.getCurrentMasterKey()); + return this.currentMasterKey; + } + + @Override + public void setCurrentMasterKey(MasterKey currentMasterKey) { + maybeInitBuilder(); + if (currentMasterKey == null) + builder.clearCurrentMasterKey(); + this.currentMasterKey = currentMasterKey; + } + + @Override + public MasterKey getNextMasterKey() { + AMRMTokenSecretManagerStateProtoOrBuilder p = viaProto ? proto : builder; + if (this.nextMasterKey != null) { + return this.nextMasterKey; + } + if (!p.hasNextMasterKey()) { + return null; + } + this.nextMasterKey = convertFromProtoFormat(p.getNextMasterKey()); + return this.nextMasterKey; + } + + @Override + public void setNextMasterKey(MasterKey nextMasterKey) { + maybeInitBuilder(); + if (nextMasterKey == null) + builder.clearNextMasterKey(); + this.nextMasterKey = nextMasterKey; + } + + private MasterKeyProto convertToProtoFormat(MasterKey t) { + return ((MasterKeyPBImpl) t).getProto(); + } + + private MasterKeyPBImpl convertFromProtoFormat(MasterKeyProto p) { + return new MasterKeyPBImpl(p); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java index c498b529bf6..a3132bc19e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java @@ -38,6 +38,10 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.security.MasterKeyData; import com.google.common.annotations.VisibleForTesting; @@ -66,6 +70,7 @@ public class AMRMTokenSecretManager extends private final Timer timer; private final long rollingInterval; private final long activationDelay; + private RMContext rmContext; private final Set appAttemptSet = new HashSet(); @@ -73,7 +78,8 @@ public class AMRMTokenSecretManager extends /** * Create an {@link AMRMTokenSecretManager} */ - public AMRMTokenSecretManager(Configuration conf) { + public AMRMTokenSecretManager(Configuration conf, RMContext rmContext) { + this.rmContext = rmContext; this.timer = new Timer(); this.rollingInterval = conf @@ -98,6 +104,11 @@ public class AMRMTokenSecretManager extends public void start() { if (this.currentMasterKey == null) { this.currentMasterKey = createNewMasterKey(); + AMRMTokenSecretManagerState state = + AMRMTokenSecretManagerState.newInstance( + this.currentMasterKey.getMasterKey(), null); + rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state, + false); } this.timer.scheduleAtFixedRate(new MasterKeyRoller(), rollingInterval, rollingInterval); @@ -130,6 +141,12 @@ public class AMRMTokenSecretManager extends try { LOG.info("Rolling master-key for amrm-tokens"); this.nextMasterKey = createNewMasterKey(); + AMRMTokenSecretManagerState state = + AMRMTokenSecretManagerState.newInstance( + this.currentMasterKey.getMasterKey(), + this.nextMasterKey.getMasterKey()); + rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state, + true); this.timer.schedule(new NextKeyActivator(), this.activationDelay); } finally { this.writeLock.unlock(); @@ -225,8 +242,8 @@ public class AMRMTokenSecretManager extends LOG.debug("Trying to retrieve password for " + applicationAttemptId); } if (!appAttemptSet.contains(applicationAttemptId)) { - throw new InvalidToken("Password not found for ApplicationAttempt " - + applicationAttemptId); + throw new InvalidToken(applicationAttemptId + + " not found in AMRMTokenSecretManager."); } if (identifier.getKeyId() == this.currentMasterKey.getMasterKey() .getKeyId()) { @@ -238,9 +255,7 @@ public class AMRMTokenSecretManager extends return createPassword(identifier.getBytes(), this.nextMasterKey.getSecretKey()); } - throw new InvalidToken("Given AMRMToken for application : " - + applicationAttemptId.toString() - + " seems to have been generated illegally."); + throw new InvalidToken("Invalid AMRMToken from " + applicationAttemptId); } finally { this.readLock.unlock(); } @@ -291,4 +306,25 @@ public class AMRMTokenSecretManager extends this.readLock.unlock(); } } + + public void recover(RMState state) { + if (state.getAMRMTokenSecretManagerState() != null) { + // recover the current master key + MasterKey currentKey = + state.getAMRMTokenSecretManagerState().getCurrentMasterKey(); + this.currentMasterKey = + new MasterKeyData(currentKey, createSecretKey(currentKey.getBytes() + .array())); + + // recover the next master key if not null + MasterKey nextKey = + state.getAMRMTokenSecretManagerState().getNextMasterKey(); + if (nextKey != null) { + this.nextMasterKey = + new MasterKeyData(nextKey, createSecretKey(nextKey.getBytes() + .array())); + this.timer.schedule(new NextKeyActivator(), this.activationDelay); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto new file mode 100644 index 00000000000..ae56b9fd346 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +option java_package = "org.apache.hadoop.yarn.proto"; +option java_outer_classname = "YarnServerResourceManagerRecoveryProtos"; +option java_generic_services = true; +option java_generate_equals_and_hash = true; +package hadoop.yarn; + +import "yarn_server_common_protos.proto"; + +message AMRMTokenSecretManagerStateProto { + optional MasterKeyProto current_master_key = 1; + optional MasterKeyProto next_master_key = 2; +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 8966af7e2a5..dc3e9f18178 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -1250,11 +1250,10 @@ public class TestRMRestart { .getEncoded()); // assert AMRMTokenSecretManager also knows about the AMRMToken password - // TODO: fix this on YARN-2211 -// Token amrmToken = loadedAttempt1.getAMRMToken(); -// Assert.assertArrayEquals(amrmToken.getPassword(), -// rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword( -// amrmToken.decodeIdentifier())); + Token amrmToken = loadedAttempt1.getAMRMToken(); + Assert.assertArrayEquals(amrmToken.getPassword(), + rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword( + amrmToken.decodeIdentifier())); rm1.stop(); rm2.stop(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index 04f034818c0..a61f23f5a71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -55,10 +55,12 @@ import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMDTSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -176,8 +178,12 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ TestDispatcher dispatcher = new TestDispatcher(); store.setRMDispatcher(dispatcher); - AMRMTokenSecretManager appTokenMgr = spy( - new AMRMTokenSecretManager(conf)); + RMContext rmContext = mock(RMContext.class); + when(rmContext.getStateStore()).thenReturn(store); + + AMRMTokenSecretManager appTokenMgr = + spy(new AMRMTokenSecretManager(conf, rmContext)); + MasterKeyData masterKeyData = appTokenMgr.createNewMasterKey(); when(appTokenMgr.getMasterKey()).thenReturn(masterKeyData); @@ -576,4 +582,65 @@ public class RMStateStoreTestBase extends ClientBaseWithFixes{ } + public void testAMRMTokenSecretManagerStateStore( + RMStateStoreHelper stateStoreHelper) throws Exception { + System.out.println("Start testing"); + RMStateStore store = stateStoreHelper.getRMStateStore(); + TestDispatcher dispatcher = new TestDispatcher(); + store.setRMDispatcher(dispatcher); + + RMContext rmContext = mock(RMContext.class); + when(rmContext.getStateStore()).thenReturn(store); + Configuration conf = new YarnConfiguration(); + AMRMTokenSecretManager appTokenMgr = + new AMRMTokenSecretManager(conf, rmContext); + + //create and save the first masterkey + MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey(); + + AMRMTokenSecretManagerState state1 = + AMRMTokenSecretManagerState.newInstance( + firstMasterKeyData.getMasterKey(), null); + rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state1, + false); + + // load state + store = stateStoreHelper.getRMStateStore(); + store.setRMDispatcher(dispatcher); + RMState state = store.loadState(); + Assert.assertNotNull(state.getAMRMTokenSecretManagerState()); + Assert.assertEquals(firstMasterKeyData.getMasterKey(), state + .getAMRMTokenSecretManagerState().getCurrentMasterKey()); + Assert.assertNull(state + .getAMRMTokenSecretManagerState().getNextMasterKey()); + + //create and save the second masterkey + MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey(); + AMRMTokenSecretManagerState state2 = + AMRMTokenSecretManagerState + .newInstance(firstMasterKeyData.getMasterKey(), + secondMasterKeyData.getMasterKey()); + rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state2, + true); + + // load state + store = stateStoreHelper.getRMStateStore(); + store.setRMDispatcher(dispatcher); + RMState state_2 = store.loadState(); + Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState()); + Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2 + .getAMRMTokenSecretManagerState().getCurrentMasterKey()); + Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2 + .getAMRMTokenSecretManagerState().getNextMasterKey()); + + // re-create the masterKeyData based on the recovered masterkey + // should have the same secretKey + appTokenMgr.recover(state_2); + Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(), + firstMasterKeyData.getSecretKey()); + Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(), + secondMasterKeyData.getSecretKey()); + + store.close(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java index ea90c3d76f3..f5b3e8a8a67 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -161,6 +160,7 @@ public class TestFSRMStateStore extends RMStateStoreTestBase { testEpoch(fsTester); testAppDeletion(fsTester); testDeleteStore(fsTester); + testAMRMTokenSecretManagerStateStore(fsTester); } finally { cluster.shutdown(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index e56f7757967..1dee533ac05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -123,6 +123,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { testEpoch(zkTester); testAppDeletion(zkTester); testDeleteStore(zkTester); + testAMRMTokenSecretManagerStateStore(zkTester); } private Configuration createHARMConf( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 0fd3c3c5c99..9ea51b120fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -193,7 +193,7 @@ public class TestRMAppTransitions { this.rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, - null, new AMRMTokenSecretManager(conf), + null, new AMRMTokenSecretManager(conf, this.rmContext), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 1de35fcfa98..01a6973e69c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -134,7 +134,8 @@ public class TestRMAppAttemptTransitions { private RMAppAttempt applicationAttempt; private Configuration conf = new Configuration(); - private AMRMTokenSecretManager amRMTokenManager = spy(new AMRMTokenSecretManager(conf)); + private AMRMTokenSecretManager amRMTokenManager = + spy(new AMRMTokenSecretManager(conf, rmContext)); private ClientToAMTokenSecretManagerInRM clientToAMTokenManager = spy(new ClientToAMTokenSecretManagerInRM()); private NMTokenSecretManagerInRM nmTokenManager = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index db28dcaa558..e5486617bb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -86,13 +86,12 @@ public class TestUtils { Configuration conf = new Configuration(); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); - RMContext rmContext = + RMContextImpl rmContext = new RMContextImpl(nullDispatcher, cae, null, null, null, - new AMRMTokenSecretManager(conf), + new AMRMTokenSecretManager(conf, null), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), writer); - return rmContext; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java index b11aadd7912..14385c4c69b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java @@ -184,8 +184,8 @@ public class TestAMRMTokens { // The exception will still have the earlier appAttemptId as it picks it // up from the token. Assert.assertTrue(t.getCause().getMessage().contains( - "Password not found for ApplicationAttempt " + - applicationAttemptId.toString())); + applicationAttemptId.toString() + + " not found in AMRMTokenSecretManager.")); } } finally { From 57d8f829d930091d82ec50b2ff7d327d0301e9d6 Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 25 Jul 2014 22:12:37 +0000 Subject: [PATCH 26/37] HDFS-6755. There is an unnecessary sleep in the code path where DFSOutputStream#close gives up its attempt to contact the namenode (mitdesai21 via cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613522 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 65f0665653a..2b6f1c6750f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -317,6 +317,10 @@ Release 2.6.0 - UNRELEASED HDFS-6701. Make seed optional in NetworkTopology#sortByDistance. (Ashwin Shankar via wang) + HDFS-6755. There is an unnecessary sleep in the code path where + DFSOutputStream#close gives up its attempt to contact the namenode + (mitdesai21 via cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index a7cb92fa269..debf83ca1ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -2136,12 +2136,12 @@ public class DFSOutputStream extends FSOutputSummer throw new IOException(msg); } try { - Thread.sleep(localTimeout); if (retries == 0) { throw new IOException("Unable to close file because the last block" + " does not have enough number of replicas."); } retries--; + Thread.sleep(localTimeout); localTimeout *= 2; if (Time.now() - localstart > 5000) { DFSClient.LOG.info("Could not complete " + src + " retrying..."); From e85a3fecc68b48a3dc9af5daa466a24f3b39545b Mon Sep 17 00:00:00 2001 From: Colin McCabe Date: Fri, 25 Jul 2014 23:56:47 +0000 Subject: [PATCH 27/37] HDFS-6750. The DataNode should use its shared memory segment to mark short-circuit replicas that have been unlinked as stale (cmccabe) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613537 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../server/datanode/ShortCircuitRegistry.java | 29 +++++- .../fsdataset/impl/FsDatasetImpl.java | 8 ++ .../hdfs/shortcircuit/DfsClientShm.java | 21 ++-- .../shortcircuit/DfsClientShmManager.java | 10 +- .../hdfs/shortcircuit/ShortCircuitShm.java | 10 +- .../shortcircuit/TestShortCircuitCache.java | 98 +++++++++++++++++++ 7 files changed, 163 insertions(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2b6f1c6750f..d3748106f8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -321,6 +321,9 @@ Release 2.6.0 - UNRELEASED DFSOutputStream#close gives up its attempt to contact the namenode (mitdesai21 via cmccabe) + HDFS-6750. The DataNode should use its shared memory segment to mark + short-circuit replicas that have been unlinked as stale (cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java index 9dba6a2085d..a252a17855a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ShortCircuitRegistry.java @@ -74,7 +74,7 @@ import com.google.common.collect.HashMultimap; * DN also marks the block's slots as "unanchorable" to prevent additional * clients from initiating these operations in the future. * - * The counterpart fo this class on the client is {@link DfsClientShmManager}. + * The counterpart of this class on the client is {@link DfsClientShmManager}. */ public class ShortCircuitRegistry { public static final Log LOG = LogFactory.getLog(ShortCircuitRegistry.class); @@ -217,7 +217,32 @@ public class ShortCircuitRegistry { } return allowMunlock; } - + + /** + * Invalidate any slot associated with a blockId that we are invalidating + * (deleting) from this DataNode. When a slot is invalid, the DFSClient will + * not use the corresponding replica for new read or mmap operations (although + * existing, ongoing read or mmap operations will complete.) + * + * @param blockId The block ID. + */ + public synchronized void processBlockInvalidation(ExtendedBlockId blockId) { + if (!enabled) return; + final Set affectedSlots = slots.get(blockId); + if (!affectedSlots.isEmpty()) { + final StringBuilder bld = new StringBuilder(); + String prefix = ""; + bld.append("Block ").append(blockId).append(" has been invalidated. "). + append("Marking short-circuit slots as invalid: "); + for (Slot slot : affectedSlots) { + slot.makeInvalid(); + bld.append(prefix).append(slot.toString()); + prefix = ", "; + } + LOG.info(bld.toString()); + } + } + public static class NewShmInfo implements Closeable { public final ShmId shmId; public final FileInputStream stream; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index b068c664fe3..e8a06aec8ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -44,6 +44,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.ExtendedBlockId; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; @@ -1232,8 +1233,15 @@ class FsDatasetImpl implements FsDatasetSpi { } volumeMap.remove(bpid, invalidBlks[i]); } + + // If a DFSClient has the replica in its cache of short-circuit file + // descriptors (and the client is using ShortCircuitShm), invalidate it. + datanode.getShortCircuitRegistry().processBlockInvalidation( + new ExtendedBlockId(invalidBlks[i].getBlockId(), bpid)); + // If the block is cached, start uncaching it. cacheManager.uncacheBlock(bpid, invalidBlks[i].getBlockId()); + // Delete the block asynchronously to make sure we can do it fast enough. // It's ok to unlink the block file before the uncache operation // finishes. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java index 1c9a2e5a742..81cc68da072 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShm.java @@ -32,11 +32,16 @@ import com.google.common.base.Preconditions; * DfsClientShm is a subclass of ShortCircuitShm which is used by the * DfsClient. * When the UNIX domain socket associated with this shared memory segment - * closes unexpectedly, we mark the slots inside this segment as stale. - * ShortCircuitReplica objects that contain stale slots are themselves stale, + * closes unexpectedly, we mark the slots inside this segment as disconnected. + * ShortCircuitReplica objects that contain disconnected slots are stale, * and will not be used to service new reads or mmap operations. * However, in-progress read or mmap operations will continue to proceed. * Once the last slot is deallocated, the segment can be safely munmapped. + * + * Slots may also become stale because the associated replica has been deleted + * on the DataNode. In this case, the DataNode will clear the 'valid' bit. + * The client will then see these slots as stale (see + * #{ShortCircuitReplica#isStale}). */ public class DfsClientShm extends ShortCircuitShm implements DomainSocketWatcher.Handler { @@ -58,7 +63,7 @@ public class DfsClientShm extends ShortCircuitShm * * {@link DfsClientShm#handle} sets this to true. */ - private boolean stale = false; + private boolean disconnected = false; DfsClientShm(ShmId shmId, FileInputStream stream, EndpointShmManager manager, DomainPeer peer) throws IOException { @@ -76,14 +81,14 @@ public class DfsClientShm extends ShortCircuitShm } /** - * Determine if the shared memory segment is stale. + * Determine if the shared memory segment is disconnected from the DataNode. * * This must be called with the DfsClientShmManager lock held. * * @return True if the shared memory segment is stale. */ - public synchronized boolean isStale() { - return stale; + public synchronized boolean isDisconnected() { + return disconnected; } /** @@ -97,8 +102,8 @@ public class DfsClientShm extends ShortCircuitShm public boolean handle(DomainSocket sock) { manager.unregisterShm(getShmId()); synchronized (this) { - Preconditions.checkState(!stale); - stale = true; + Preconditions.checkState(!disconnected); + disconnected = true; boolean hadSlots = false; for (Iterator iter = slotIterator(); iter.hasNext(); ) { Slot slot = iter.next(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java index ca9e8e6e0a5..6dbaf84d269 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/DfsClientShmManager.java @@ -271,12 +271,12 @@ public class DfsClientShmManager implements Closeable { loading = false; finishedLoading.signalAll(); } - if (shm.isStale()) { + if (shm.isDisconnected()) { // If the peer closed immediately after the shared memory segment // was created, the DomainSocketWatcher callback might already have - // fired and marked the shm as stale. In this case, we obviously - // don't want to add the SharedMemorySegment to our list of valid - // not-full segments. + // fired and marked the shm as disconnected. In this case, we + // obviously don't want to add the SharedMemorySegment to our list + // of valid not-full segments. if (LOG.isDebugEnabled()) { LOG.debug(this + ": the UNIX domain socket associated with " + "this short-circuit memory closed before we could make " + @@ -299,7 +299,7 @@ public class DfsClientShmManager implements Closeable { void freeSlot(Slot slot) { DfsClientShm shm = (DfsClientShm)slot.getShm(); shm.unregisterSlot(slot.getSlotIdx()); - if (shm.isStale()) { + if (shm.isDisconnected()) { // Stale shared memory segments should not be tracked here. Preconditions.checkState(!full.containsKey(shm.getShmId())); Preconditions.checkState(!notFull.containsKey(shm.getShmId())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java index d860c8b174c..7b89d0a978d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/shortcircuit/ShortCircuitShm.java @@ -306,6 +306,13 @@ public class ShortCircuitShm { (slotAddress - baseAddress) / BYTES_PER_SLOT); } + /** + * Clear the slot. + */ + void clear() { + unsafe.putLongVolatile(null, this.slotAddress, 0); + } + private boolean isSet(long flag) { long prev = unsafe.getLongVolatile(null, this.slotAddress); return (prev & flag) != 0; @@ -535,6 +542,7 @@ public class ShortCircuitShm { } allocatedSlots.set(idx, true); Slot slot = new Slot(calculateSlotAddress(idx), blockId); + slot.clear(); slot.makeValid(); slots[idx] = slot; if (LOG.isTraceEnabled()) { @@ -583,7 +591,7 @@ public class ShortCircuitShm { Slot slot = new Slot(calculateSlotAddress(slotIdx), blockId); if (!slot.isValid()) { throw new InvalidRequestException(this + ": slot " + slotIdx + - " has not been allocated."); + " is not marked as valid."); } slots[slotIdx] = slot; allocatedSlots.set(slotIdx, true); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java index a2d2bf830f3..ca30e029942 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY; import static org.hamcrest.CoreMatchers.equalTo; import java.io.DataOutputStream; @@ -30,7 +31,9 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.Arrays; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import org.apache.commons.lang.mutable.MutableBoolean; @@ -462,6 +465,7 @@ public class TestShortCircuitCache { } }, 10, 60000); cluster.shutdown(); + sockDir.close(); } @Test(timeout=60000) @@ -516,4 +520,98 @@ public class TestShortCircuitCache { }); cluster.shutdown(); } + + /** + * Test unlinking a file whose blocks we are caching in the DFSClient. + * The DataNode will notify the DFSClient that the replica is stale via the + * ShortCircuitShm. + */ + @Test(timeout=60000) + public void testUnlinkingReplicasInFileDescriptorCache() throws Exception { + BlockReaderTestUtil.enableShortCircuitShmTracing(); + TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); + Configuration conf = createShortCircuitConf( + "testUnlinkingReplicasInFileDescriptorCache", sockDir); + // We don't want the CacheCleaner to time out short-circuit shared memory + // segments during the test, so set the timeout really high. + conf.setLong(DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY, + 1000000000L); + MiniDFSCluster cluster = + new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + cluster.waitActive(); + DistributedFileSystem fs = cluster.getFileSystem(); + final ShortCircuitCache cache = + fs.getClient().getClientContext().getShortCircuitCache(); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + // The ClientShmManager starts off empty. + Assert.assertEquals(0, info.size()); + } + }); + final Path TEST_PATH = new Path("/test_file"); + final int TEST_FILE_LEN = 8193; + final int SEED = 0xFADE0; + DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN, + (short)1, SEED); + byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH); + byte expected[] = DFSTestUtil. + calculateFileContentsFromSeed(SEED, TEST_FILE_LEN); + Assert.assertTrue(Arrays.equals(contents, expected)); + // Loading this file brought the ShortCircuitReplica into our local + // replica cache. + final DatanodeInfo datanode = + new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId()); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) + throws IOException { + Assert.assertTrue(info.get(datanode).full.isEmpty()); + Assert.assertFalse(info.get(datanode).disabled); + Assert.assertEquals(1, info.get(datanode).notFull.values().size()); + DfsClientShm shm = + info.get(datanode).notFull.values().iterator().next(); + Assert.assertFalse(shm.isDisconnected()); + } + }); + // Remove the file whose blocks we just read. + fs.delete(TEST_PATH, false); + + // Wait for the replica to be purged from the DFSClient's cache. + GenericTestUtils.waitFor(new Supplier() { + MutableBoolean done = new MutableBoolean(true); + @Override + public Boolean get() { + try { + done.setValue(true); + cache.getDfsClientShmManager().visit(new Visitor() { + @Override + public void visit(HashMap info) throws IOException { + Assert.assertTrue(info.get(datanode).full.isEmpty()); + Assert.assertFalse(info.get(datanode).disabled); + Assert.assertEquals(1, + info.get(datanode).notFull.values().size()); + DfsClientShm shm = info.get(datanode).notFull.values(). + iterator().next(); + // Check that all slots have been invalidated. + for (Iterator iter = shm.slotIterator(); + iter.hasNext(); ) { + Slot slot = iter.next(); + if (slot.isValid()) { + done.setValue(false); + } + } + } + }); + } catch (IOException e) { + LOG.error("error running visitor", e); + } + return done.booleanValue(); + } + }, 10, 60000); + cluster.shutdown(); + sockDir.close(); + } } From 9cfde45b45d6ca0f9665510d687dcd19f4ae3ef3 Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sat, 26 Jul 2014 00:48:42 +0000 Subject: [PATCH 28/37] HADOOP-10896. Update compatibility doc to capture visibility of un-annotated classes/ methods. (kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613543 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/site/apt/Compatibility.apt.vm | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 419d45c6831..34bd95103cd 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -621,6 +621,9 @@ Release 2.5.0 - UNRELEASED HADOOP-10782. Fix typo in DataChecksum class. (Jingguo Yao via suresh) + HADOOP-10896. Update compatibility doc to capture visibility of + un-annotated classes/ methods. (kasha) + OPTIMIZATIONS HADOOP-10674. Improve PureJavaCrc32 performance and use java.util.zip.CRC32 diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm index ecf6e75f3bc..98d1f57166f 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm @@ -72,10 +72,13 @@ Apache Hadoop Compatibility * Private-Stable APIs can change across major releases, but not within a major release. + * Classes not annotated are implicitly "Private". Class members not + annotated inherit the annotations of the enclosing class. + * Note: APIs generated from the proto files need to be compatible for -rolling-upgrades. See the section on wire-compatibility for more details. The -compatibility policies for APIs and wire-communication need to go -hand-in-hand to address this. + rolling-upgrades. See the section on wire-compatibility for more details. + The compatibility policies for APIs and wire-communication need to go + hand-in-hand to address this. ** Semantic compatibility From b0c2c856192e71145c012504511fad172436131b Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sat, 26 Jul 2014 01:29:25 +0000 Subject: [PATCH 29/37] YARN-1726. ResourceSchedulerWrapper broken due to AbstractYarnScheduler. (Wei Yan via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613547 13f79535-47bb-0310-9956-ffa450edef68 --- .../yarn/sls/appmaster/AMSimulator.java | 69 +++++++++---------- .../yarn/sls/appmaster/MRAMSimulator.java | 5 +- .../yarn/sls/nodemanager/NMSimulator.java | 68 ++++++++++-------- .../scheduler/ResourceSchedulerWrapper.java | 40 +++++++---- .../sls/scheduler/SLSCapacityScheduler.java | 22 +++--- .../hadoop/yarn/sls/scheduler/TaskRunner.java | 16 ++--- .../apache/hadoop/yarn/sls/TestSLSRunner.java | 30 +++++++- hadoop-yarn-project/CHANGES.txt | 3 + 8 files changed, 149 insertions(+), 104 deletions(-) diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java index 5af4eaa2de4..2272e3ed911 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/AMSimulator.java @@ -63,6 +63,8 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.Records; import org.apache.log4j.Logger; @@ -133,8 +135,7 @@ public abstract class AMSimulator extends TaskRunner.Task { * register with RM */ @Override - public void firstStep() - throws YarnException, IOException, InterruptedException { + public void firstStep() throws Exception { simulateStartTimeMS = System.currentTimeMillis() - SLSRunner.getRunner().getStartTimeMS(); @@ -149,8 +150,7 @@ public abstract class AMSimulator extends TaskRunner.Task { } @Override - public void middleStep() - throws InterruptedException, YarnException, IOException { + public void middleStep() throws Exception { // process responses in the queue processResponseQueue(); @@ -162,7 +162,7 @@ public abstract class AMSimulator extends TaskRunner.Task { } @Override - public void lastStep() { + public void lastStep() throws Exception { LOG.info(MessageFormat.format("Application {0} is shutting down.", appId)); // unregister tracking if (isTracked) { @@ -173,26 +173,19 @@ public abstract class AMSimulator extends TaskRunner.Task { .newRecordInstance(FinishApplicationMasterRequest.class); finishAMRequest.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); - try { - UserGroupInformation ugi = - UserGroupInformation.createRemoteUser(appAttemptId.toString()); - Token token = - rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId()) - .getRMAppAttempt(appAttemptId).getAMRMToken(); - ugi.addTokenIdentifier(token.decodeIdentifier()); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - rm.getApplicationMasterService() - .finishApplicationMaster(finishAMRequest); - return null; - } - }); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { - e.printStackTrace(); - } + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser(appAttemptId.toString()); + Token token = rm.getRMContext().getRMApps().get(appId) + .getRMAppAttempt(appAttemptId).getAMRMToken(); + ugi.addTokenIdentifier(token.decodeIdentifier()); + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + rm.getApplicationMasterService() + .finishApplicationMaster(finishAMRequest); + return null; + } + }); simulateFinishTimeMS = System.currentTimeMillis() - SLSRunner.getRunner().getStartTimeMS(); @@ -230,11 +223,9 @@ public abstract class AMSimulator extends TaskRunner.Task { return createAllocateRequest(ask, new ArrayList()); } - protected abstract void processResponseQueue() - throws InterruptedException, YarnException, IOException; + protected abstract void processResponseQueue() throws Exception; - protected abstract void sendContainerRequest() - throws YarnException, IOException, InterruptedException; + protected abstract void sendContainerRequest() throws Exception; protected abstract void checkStop(); @@ -280,11 +271,18 @@ public abstract class AMSimulator extends TaskRunner.Task { // waiting until application ACCEPTED RMApp app = rm.getRMContext().getRMApps().get(appId); while(app.getState() != RMAppState.ACCEPTED) { - Thread.sleep(50); + Thread.sleep(10); } - appAttemptId = rm.getRMContext().getRMApps().get(appId) - .getCurrentAppAttempt().getAppAttemptId(); + // Waiting until application attempt reach LAUNCHED + // "Unmanaged AM must register after AM attempt reaches LAUNCHED state" + this.appAttemptId = rm.getRMContext().getRMApps().get(appId) + .getCurrentAppAttempt().getAppAttemptId(); + RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId) + .getCurrentAppAttempt(); + while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) { + Thread.sleep(10); + } } private void registerAM() @@ -297,10 +295,9 @@ public abstract class AMSimulator extends TaskRunner.Task { amRegisterRequest.setTrackingUrl("localhost:1000"); UserGroupInformation ugi = - UserGroupInformation.createRemoteUser(appAttemptId.toString()); - Token token = - rm.getRMContext().getRMApps().get(appAttemptId.getApplicationId()) - .getRMAppAttempt(appAttemptId).getAMRMToken(); + UserGroupInformation.createRemoteUser(appAttemptId.toString()); + Token token = rm.getRMContext().getRMApps().get(appId) + .getRMAppAttempt(appAttemptId).getAMRMToken(); ugi.addTokenIdentifier(token.decodeIdentifier()); ugi.doAs( diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java index d24510ba6fd..fb702059ade 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/appmaster/MRAMSimulator.java @@ -145,8 +145,7 @@ public class MRAMSimulator extends AMSimulator { } @Override - public void firstStep() - throws YarnException, IOException, InterruptedException { + public void firstStep() throws Exception { super.firstStep(); requestAMContainer(); @@ -390,7 +389,7 @@ public class MRAMSimulator extends AMSimulator { } @Override - public void lastStep() { + public void lastStep() throws Exception { super.lastStep(); // clear data structures diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java index 4112685e152..0947ba8a18b 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NMSimulator.java @@ -27,6 +27,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.DelayQueue; +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -107,12 +108,12 @@ public class NMSimulator extends TaskRunner.Task { } @Override - public void firstStep() throws YarnException, IOException { + public void firstStep() { // do nothing } @Override - public void middleStep() { + public void middleStep() throws Exception { // we check the lifetime for each running containers ContainerSimulator cs = null; synchronized(completedContainerList) { @@ -136,37 +137,31 @@ public class NMSimulator extends TaskRunner.Task { ns.setResponseId(RESPONSE_ID ++); ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0)); beatRequest.setNodeStatus(ns); - try { - NodeHeartbeatResponse beatResponse = - rm.getResourceTrackerService().nodeHeartbeat(beatRequest); - if (! beatResponse.getContainersToCleanup().isEmpty()) { - // remove from queue - synchronized(releasedContainerList) { - for (ContainerId containerId : beatResponse.getContainersToCleanup()){ - if (amContainerList.contains(containerId)) { - // AM container (not killed?, only release) - synchronized(amContainerList) { - amContainerList.remove(containerId); - } - LOG.debug(MessageFormat.format("NodeManager {0} releases " + - "an AM ({1}).", node.getNodeID(), containerId)); - } else { - cs = runningContainers.remove(containerId); - containerQueue.remove(cs); - releasedContainerList.add(containerId); - LOG.debug(MessageFormat.format("NodeManager {0} releases a " + - "container ({1}).", node.getNodeID(), containerId)); + NodeHeartbeatResponse beatResponse = + rm.getResourceTrackerService().nodeHeartbeat(beatRequest); + if (! beatResponse.getContainersToCleanup().isEmpty()) { + // remove from queue + synchronized(releasedContainerList) { + for (ContainerId containerId : beatResponse.getContainersToCleanup()){ + if (amContainerList.contains(containerId)) { + // AM container (not killed?, only release) + synchronized(amContainerList) { + amContainerList.remove(containerId); } + LOG.debug(MessageFormat.format("NodeManager {0} releases " + + "an AM ({1}).", node.getNodeID(), containerId)); + } else { + cs = runningContainers.remove(containerId); + containerQueue.remove(cs); + releasedContainerList.add(containerId); + LOG.debug(MessageFormat.format("NodeManager {0} releases a " + + "container ({1}).", node.getNodeID(), containerId)); } } } - if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) { - lastStep(); - } - } catch (YarnException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); + } + if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) { + lastStep(); } } @@ -262,4 +257,19 @@ public class NMSimulator extends TaskRunner.Task { completedContainerList.add(containerId); } } + + @VisibleForTesting + Map getRunningContainers() { + return runningContainers; + } + + @VisibleForTesting + List getAMContainers() { + return amContainerList; + } + + @VisibleForTesting + List getCompletedContainers() { + return completedContainerList; + } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java index 0bd0c87d2d0..6ccae98bd86 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport; @@ -101,7 +102,6 @@ final public class ResourceSchedulerWrapper private static final String EOL = System.getProperty("line.separator"); private static final int SAMPLING_SIZE = 60; private ScheduledExecutorService pool; - private RMContext rmContext; // counters for scheduler allocate/handle operations private Counter schedulerAllocateCounter; private Counter schedulerHandleCounter; @@ -576,7 +576,7 @@ final public class ResourceSchedulerWrapper new Gauge() { @Override public Integer getValue() { - if(scheduler == null || scheduler.getRootQueueMetrics() == null) { + if (scheduler == null || scheduler.getRootQueueMetrics() == null) { return 0; } else { return scheduler.getRootQueueMetrics().getAppsRunning(); @@ -723,17 +723,18 @@ final public class ResourceSchedulerWrapper public void addAMRuntime(ApplicationId appId, long traceStartTimeMS, long traceEndTimeMS, long simulateStartTimeMS, long simulateEndTimeMS) { - - try { - // write job runtime information - StringBuilder sb = new StringBuilder(); - sb.append(appId).append(",").append(traceStartTimeMS).append(",") - .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) - .append(",").append(simulateEndTimeMS); - jobRuntimeLogBW.write(sb.toString() + EOL); - jobRuntimeLogBW.flush(); - } catch (IOException e) { - e.printStackTrace(); + if (metricsON) { + try { + // write job runtime information + StringBuilder sb = new StringBuilder(); + sb.append(appId).append(",").append(traceStartTimeMS).append(",") + .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) + .append(",").append(simulateEndTimeMS); + jobRuntimeLogBW.write(sb.toString() + EOL); + jobRuntimeLogBW.flush(); + } catch (IOException e) { + e.printStackTrace(); + } } } @@ -919,4 +920,17 @@ final public class ResourceSchedulerWrapper public Resource getClusterResource() { return null; } + + @Override + public synchronized List getTransferredContainers( + ApplicationAttemptId currentAttempt) { + return new ArrayList(); + } + + @Override + public Map> + getSchedulerApplications() { + return new HashMap>(); + } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java index 44a872198d6..06addfb28fd 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java @@ -729,16 +729,18 @@ public class SLSCapacityScheduler extends CapacityScheduler implements long traceStartTimeMS, long traceEndTimeMS, long simulateStartTimeMS, long simulateEndTimeMS) { - try { - // write job runtime information - StringBuilder sb = new StringBuilder(); - sb.append(appId).append(",").append(traceStartTimeMS).append(",") - .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) - .append(",").append(simulateEndTimeMS); - jobRuntimeLogBW.write(sb.toString() + EOL); - jobRuntimeLogBW.flush(); - } catch (IOException e) { - e.printStackTrace(); + if (metricsON) { + try { + // write job runtime information + StringBuilder sb = new StringBuilder(); + sb.append(appId).append(",").append(traceStartTimeMS).append(",") + .append(traceEndTimeMS).append(",").append(simulateStartTimeMS) + .append(",").append(simulateEndTimeMS); + jobRuntimeLogBW.write(sb.toString() + EOL); + jobRuntimeLogBW.flush(); + } catch (IOException e) { + e.printStackTrace(); + } } } diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java index c936dd93180..d35290428c7 100644 --- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java +++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/TaskRunner.java @@ -99,12 +99,10 @@ public class TaskRunner { } else { lastStep(); } - } catch (YarnException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } catch (InterruptedException e) { + } catch (Exception e) { e.printStackTrace(); + Thread.getDefaultUncaughtExceptionHandler() + .uncaughtException(Thread.currentThread(), e); } } @@ -124,13 +122,11 @@ public class TaskRunner { } - public abstract void firstStep() - throws YarnException, IOException, InterruptedException; + public abstract void firstStep() throws Exception; - public abstract void middleStep() - throws YarnException, InterruptedException, IOException; + public abstract void middleStep() throws Exception; - public abstract void lastStep() throws YarnException; + public abstract void lastStep() throws Exception; public void setEndTime(long et) { endTime = et; diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java index b05972734f2..9da8ef34a20 100644 --- a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/TestSLSRunner.java @@ -18,10 +18,13 @@ package org.apache.hadoop.yarn.sls; -import org.apache.commons.io.FileUtils; +import org.junit.Assert; import org.junit.Test; import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.UUID; public class TestSLSRunner { @@ -30,6 +33,15 @@ public class TestSLSRunner { @SuppressWarnings("all") public void testSimulatorRunning() throws Exception { File tempDir = new File("target", UUID.randomUUID().toString()); + final List exceptionList = + Collections.synchronizedList(new ArrayList()); + + Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + exceptionList.add(e); + } + }); // start the simulator File slsOutputDir = new File(tempDir.getAbsolutePath() + "/slsoutput/"); @@ -38,8 +50,20 @@ public class TestSLSRunner { "-output", slsOutputDir.getAbsolutePath()}; SLSRunner.main(args); - // wait for 45 seconds before stop - Thread.sleep(45 * 1000); + // wait for 20 seconds before stop + int count = 20; + while (count >= 0) { + Thread.sleep(1000); + + if (! exceptionList.isEmpty()) { + SLSRunner.getRunner().stop(); + Assert.fail("TestSLSRunner catched exception from child thread " + + "(TaskRunner.Task): " + exceptionList.get(0).getMessage()); + break; + } + count--; + } + SLSRunner.getRunner().stop(); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e6fbea9350e..3d3429a674e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -444,6 +444,9 @@ Release 2.5.0 - UNRELEASED YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha) + YARN-1726. ResourceSchedulerWrapper broken due to AbstractYarnScheduler. + (Wei Yan via kasha) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES From 5d4677b57b05c1690ab0a825869df9a4f6c173bc Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Sat, 26 Jul 2014 01:51:35 +0000 Subject: [PATCH 30/37] YARN-1796. container-executor shouldn't require o-r permissions. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613548 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 ++ .../container-executor/impl/container-executor.c | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3d3429a674e..fb164f8d303 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -106,6 +106,8 @@ Release 2.6.0 - UNRELEASED YARN-2147. client lacks delegation token exception details when application submit fails (Chen He via jlowe) + YARN-1796. container-executor shouldn't require o-r permissions (atm) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 9387ba4f1e0..16ede961edc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -111,16 +111,16 @@ int check_executor_permissions(char *executable_file) { return -1; } - // check others do not have read/write/execute permissions - if ((filestat.st_mode & S_IROTH) == S_IROTH || (filestat.st_mode & S_IWOTH) - == S_IWOTH || (filestat.st_mode & S_IXOTH) == S_IXOTH) { + // check others do not have write/execute permissions + if ((filestat.st_mode & S_IWOTH) == S_IWOTH || + (filestat.st_mode & S_IXOTH) == S_IXOTH) { fprintf(LOGFILE, - "The container-executor binary should not have read or write or" - " execute for others.\n"); + "The container-executor binary should not have write or execute " + "for others.\n"); return -1; } - // Binary should be setuid/setgid executable + // Binary should be setuid executable if ((filestat.st_mode & S_ISUID) == 0) { fprintf(LOGFILE, "The container-executor binary should be set setuid.\n"); return -1; From 4625792c20d8e297a34fe9599d9b474a6b1cc77a Mon Sep 17 00:00:00 2001 From: Karthik Kambatla Date: Sat, 26 Jul 2014 02:03:18 +0000 Subject: [PATCH 31/37] YARN-1726. Add missing files. ResourceSchedulerWrapper broken due to AbstractYarnScheduler. (Wei Yan via kasha) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613552 13f79535-47bb-0310-9956-ffa450edef68 --- .../yarn/sls/appmaster/TestAMSimulator.java | 86 +++++++++++++++ .../yarn/sls/nodemanager/TestNMSimulator.java | 100 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java create mode 100644 hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java new file mode 100644 index 00000000000..83482c33686 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/appmaster/TestAMSimulator.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.sls.appmaster; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.sls.scheduler.ContainerSimulator; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class TestAMSimulator { + private ResourceManager rm; + private YarnConfiguration conf; + + @Before + public void setup() { + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_SCHEDULER, + "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper"); + conf.set(SLSConfiguration.RM_SCHEDULER, + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false); + rm = new ResourceManager(); + rm.init(conf); + rm.start(); + } + + class MockAMSimulator extends AMSimulator { + @Override + protected void processResponseQueue() + throws InterruptedException, YarnException, IOException { + } + + @Override + protected void sendContainerRequest() + throws YarnException, IOException, InterruptedException { + } + + @Override + protected void checkStop() { + } + } + + @Test + public void testAMSimulator() throws Exception { + // Register one app + MockAMSimulator app = new MockAMSimulator(); + List containers = new ArrayList(); + app.init(1, 1000, containers, rm, null, 0, 1000000l, "user1", "default", + false, "app1"); + app.firstStep(); + Assert.assertEquals(1, rm.getRMContext().getRMApps().size()); + Assert.assertNotNull(rm.getRMContext().getRMApps().get(app.appId)); + + // Finish this app + app.lastStep(); + } + + @After + public void tearDown() { + rm.stop(); + } +} \ No newline at end of file diff --git a/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java new file mode 100644 index 00000000000..84be2313cf4 --- /dev/null +++ b/hadoop-tools/hadoop-sls/src/test/java/org/apache/hadoop/yarn/sls/nodemanager/TestNMSimulator.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.sls.nodemanager; + +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class TestNMSimulator { + private final int GB = 1024; + private ResourceManager rm; + private YarnConfiguration conf; + + @Before + public void setup() { + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_SCHEDULER, + "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper"); + conf.set(SLSConfiguration.RM_SCHEDULER, + "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"); + conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false); + rm = new ResourceManager(); + rm.init(conf); + rm.start(); + } + + @Test + public void testNMSimulator() throws Exception { + // Register one node + NMSimulator node1 = new NMSimulator(); + node1.init("rack1/node1", GB * 10, 10, 0, 1000, rm); + node1.middleStep(); + + Assert.assertEquals(1, rm.getResourceScheduler().getNumClusterNodes()); + Assert.assertEquals(GB * 10, + rm.getResourceScheduler().getRootQueueMetrics().getAvailableMB()); + Assert.assertEquals(10, + rm.getResourceScheduler().getRootQueueMetrics() + .getAvailableVirtualCores()); + + // Allocate one container on node1 + ContainerId cId1 = newContainerId(1, 1, 1); + Container container1 = Container.newInstance(cId1, null, null, + Resources.createResource(GB, 1), null, null); + node1.addNewContainer(container1, 100000l); + Assert.assertTrue("Node1 should have one running container.", + node1.getRunningContainers().containsKey(cId1)); + + // Allocate one AM container on node1 + ContainerId cId2 = newContainerId(2, 1, 1); + Container container2 = Container.newInstance(cId2, null, null, + Resources.createResource(GB, 1), null, null); + node1.addNewContainer(container2, -1l); + Assert.assertTrue("Node1 should have one running AM container", + node1.getAMContainers().contains(cId2)); + + // Remove containers + node1.cleanupContainer(cId1); + Assert.assertTrue("Container1 should be removed from Node1.", + node1.getCompletedContainers().contains(cId1)); + node1.cleanupContainer(cId2); + Assert.assertFalse("Container2 should be removed from Node1.", + node1.getAMContainers().contains(cId2)); + } + + private ContainerId newContainerId(int appId, int appAttemptId, int cId) { + return BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(System.currentTimeMillis(), appId), + appAttemptId), cId); + } + + @After + public void tearDown() throws Exception { + rm.stop(); + } +} From 9bce3eca42b40f9b1f9fc23fc4418ed23b7beda5 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Sat, 26 Jul 2014 04:18:28 +0000 Subject: [PATCH 32/37] HDFS-6749. FSNamesystem methods should call resolvePath. Contributed by Charles Lamb. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613561 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/namenode/FSNamesystem.java | 8 ++++++++ .../hadoop/hdfs/server/namenode/TestINodeFile.java | 14 ++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d3748106f8c..eb524b209df 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -383,6 +383,9 @@ Release 2.6.0 - UNRELEASED HDFS-6752. Avoid Address bind errors in TestDatanodeConfig#testMemlockLimit (vinayakumarb) + HDFS-6749. FSNamesystem methods should call resolvePath. + (Charles Lamb via cnauroth) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 6ce6a70ce2b..a705f17dd27 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3720,8 +3720,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, StandbyException, IOException { FSPermissionChecker pc = getPermissionChecker(); checkOperation(OperationCategory.READ); + byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); try { + src = FSDirectory.resolvePath(src, pathComponents, dir); checkOperation(OperationCategory.READ); if (isPermissionEnabled) { checkTraverse(pc, src); @@ -8183,9 +8185,11 @@ public class FSNamesystem implements Namesystem, FSClusterStats, nnConf.checkAclsConfigFlag(); FSPermissionChecker pc = getPermissionChecker(); checkOperation(OperationCategory.READ); + byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); try { checkOperation(OperationCategory.READ); + src = FSDirectory.resolvePath(src, pathComponents, dir); if (isPermissionEnabled) { checkPermission(pc, src, false, null, null, null, null); } @@ -8288,8 +8292,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } checkOperation(OperationCategory.READ); + byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); try { + src = FSDirectory.resolvePath(src, pathComponents, dir); checkOperation(OperationCategory.READ); if (isPermissionEnabled) { checkPathAccess(pc, src, FsAction.READ); @@ -8333,8 +8339,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, nnConf.checkXAttrsConfigFlag(); final FSPermissionChecker pc = getPermissionChecker(); checkOperation(OperationCategory.READ); + byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); readLock(); try { + src = FSDirectory.resolvePath(src, pathComponents, dir); checkOperation(OperationCategory.READ); if (isPermissionEnabled) { /* To access xattr names, you need EXECUTE in the owning directory. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java index 3cee355840c..704bc1669d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java @@ -521,6 +521,7 @@ public class TestINodeFile { Configuration conf = new Configuration(); conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); @@ -568,6 +569,19 @@ public class TestINodeFile { // ClientProtocol#getPreferredBlockSize assertEquals(testFileBlockSize, nnRpc.getPreferredBlockSize(testFileInodePath.toString())); + + /* + * HDFS-6749 added missing calls to FSDirectory.resolvePath in the + * following four methods. The calls below ensure that + * /.reserved/.inodes paths work properly. No need to check return + * values as these methods are tested elsewhere. + */ + { + fs.isFileClosed(testFileInodePath); + fs.getAclStatus(testFileInodePath); + fs.getXAttrs(testFileInodePath); + fs.listXAttrs(testFileInodePath); + } // symbolic link related tests From 549bcc2c02983086ee6694982d5f3503f5f4517f Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Sun, 27 Jul 2014 01:37:51 +0000 Subject: [PATCH 33/37] MAPREDUCE-6002. Made MR task avoid reporting error to AM when the task process is shutting down. Contributed by Wangda Tan. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613743 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapred/LocalContainerLauncher.java | 18 ++++++++++----- .../org/apache/hadoop/mapred/YarnChild.java | 22 +++++++++++++------ .../java/org/apache/hadoop/mapred/Task.java | 6 +++++ 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c8a83bf64e5..5760cef3060 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -325,6 +325,9 @@ Release 2.5.0 - UNRELEASED MAPREDUCE-5952. LocalContainerLauncher#renameMapOutputForReduce incorrectly assumes a single dir for mapOutIndex. (Gera Shegalov via kasha) + MAPREDUCE-6002. Made MR task avoid reporting error to AM when the task process + is shutting down. (Wangda Tan via zjshen) + Release 2.4.1 - 2014-06-23 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index c7898ed966f..218ac835d27 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -31,6 +31,7 @@ import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import com.google.common.annotations.VisibleForTesting; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FSError; @@ -57,6 +58,7 @@ import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; @@ -406,7 +408,9 @@ public class LocalContainerLauncher extends AbstractService implements } catch (FSError e) { LOG.fatal("FSError from child", e); // umbilical: MRAppMaster creates (taskAttemptListener), passes to us - umbilical.fsError(classicAttemptID, e.getMessage()); + if (!ShutdownHookManager.get().isShutdownInProgress()) { + umbilical.fsError(classicAttemptID, e.getMessage()); + } throw new RuntimeException(); } catch (Exception exception) { @@ -429,11 +433,13 @@ public class LocalContainerLauncher extends AbstractService implements } catch (Throwable throwable) { LOG.fatal("Error running local (uberized) 'child' : " + StringUtils.stringifyException(throwable)); - Throwable tCause = throwable.getCause(); - String cause = (tCause == null) - ? throwable.getMessage() - : StringUtils.stringifyException(tCause); - umbilical.fatalError(classicAttemptID, cause); + if (!ShutdownHookManager.get().isShutdownInProgress()) { + Throwable tCause = throwable.getCause(); + String cause = + (tCause == null) ? throwable.getMessage() : StringUtils + .stringifyException(tCause); + umbilical.fatalError(classicAttemptID, cause); + } throw new RuntimeException(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 9212bfd154a..4ba1991ed9b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -56,6 +56,7 @@ import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DiskChecker.DiskErrorException; +import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.api.ApplicationConstants; @@ -176,7 +177,9 @@ class YarnChild { }); } catch (FSError e) { LOG.fatal("FSError from child", e); - umbilical.fsError(taskid, e.getMessage()); + if (!ShutdownHookManager.get().isShutdownInProgress()) { + umbilical.fsError(taskid, e.getMessage()); + } } catch (Exception exception) { LOG.warn("Exception running child : " + StringUtils.stringifyException(exception)); @@ -201,17 +204,22 @@ class YarnChild { } // Report back any failures, for diagnostic purposes if (taskid != null) { - umbilical.fatalError(taskid, StringUtils.stringifyException(exception)); + if (!ShutdownHookManager.get().isShutdownInProgress()) { + umbilical.fatalError(taskid, + StringUtils.stringifyException(exception)); + } } } catch (Throwable throwable) { LOG.fatal("Error running child : " + StringUtils.stringifyException(throwable)); if (taskid != null) { - Throwable tCause = throwable.getCause(); - String cause = tCause == null - ? throwable.getMessage() - : StringUtils.stringifyException(tCause); - umbilical.fatalError(taskid, cause); + if (!ShutdownHookManager.get().isShutdownInProgress()) { + Throwable tCause = throwable.getCause(); + String cause = + tCause == null ? throwable.getMessage() : StringUtils + .stringifyException(tCause); + umbilical.fatalError(taskid, cause); + } } } finally { RPC.stopProxy(umbilical); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java index 4815f191f7e..3a4c513f3f1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java @@ -66,6 +66,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.util.Progress; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.util.StringInterner; import org.apache.hadoop.util.StringUtils; @@ -322,6 +323,11 @@ abstract public class Task implements Writable, Configurable { protected void reportFatalError(TaskAttemptID id, Throwable throwable, String logMsg) { LOG.fatal(logMsg); + + if (ShutdownHookManager.get().isShutdownInProgress()) { + return; + } + Throwable tCause = throwable.getCause(); String cause = tCause == null ? StringUtils.stringifyException(throwable) From d6532d3a77abe5a9b1760c1e26a899d49aa661dc Mon Sep 17 00:00:00 2001 From: Zhijie Shen Date: Sun, 27 Jul 2014 17:55:06 +0000 Subject: [PATCH 34/37] YARN-2247. Made RM web services authenticate users via kerberos and delegation token. Contributed by Varun Vasudev. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1613821 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 11 + .../src/main/resources/yarn-default.xml | 9 + .../security/http/RMAuthenticationFilter.java | 63 ++++ .../RMAuthenticationFilterInitializer.java | 121 ++++++ .../resourcemanager/ResourceManager.java | 61 +++ .../security/RMAuthenticationHandler.java | 157 ++++++++ .../resourcemanager/webapp/RMWebServices.java | 24 +- ...ServicesDelegationTokenAuthentication.java | 354 ++++++++++++++++++ .../webapp/TestRMWebappAuthentication.java | 272 ++++++++++++++ .../src/site/apt/ResourceManagerRest.apt.vm | 21 ++ 11 files changed, 1095 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index fb164f8d303..c0405106a01 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -147,6 +147,9 @@ Release 2.5.0 - UNRELEASED YARN-2233. Implemented ResourceManager web-services to create, renew and cancel delegation tokens. (Varun Vasudev via vinodkv) + YARN-2247. Made RM web services authenticate users via kerberos and delegation + token. (Varun Vasudev via zjshen) + IMPROVEMENTS YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 59e108a6fb1..ab6b20e574e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -263,6 +263,17 @@ public class YarnConfiguration extends Configuration { public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY = RM_PREFIX + "webapp.spnego-keytab-file"; + /** + * Flag to enable override of the default kerberos authentication filter with + * the RM authentication filter to allow authentication using delegation + * tokens(fallback to kerberos if the tokens are missing). Only applicable + * when the http authentication type is kerberos. + */ + public static final String RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = RM_PREFIX + + "webapp.delegation-token-auth-filter.enabled"; + public static final boolean DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER = + true; + /** How long to wait until a container is considered dead.*/ public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS = RM_PREFIX + "rm.container-allocation.expiry-interval-ms"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 8bc49e69769..edc2f8cab61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -194,6 +194,15 @@ /etc/krb5.keytab + + Flag to enable override of the default kerberos authentication + filter with the RM authentication filter to allow authentication using + delegation tokens(fallback to kerberos if the tokens are missing). Only + applicable when the http authentication type is kerberos. + yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled + true + + How long to wait until a node manager is considered dead. yarn.nm.liveness-monitor.expiry-interval-ms diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java new file mode 100644 index 00000000000..651b5b0c762 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.security.http; + +import java.util.Properties; + +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; + +@Private +@Unstable +public class RMAuthenticationFilter extends AuthenticationFilter { + + public static final String AUTH_HANDLER_PROPERTY = + "yarn.resourcemanager.authentication-handler"; + + public RMAuthenticationFilter() { + } + + @Override + protected Properties getConfiguration(String configPrefix, + FilterConfig filterConfig) throws ServletException { + + // In yarn-site.xml, we can simply set type to "kerberos". However, we need + // to replace the name here to use the customized Kerberos + DT service + // instead of the standard Kerberos handler. + + Properties properties = super.getConfiguration(configPrefix, filterConfig); + String yarnAuthHandler = properties.getProperty(AUTH_HANDLER_PROPERTY); + if (yarnAuthHandler == null || yarnAuthHandler.isEmpty()) { + // if http auth type is simple, the default authentication filter + // will handle it, else throw an exception + if (!properties.getProperty(AUTH_TYPE).equals("simple")) { + throw new ServletException("Authentication handler class is empty"); + } + } + if (properties.getProperty(AUTH_TYPE).equalsIgnoreCase("kerberos")) { + properties.setProperty(AUTH_TYPE, yarnAuthHandler); + } + return properties; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java new file mode 100644 index 00000000000..2227833e7cf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.security.http; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.http.FilterContainer; +import org.apache.hadoop.http.FilterInitializer; +import org.apache.hadoop.http.HttpServer2; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; + +@Unstable +public class RMAuthenticationFilterInitializer extends FilterInitializer { + + String configPrefix; + String signatureSecretFileProperty; + String kerberosPrincipalProperty; + String cookiePath; + + public RMAuthenticationFilterInitializer() { + this.configPrefix = "hadoop.http.authentication."; + this.signatureSecretFileProperty = + AuthenticationFilter.SIGNATURE_SECRET + ".file"; + this.kerberosPrincipalProperty = KerberosAuthenticationHandler.PRINCIPAL; + this.cookiePath = "/"; + } + + protected Map createFilterConfig(Configuration conf) { + Map filterConfig = new HashMap(); + + // setting the cookie path to root '/' so it is used for all resources. + filterConfig.put(AuthenticationFilter.COOKIE_PATH, cookiePath); + + for (Map.Entry entry : conf) { + String name = entry.getKey(); + if (name.startsWith(configPrefix)) { + String value = conf.get(name); + name = name.substring(configPrefix.length()); + filterConfig.put(name, value); + } + } + + String signatureSecretFile = filterConfig.get(signatureSecretFileProperty); + if (signatureSecretFile != null) { + Reader reader = null; + try { + StringBuilder secret = new StringBuilder(); + reader = + new InputStreamReader(new FileInputStream(signatureSecretFile), + "UTF-8"); + int c = reader.read(); + while (c > -1) { + secret.append((char) c); + c = reader.read(); + } + filterConfig.put(AuthenticationFilter.SIGNATURE_SECRET, + secret.toString()); + } catch (IOException ex) { + // if running in non-secure mode, this filter only gets added + // because the user has not setup his own filter so just generate + // a random secret. in secure mode, the user needs to setup security + if (UserGroupInformation.isSecurityEnabled()) { + throw new RuntimeException( + "Could not read HTTP signature secret file: " + signatureSecretFile); + } + } finally { + IOUtils.closeQuietly(reader); + } + } + + // Resolve _HOST into bind address + String bindAddress = conf.get(HttpServer2.BIND_ADDRESS); + String principal = filterConfig.get(kerberosPrincipalProperty); + if (principal != null) { + try { + principal = SecurityUtil.getServerPrincipal(principal, bindAddress); + } catch (IOException ex) { + throw new RuntimeException( + "Could not resolve Kerberos principal name: " + ex.toString(), ex); + } + filterConfig.put(KerberosAuthenticationHandler.PRINCIPAL, principal); + } + return filterConfig; + } + + @Override + public void initFilter(FilterContainer container, Configuration conf) { + + Map filterConfig = createFilterConfig(conf); + container.addFilter("YARNAuthenticationFilter", + RMAuthenticationFilter.class.getName(), filterConfig); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 7dceda249da..4b5d94875ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -32,11 +32,13 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; +import org.apache.hadoop.http.lib.StaticUserWebFilter; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; import org.apache.hadoop.security.Groups; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; import org.apache.hadoop.security.authorize.ProxyUsers; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.service.CompositeService; @@ -88,8 +90,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.RMAuthenticationHandler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; +import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter; +import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer; import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher; import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils; import org.apache.hadoop.yarn.server.webproxy.WebAppProxy; @@ -789,6 +794,62 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected void startWepApp() { + + // Use the customized yarn filter instead of the standard kerberos filter to + // allow users to authenticate using delegation tokens + // 3 conditions need to be satisfied - + // 1. security is enabled + // 2. http auth type is set to kerberos + // 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true + + Configuration conf = getConfig(); + boolean useYarnAuthenticationFilter = + conf.getBoolean( + YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, + YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER); + String authPrefix = "hadoop.http.authentication."; + String authTypeKey = authPrefix + "type"; + String initializers = conf.get("hadoop.http.filter.initializers"); + if (UserGroupInformation.isSecurityEnabled() + && useYarnAuthenticationFilter + && conf.get(authTypeKey, "").equalsIgnoreCase( + KerberosAuthenticationHandler.TYPE)) { + LOG.info("Using RM authentication filter(kerberos/delegation-token)" + + " for RM webapp authentication"); + RMAuthenticationHandler + .setSecretManager(getClientRMService().rmDTSecretManager); + String yarnAuthKey = + authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY; + conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName()); + + initializers = + initializers == null || initializers.isEmpty() ? "" : "," + + initializers; + if (!initializers.contains(RMAuthenticationFilterInitializer.class + .getName())) { + conf.set("hadoop.http.filter.initializers", + RMAuthenticationFilterInitializer.class.getName() + initializers); + } + } + + // if security is not enabled and the default filter initializer has been + // set, set the initializer to include the + // RMAuthenticationFilterInitializer which in turn will set up the simple + // auth filter. + + if (!UserGroupInformation.isSecurityEnabled()) { + if (initializers == null || initializers.isEmpty()) { + conf.set("hadoop.http.filter.initializers", + RMAuthenticationFilterInitializer.class.getName()); + conf.set(authTypeKey, "simple"); + } else if (initializers.equals(StaticUserWebFilter.class.getName())) { + conf.set("hadoop.http.filter.initializers", + RMAuthenticationFilterInitializer.class.getName() + "," + + initializers); + conf.set(authTypeKey, "simple"); + } + } + Builder builder = WebApps .$for("cluster", ApplicationMasterService.class, masterService, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java new file mode 100644 index 00000000000..798c479c287 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMAuthenticationHandler.java @@ -0,0 +1,157 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.security; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.client.AuthenticationException; +import org.apache.hadoop.security.authentication.server.AuthenticationToken; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; + +public class RMAuthenticationHandler extends KerberosAuthenticationHandler { + + public static final String TYPE = "kerberos-dt"; + public static final String HEADER = "Hadoop-YARN-Auth-Delegation-Token"; + + static RMDelegationTokenSecretManager secretManager; + static boolean secretManagerInitialized = false; + + public RMAuthenticationHandler() { + super(); + } + + /** + * Returns authentication type of the handler. + * + * @return kerberos-dt + */ + @Override + public String getType() { + return TYPE; + } + + @Override + public boolean managementOperation(AuthenticationToken token, + HttpServletRequest request, HttpServletResponse response) { + return true; + } + + /** + * Authenticates a request looking for the delegation header and + * verifying it is a valid token. If the header is missing, it delegates the + * authentication to the {@link KerberosAuthenticationHandler} unless it is + * disabled. + * + * @param request + * the HTTP client request. + * @param response + * the HTTP client response. + * + * @return the authentication token for the authenticated request. + * @throws IOException + * thrown if an IO error occurred. + * @throws AuthenticationException + * thrown if the authentication failed. + */ + @Override + public AuthenticationToken authenticate(HttpServletRequest request, + HttpServletResponse response) throws IOException, AuthenticationException { + + AuthenticationToken token; + String delegationParam = this.getEncodedDelegationTokenFromRequest(request); + if (delegationParam != null) { + Token dt = + new Token(); + ; + dt.decodeFromUrlString(delegationParam); + UserGroupInformation ugi = this.verifyToken(dt); + if (ugi == null) { + throw new AuthenticationException("Invalid token"); + } + final String shortName = ugi.getShortUserName(); + token = new AuthenticationToken(shortName, ugi.getUserName(), getType()); + } else { + token = super.authenticate(request, response); + if (token != null) { + // create a token with auth type set correctly + token = + new AuthenticationToken(token.getUserName(), token.getName(), + super.getType()); + } + } + return token; + } + + /** + * Verifies a delegation token. + * + * @param token + * delegation token to verify. + * @return the UGI for the token; null if the verification fails + * @throws IOException + * thrown if the token could not be verified. + */ + protected UserGroupInformation verifyToken( + Token token) throws IOException { + if (secretManagerInitialized == false) { + throw new IllegalStateException("Secret manager not initialized"); + } + ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); + DataInputStream dis = new DataInputStream(buf); + RMDelegationTokenIdentifier id = secretManager.createIdentifier(); + try { + id.readFields(dis); + secretManager.verifyToken(id, token.getPassword()); + } catch (Throwable t) { + return null; + } finally { + dis.close(); + } + return id.getUser(); + } + + /** + * Extract encoded delegation token from request + * + * @param req + * HTTPServletRequest object + * + * @return String containing the encoded token; null if encoded token not + * found + * + */ + protected String getEncodedDelegationTokenFromRequest(HttpServletRequest req) { + String header = req.getHeader(HEADER); + return header; + } + + public static void setSecretManager(RMDelegationTokenSecretManager manager) { + secretManager = manager; + secretManagerInitialized = true; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 0493efdb7d4..a8ec19260ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -55,6 +55,7 @@ import org.apache.commons.codec.binary.Base64; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; @@ -680,6 +681,11 @@ public class RMWebServices { throw new AuthorizationException(msg); } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + String userName = callerUGI.getUserName(); RMApp app = null; try { @@ -800,6 +806,13 @@ public class RMWebServices { return callerUGI; } + private boolean isStaticUser(UserGroupInformation callerUGI) { + String staticUser = + conf.get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER, + CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER); + return staticUser.equals(callerUGI.getUserName()); + } + /** * Generates a new ApplicationId which is then sent to the client * @@ -822,6 +835,10 @@ public class RMWebServices { throw new AuthorizationException("Unable to obtain user name, " + "user not authenticated"); } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } NewApplication appId = createNewApplication(); return Response.status(Status.OK).entity(appId).build(); @@ -859,6 +876,11 @@ public class RMWebServices { + "user not authenticated"); } + if (UserGroupInformation.isSecurityEnabled() && isStaticUser(callerUGI)) { + String msg = "The default static user cannot carry out this operation."; + return Response.status(Status.FORBIDDEN).entity(msg).build(); + } + ApplicationSubmissionContext appContext = createAppSubmissionContext(newApp); final SubmitApplicationRequest req = @@ -975,7 +997,7 @@ public class RMWebServices { * * @param newApp * the information provided by the user - * @return + * @return created context * @throws BadRequestException * @throws IOException */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java new file mode 100644 index 00000000000..34a914a7541 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java @@ -0,0 +1,354 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.concurrent.Callable; + +import javax.ws.rs.core.MediaType; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.Marshaller; + +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.KerberosTestUtils; +import org.apache.hadoop.security.authentication.server.AuthenticationFilter; +import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.codehaus.jettison.json.JSONObject; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.sun.jersey.api.client.ClientResponse.Status; + +public class TestRMWebServicesDelegationTokenAuthentication { + + private static final File testRootDir = new File("target", + TestRMWebServicesDelegationTokenAuthentication.class.getName() + "-root"); + private static File httpSpnegoKeytabFile = new File( + KerberosTestUtils.getKeytabFile()); + + private static String httpSpnegoPrincipal = KerberosTestUtils + .getServerPrincipal(); + + private static boolean miniKDCStarted = false; + private static MiniKdc testMiniKDC; + private static MockRM rm; + + // use published header name + final static String DelegationTokenHeader = + "Hadoop-YARN-Auth-Delegation-Token"; + + @BeforeClass + public static void setUp() { + try { + testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); + setupKDC(); + setupAndStartRM(); + } catch (Exception e) { + assertTrue("Couldn't create MiniKDC", false); + } + } + + @AfterClass + public static void tearDown() { + if (testMiniKDC != null) { + testMiniKDC.stop(); + } + if (rm != null) { + rm.stop(); + } + } + + public TestRMWebServicesDelegationTokenAuthentication() throws Exception { + super(); + } + + private static void setupAndStartRM() throws Exception { + Configuration rmconf = new Configuration(); + rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + String httpPrefix = "hadoop.http.authentication."; + rmconf.setStrings(httpPrefix + "type", "kerberos"); + rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL, + httpSpnegoPrincipal); + rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + // use any file for signature secret + rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file", + httpSpnegoKeytabFile.getAbsolutePath()); + rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, + "kerberos"); + rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, + true); + rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, + httpSpnegoPrincipal); + rmconf.set(YarnConfiguration.RM_KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, + httpSpnegoKeytabFile.getAbsolutePath()); + rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, + httpSpnegoPrincipal); + rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, + httpSpnegoKeytabFile.getAbsolutePath()); + rmconf.setBoolean("mockrm.webapp.enabled", true); + UserGroupInformation.setConfiguration(rmconf); + rm = new MockRM(rmconf); + rm.start(); + + } + + private static void setupKDC() throws Exception { + if (miniKDCStarted == false) { + testMiniKDC.start(); + getKdc().createPrincipal(httpSpnegoKeytabFile, "HTTP/localhost", + "client", UserGroupInformation.getLoginUser().getShortUserName()); + miniKDCStarted = true; + } + } + + private static MiniKdc getKdc() { + return testMiniKDC; + } + + // Test that you can authenticate with only delegation tokens + // 1. Get a delegation token using Kerberos auth(this ends up + // testing the fallback authenticator) + // 2. Submit an app without kerberos or delegation-token + // - we should get an UNAUTHORIZED response + // 3. Submit same app with delegation-token + // - we should get OK response + // - confirm owner of the app is the user whose + // delegation-token we used + + @Test + public void testDelegationTokenAuth() throws Exception { + final String token = getDelegationToken("test"); + + ApplicationSubmissionContextInfo app = + new ApplicationSubmissionContextInfo(); + String appid = "application_123_0"; + app.setApplicationId(appid); + String requestBody = getMarshalledAppInfo(app); + + URL url = new URL("http://localhost:8088/ws/v1/cluster/apps"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + setupConn(conn, "POST", "application/xml", requestBody); + + // this should fail with unauthorized because only + // auth is kerberos or delegation token + try { + conn.getInputStream(); + fail("we should not be here"); + } catch (IOException e) { + assertEquals(Status.UNAUTHORIZED.getStatusCode(), conn.getResponseCode()); + } + + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty(DelegationTokenHeader, token); + setupConn(conn, "POST", MediaType.APPLICATION_XML, requestBody); + + // this should not fail + conn.getInputStream(); + boolean appExists = + rm.getRMContext().getRMApps() + .containsKey(ConverterUtils.toApplicationId(appid)); + assertTrue(appExists); + RMApp actualApp = + rm.getRMContext().getRMApps() + .get(ConverterUtils.toApplicationId(appid)); + String owner = actualApp.getUser(); + assertEquals("client", owner); + + return; + } + + // Test to make sure that cancelled delegation tokens + // are rejected + @Test + public void testCancelledDelegationToken() throws Exception { + String token = getDelegationToken("client"); + cancelDelegationToken(token); + ApplicationSubmissionContextInfo app = + new ApplicationSubmissionContextInfo(); + String appid = "application_123_0"; + app.setApplicationId(appid); + String requestBody = getMarshalledAppInfo(app); + + URL url = new URL("http://localhost:8088/ws/v1/cluster/apps"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty(DelegationTokenHeader, token); + setupConn(conn, "POST", MediaType.APPLICATION_XML, requestBody); + + // this should fail with unauthorized because only + // auth is kerberos or delegation token + try { + conn.getInputStream(); + fail("Authentication should fail with expired delegation tokens"); + } catch (IOException e) { + assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode()); + } + return; + } + + // Test to make sure that we can't do delegation token + // functions using just delegation token auth + @Test + public void testDelegationTokenOps() throws Exception { + String token = getDelegationToken("client"); + String createRequest = "{\"renewer\":\"test\"}"; + String renewRequest = "{\"token\": \"" + token + "\"}"; + + // first test create and renew + String[] requests = { createRequest, renewRequest }; + for (String requestBody : requests) { + URL url = new URL("http://localhost:8088/ws/v1/cluster/delegation-token"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty(DelegationTokenHeader, token); + setupConn(conn, "POST", MediaType.APPLICATION_JSON, requestBody); + try { + conn.getInputStream(); + fail("Creation/Renewing delegation tokens should not be " + + "allowed with token auth"); + } catch (IOException e) { + assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode()); + } + } + + // test cancel + URL url = new URL("http://localhost:8088/ws/v1/cluster/delegation-token"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty(DelegationTokenHeader, token); + conn.setRequestProperty(RMWebServices.DELEGATION_TOKEN_HEADER, token); + setupConn(conn, "DELETE", null, null); + try { + conn.getInputStream(); + fail("Cancelling delegation tokens should not be allowed with token auth"); + } catch (IOException e) { + assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode()); + } + return; + } + + private String getDelegationToken(final String renewer) throws Exception { + String token = KerberosTestUtils.doAsClient(new Callable() { + @Override + public String call() throws Exception { + String ret = null; + String body = "{\"renewer\":\"" + renewer + "\"}"; + URL url = + new URL("http://localhost:8088/ws/v1/cluster/delegation-token"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + setupConn(conn, "POST", MediaType.APPLICATION_JSON, body); + InputStream response = conn.getInputStream(); + assertEquals(Status.OK.getStatusCode(), conn.getResponseCode()); + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(response, "UTF8")); + for (String line; (line = reader.readLine()) != null;) { + JSONObject obj = new JSONObject(line); + if (obj.has("token")) { + reader.close(); + response.close(); + ret = obj.getString("token"); + break; + } + } + } finally { + IOUtils.closeQuietly(reader); + IOUtils.closeQuietly(response); + } + return ret; + } + }); + return token; + } + + private void cancelDelegationToken(final String tokenString) throws Exception { + + KerberosTestUtils.doAsClient(new Callable() { + @Override + public Void call() throws Exception { + URL url = + new URL("http://localhost:8088/ws/v1/cluster/delegation-token"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestProperty(RMWebServices.DELEGATION_TOKEN_HEADER, + tokenString); + setupConn(conn, "DELETE", null, null); + InputStream response = conn.getInputStream(); + assertEquals(Status.OK.getStatusCode(), conn.getResponseCode()); + response.close(); + return null; + } + }); + return; + } + + static String getMarshalledAppInfo(ApplicationSubmissionContextInfo appInfo) + throws Exception { + + StringWriter writer = new StringWriter(); + JAXBContext context = + JAXBContext.newInstance(ApplicationSubmissionContextInfo.class); + Marshaller m = context.createMarshaller(); + m.marshal(appInfo, writer); + return writer.toString(); + } + + static void setupConn(HttpURLConnection conn, String method, + String contentType, String body) throws Exception { + conn.setRequestMethod(method); + conn.setDoOutput(true); + conn.setRequestProperty("Accept-Charset", "UTF8"); + if (contentType != null && !contentType.isEmpty()) { + conn.setRequestProperty("Content-Type", contentType + ";charset=UTF8"); + if (body != null && !body.isEmpty()) { + OutputStream stream = conn.getOutputStream(); + stream.write(body.getBytes("UTF8")); + stream.close(); + } + } + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java new file mode 100644 index 00000000000..2f6a02287c4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebappAuthentication.java @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Arrays; +import java.util.Collection; + +import javax.ws.rs.core.MediaType; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.KerberosTestUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +import com.sun.jersey.api.client.ClientResponse.Status; + +/* Just a simple test class to ensure that the RM handles the static web user + * correctly for secure and un-secure modes + * + */ +@RunWith(Parameterized.class) +public class TestRMWebappAuthentication { + + private static MockRM rm; + private static Configuration simpleConf; + private static Configuration kerberosConf; + + private static final File testRootDir = new File("target", + TestRMWebServicesDelegationTokenAuthentication.class.getName() + "-root"); + private static File httpSpnegoKeytabFile = new File( + KerberosTestUtils.getKeytabFile()); + + private static boolean miniKDCStarted = false; + private static MiniKdc testMiniKDC; + + static { + simpleConf = new Configuration(); + simpleConf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + simpleConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + simpleConf.setBoolean("mockrm.webapp.enabled", true); + kerberosConf = new Configuration(); + kerberosConf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + kerberosConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, + ResourceScheduler.class); + kerberosConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); + kerberosConf.set( + CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); + kerberosConf.set(YarnConfiguration.RM_KEYTAB, + httpSpnegoKeytabFile.getAbsolutePath()); + kerberosConf.setBoolean("mockrm.webapp.enabled", true); + } + + @Parameters + public static Collection params() { + return Arrays.asList(new Object[][] { { 1, simpleConf }, + { 2, kerberosConf } }); + } + + public TestRMWebappAuthentication(int run, Configuration conf) { + super(); + setupAndStartRM(conf); + } + + @BeforeClass + public static void setUp() { + try { + testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir); + setupKDC(); + } catch (Exception e) { + assertTrue("Couldn't create MiniKDC", false); + } + } + + @AfterClass + public static void tearDown() { + if (testMiniKDC != null) { + testMiniKDC.stop(); + } + } + + private static void setupKDC() throws Exception { + if (!miniKDCStarted) { + testMiniKDC.start(); + getKdc().createPrincipal(httpSpnegoKeytabFile, "HTTP/localhost", + "client", UserGroupInformation.getLoginUser().getShortUserName()); + miniKDCStarted = true; + } + } + + private static MiniKdc getKdc() { + return testMiniKDC; + } + + private static void setupAndStartRM(Configuration conf) { + UserGroupInformation.setConfiguration(conf); + rm = new MockRM(conf); + } + + // ensure that in a non-secure cluster users can access + // the web pages as earlier and submit apps as anonymous + // user or by identifying themselves + @Test + public void testSimpleAuth() throws Exception { + + rm.start(); + + // ensure users can access web pages + // this should work for secure and non-secure clusters + URL url = new URL("http://localhost:8088/cluster"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + try { + conn.getInputStream(); + assertEquals(Status.OK.getStatusCode(), conn.getResponseCode()); + } catch (Exception e) { + fail("Fetching url failed"); + } + + if (UserGroupInformation.isSecurityEnabled()) { + testAnonymousKerberosUser(); + } else { + testAnonymousSimpleUser(); + } + + rm.stop(); + } + + private void testAnonymousKerberosUser() throws Exception { + + ApplicationSubmissionContextInfo app = + new ApplicationSubmissionContextInfo(); + String appid = "application_123_0"; + app.setApplicationId(appid); + String requestBody = + TestRMWebServicesDelegationTokenAuthentication + .getMarshalledAppInfo(app); + + URL url = + new URL("http://localhost:8088/ws/v1/cluster/apps/new-application"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, "POST", + "application/xml", requestBody); + + try { + conn.getInputStream(); + fail("Anonymous users should not be allowed to get new application ids in secure mode."); + } catch (IOException ie) { + assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode()); + } + + url = new URL("http://localhost:8088/ws/v1/cluster/apps"); + conn = (HttpURLConnection) url.openConnection(); + TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, "POST", + "application/xml", requestBody); + + try { + conn.getInputStream(); + fail("Anonymous users should not be allowed to submit apps in secure mode."); + } catch (IOException ie) { + assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode()); + } + + requestBody = "{ \"state\": \"KILLED\"}"; + url = + new URL( + "http://localhost:8088/ws/v1/cluster/apps/application_123_0/state"); + conn = (HttpURLConnection) url.openConnection(); + TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, "PUT", + "application/json", requestBody); + + try { + conn.getInputStream(); + fail("Anonymous users should not be allowed to kill apps in secure mode."); + } catch (IOException ie) { + assertEquals(Status.FORBIDDEN.getStatusCode(), conn.getResponseCode()); + } + } + + private void testAnonymousSimpleUser() throws Exception { + + ApplicationSubmissionContextInfo app = + new ApplicationSubmissionContextInfo(); + String appid = "application_123_0"; + app.setApplicationId(appid); + String requestBody = + TestRMWebServicesDelegationTokenAuthentication + .getMarshalledAppInfo(app); + + URL url = new URL("http://localhost:8088/ws/v1/cluster/apps"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, "POST", + "application/xml", requestBody); + + conn.getInputStream(); + assertEquals(Status.ACCEPTED.getStatusCode(), conn.getResponseCode()); + boolean appExists = + rm.getRMContext().getRMApps() + .containsKey(ConverterUtils.toApplicationId(appid)); + assertTrue(appExists); + RMApp actualApp = + rm.getRMContext().getRMApps() + .get(ConverterUtils.toApplicationId(appid)); + String owner = actualApp.getUser(); + assertEquals( + rm.getConfig().get(CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER, + CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER), owner); + + appid = "application_123_1"; + app.setApplicationId(appid); + requestBody = + TestRMWebServicesDelegationTokenAuthentication + .getMarshalledAppInfo(app); + url = new URL("http://localhost:8088/ws/v1/cluster/apps?user.name=client"); + conn = (HttpURLConnection) url.openConnection(); + TestRMWebServicesDelegationTokenAuthentication.setupConn(conn, "POST", + MediaType.APPLICATION_XML, requestBody); + + conn.getInputStream(); + appExists = + rm.getRMContext().getRMApps() + .containsKey(ConverterUtils.toApplicationId(appid)); + assertTrue(appExists); + actualApp = + rm.getRMContext().getRMApps() + .get(ConverterUtils.toApplicationId(appid)); + owner = actualApp.getUser(); + assertEquals("client", owner); + + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm index 1952e11176d..9609ba39de9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm @@ -2912,3 +2912,24 @@ Accept: application/xml +---+ No response body. + +** Authentication using delegation tokens + + This feature is in the alpha mode and may change in the future. + + You can use delegation tokens to authenticate yourself when using YARN RM webservices. However, this requires setting the right configurations. The conditions for this are: + + * Hadoop is setup in secure mode with the authentication type set to kerberos. + + * Hadoop HTTP authentication is setup with the authentication type set to kerberos + + Once setup, delegation tokens can be fetched using the web services listed above and used as shown in an example below: + ++---+ + PUT http:///ws/v1/cluster/apps/application_1399397633663_0003/state + Hadoop-YARN-Auth-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A + Content-Type: application/json; charset=UTF8 + { + "state":"KILLED" + } ++---+ From f49c2a1b625948119afd5a867db2101c4b2cfb0e Mon Sep 17 00:00:00 2001 From: Brandon Li Date: Mon, 28 Jul 2014 18:53:47 +0000 Subject: [PATCH 35/37] HDFS-6717. JIRA HDFS-5804 breaks default nfs-gateway behavior for unsecured config. Contributed by Brandon Li git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1614125 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../src/site/apt/HdfsNfsGateway.apt.vm | 60 +++++++------------ 2 files changed, 24 insertions(+), 39 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index eb524b209df..c4930e62217 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -938,6 +938,9 @@ Release 2.5.0 - UNRELEASED HDFS-6723. New NN webUI no longer displays decommissioned state for dead node. (Ming Ma via wheat9) + HDFS-6717. JIRA HDFS-5804 breaks default nfs-gateway behavior for unsecured config + (brandonli) + BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm index 54544cff46f..863ba39a739 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm @@ -44,10 +44,13 @@ HDFS NFS Gateway * {Configuration} - The user running the NFS-gateway must be able to proxy all the users using the NFS mounts. - For instance, if user 'nfsserver' is running the gateway, and users belonging to the groups 'nfs-users1' - and 'nfs-users2' use the NFS mounts, then in core-site.xml of the namenode, the following must be set - (NOTE: replace 'nfsserver' with the user name starting the gateway in your cluster): + The NFS-gateway uses proxy user to proxy all the users accessing the NFS mounts. + In non-secure mode, the user running the gateway is the proxy user, while in secure mode the + user in Kerberos keytab is the proxy user. Suppose the proxy user is 'nfsserver' + and users belonging to the groups 'nfs-users1' + and 'nfs-users2' use the NFS mounts, then in core-site.xml of the NameNode, the following + two properities must be set and only NameNode needs restart after the configuration change + (NOTE: replace the string 'nfsserver' with the proxy user name in your cluster): ---- @@ -72,7 +75,9 @@ HDFS NFS Gateway ---- The above are the only required configuration for the NFS gateway in non-secure mode. For Kerberized - hadoop clusters, the following configurations need to be added to hdfs-site.xml: + hadoop clusters, the following configurations need to be added to hdfs-site.xml for the gateway (NOTE: replace + string "nfsserver" with the proxy user name and ensure the user contained in the keytab is + also the same proxy user): ---- @@ -87,6 +92,8 @@ HDFS NFS Gateway nfsserver/_HOST@YOUR-REALM.COM ---- + + The rest of the NFS gateway configurations are optional for both secure and non-secure mode. The AIX NFS client has a {{{https://issues.apache.org/jira/browse/HDFS-6549}few known issues}} that prevent it from working correctly by default with the HDFS NFS @@ -108,7 +115,7 @@ HDFS NFS Gateway have been committed. It's strongly recommended for the users to update a few configuration properties based on their use - cases. All the related configuration properties can be added or updated in hdfs-site.xml. + cases. All the following configuration properties can be added or updated in hdfs-site.xml. * If the client mounts the export with access time update allowed, make sure the following property is not disabled in the configuration file. Only NameNode needs to restart after @@ -145,36 +152,6 @@ HDFS NFS Gateway ---- - * For optimal performance, it is recommended that rtmax be updated to - 1MB. However, note that this 1MB is a per client allocation, and not - from a shared memory pool, and therefore a larger value may adversely - affect small reads, consuming a lot of memory. The maximum value of - this property is 1MB. - ----- - - nfs.rtmax - 1048576 - This is the maximum size in bytes of a READ request - supported by the NFS gateway. If you change this, make sure you - also update the nfs mount's rsize(add rsize= # of bytes to the - mount directive). - - ----- - ----- - - nfs.wtmax - 65536 - This is the maximum size in bytes of a WRITE request - supported by the NFS gateway. If you change this, make sure you - also update the nfs mount's wsize(add wsize= # of bytes to the - mount directive). - - ----- - * By default, the export can be mounted by any client. To better control the access, users can update the following property. The value string contains machine name and access privilege, separated by whitespace @@ -238,8 +215,10 @@ HDFS NFS Gateway [[3]] Start mountd and nfsd. - No root privileges are required for this command. However, ensure that the user starting - the Hadoop cluster and the user starting the NFS gateway are same. + No root privileges are required for this command. In non-secure mode, the NFS gateway + should be started by the proxy user mentioned at the beginning of this user guide. + While in secure mode, any user can start NFS gateway + as long as the user has read access to the Kerberos keytab defined in "nfs.keytab.file". ------------------------- hadoop nfs3 @@ -339,7 +318,10 @@ HDFS NFS Gateway ------------------------------------------------------------------- Then the users can access HDFS as part of the local file system except that, - hard link and random write are not supported yet. + hard link and random write are not supported yet. To optimize the performance + of large file I/O, one can increase the NFS transfer size(rsize and wsize) during mount. + By default, NFS gateway supports 1MB as the maximum transfer size. For larger data + transfer size, one needs to update "nfs.rtmax" and "nfs.rtmax" in hdfs-site.xml. * {Allow mounts from unprivileged clients} From 4f9ffc7455ae2182df1da1b7e3f5a55e645fc8a4 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 28 Jul 2014 23:43:39 +0000 Subject: [PATCH 36/37] HDFS-6739. Add getDatanodeStorageReport to ClientProtocol. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1614215 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/DFSClient.java | 26 +++--- .../hadoop/hdfs/protocol/ClientProtocol.java | 12 ++- ...amenodeProtocolServerSideTranslatorPB.java | 19 +++- .../ClientNamenodeProtocolTranslatorPB.java | 16 ++++ ...atanodeProtocolClientSideTranslatorPB.java | 14 +-- .../hadoop/hdfs/protocolPB/PBHelper.java | 52 ++++++++++- .../blockmanagement/DatanodeDescriptor.java | 9 ++ .../blockmanagement/DatanodeStorageInfo.java | 6 ++ .../hdfs/server/namenode/FSNamesystem.java | 28 +++++- .../server/namenode/NameNodeRpcServer.java | 15 +++- .../protocol/DatanodeStorageReport.java | 42 +++++++++ .../main/proto/ClientNamenodeProtocol.proto | 15 ++++ .../src/main/proto/DatanodeProtocol.proto | 24 ----- .../hadoop-hdfs/src/main/proto/hdfs.proto | 24 +++++ .../hadoop/hdfs/TestDatanodeReport.java | 90 ++++++++++++++----- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 17 +++- 17 files changed, 326 insertions(+), 85 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorageReport.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c4930e62217..fca7e231045 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -324,6 +324,8 @@ Release 2.6.0 - UNRELEASED HDFS-6750. The DataNode should use its shared memory segment to mark short-circuit replicas that have been unlinked as stale (cmccabe) + HDFS-6739. Add getDatanodeStorageReport to ClientProtocol. (szetszwo) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 9edb3db6585..45a9011a568 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -32,19 +32,21 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHED_CONN_RETRY_ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_READS; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_DROP_BEHIND_WRITES; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CACHE_READAHEAD; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_BASE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_FAILOVER_SLEEPTIME_MAX_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_RETRY_WINDOW_BASE; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_SOCKET_CACHE_CAPACITY_KEY; @@ -60,8 +62,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT; import java.io.BufferedOutputStream; import java.io.DataInputStream; @@ -91,7 +91,6 @@ import java.util.concurrent.atomic.AtomicInteger; import javax.net.SocketFactory; -import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -112,22 +111,22 @@ import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; import org.apache.hadoop.fs.Options; -import org.apache.hadoop.fs.XAttr; -import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.VolumeId; +import org.apache.hadoop.fs.XAttr; +import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; -import org.apache.hadoop.hdfs.protocol.AclException; import org.apache.hadoop.hdfs.net.Peer; import org.apache.hadoop.hdfs.net.TcpPeerServer; +import org.apache.hadoop.hdfs.protocol.AclException; import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry; import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo; import org.apache.hadoop.hdfs.protocol.CacheDirectiveIterator; @@ -158,8 +157,8 @@ import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair; import org.apache.hadoop.hdfs.protocol.datatransfer.Op; import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure; -import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver; import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; +import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil; import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferClient; @@ -175,6 +174,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.datanode.CachingStrategy; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; @@ -200,6 +200,7 @@ import org.apache.hadoop.util.Time; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.net.InetAddresses; /******************************************************** @@ -2192,6 +2193,11 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, return namenode.getDatanodeReport(type); } + public DatanodeStorageReport[] getDatanodeStorageReport( + DatanodeReportType type) throws IOException { + return namenode.getDatanodeStorageReport(type); + } + /** * Enter, leave or get safe mode. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 9398c721a68..a2a52fef389 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.CacheFlag; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; @@ -31,11 +32,10 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.Options; -import org.apache.hadoop.fs.XAttr; -import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclStatus; @@ -47,6 +47,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.retry.AtMostOnce; @@ -654,6 +655,13 @@ public interface ClientProtocol { public DatanodeInfo[] getDatanodeReport(HdfsConstants.DatanodeReportType type) throws IOException; + /** + * Get a report on the current datanode storages. + */ + @Idempotent + public DatanodeStorageReport[] getDatanodeStorageReport( + HdfsConstants.DatanodeReportType type) throws IOException; + /** * Get the block size for the given file. * @param filename The name of the file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 3a312b0d418..df0d1b0006c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -72,6 +72,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Create import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSnapshotResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeStorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteSnapshotRequestProto; @@ -93,6 +94,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDat import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto; @@ -174,7 +177,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; -import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto; @@ -655,6 +657,21 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements } } + @Override + public GetDatanodeStorageReportResponseProto getDatanodeStorageReport( + RpcController controller, GetDatanodeStorageReportRequestProto req) + throws ServiceException { + try { + List reports = PBHelper.convertDatanodeStorageReports( + server.getDatanodeStorageReport(PBHelper.convert(req.getType()))); + return GetDatanodeStorageReportResponseProto.newBuilder() + .addAllDatanodeStorageReports(reports) + .build(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + @Override public GetPreferredBlockSizeResponseProto getPreferredBlockSize( RpcController controller, GetPreferredBlockSizeRequestProto req) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index d20ae1d14ea..0f8eba970ca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -94,6 +94,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetCon import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto; @@ -151,6 +152,7 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ProtobufHelper; @@ -580,6 +582,20 @@ public class ClientNamenodeProtocolTranslatorPB implements } } + @Override + public DatanodeStorageReport[] getDatanodeStorageReport(DatanodeReportType type) + throws IOException { + final GetDatanodeStorageReportRequestProto req + = GetDatanodeStorageReportRequestProto.newBuilder() + .setType(PBHelper.convert(type)).build(); + try { + return PBHelper.convertDatanodeStorageReports( + rpcProxy.getDatanodeStorageReport(null, req).getDatanodeStorageReportsList()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + } + @Override public long getPreferredBlockSize(String filename) throws IOException, UnresolvedLinkException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index 5775d6e2634..46023ecaa34 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -21,18 +21,13 @@ package org.apache.hadoop.hdfs.protocolPB; import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto; @@ -51,7 +46,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReportBadBlo import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageBlockReportProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto; -import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; @@ -61,14 +55,10 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; -import org.apache.hadoop.io.retry.RetryPolicies; -import org.apache.hadoop.io.retry.RetryPolicy; -import org.apache.hadoop.io.retry.RetryProxy; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.ProtocolMetaInterface; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.RpcClientUtil; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -137,9 +127,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements .setRegistration(PBHelper.convert(registration)) .setXmitsInProgress(xmitsInProgress).setXceiverCount(xceiverCount) .setFailedVolumes(failedVolumes); - for (StorageReport r : reports) { - builder.addReports(PBHelper.convert(r)); - } + builder.addAllReports(PBHelper.convertStorageReports(reports)); if (cacheCapacity != 0) { builder.setCacheCapacity(cacheCapacity); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index bd7e5f13918..5706aab062f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -90,6 +90,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheP import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolStatsProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateFlagProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeStorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeActionProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RollingUpgradeInfoProto; @@ -102,14 +103,11 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockIdComma import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto.StorageState; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.FinalizeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.KeyUpdateCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHeartbeatProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; @@ -125,6 +123,8 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto.AdminState; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfosProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeLocalInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto.StorageState; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DirectoryListingProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; @@ -149,6 +149,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryListingProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectoryStatusProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto; @@ -182,6 +183,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand; import org.apache.hadoop.hdfs.server.protocol.JournalInfo; import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand; @@ -620,6 +622,41 @@ public class PBHelper { return builder.build(); } + public static DatanodeStorageReportProto convertDatanodeStorageReport( + DatanodeStorageReport report) { + return DatanodeStorageReportProto.newBuilder() + .setDatanodeInfo(convert(report.getDatanodeInfo())) + .addAllStorageReports(convertStorageReports(report.getStorageReports())) + .build(); + } + + public static List convertDatanodeStorageReports( + DatanodeStorageReport[] reports) { + final List protos + = new ArrayList(reports.length); + for(int i = 0; i < reports.length; i++) { + protos.add(convertDatanodeStorageReport(reports[i])); + } + return protos; + } + + public static DatanodeStorageReport convertDatanodeStorageReport( + DatanodeStorageReportProto proto) { + return new DatanodeStorageReport( + convert(proto.getDatanodeInfo()), + convertStorageReports(proto.getStorageReportsList())); + } + + public static DatanodeStorageReport[] convertDatanodeStorageReports( + List protos) { + final DatanodeStorageReport[] reports + = new DatanodeStorageReport[protos.size()]; + for(int i = 0; i < reports.length; i++) { + reports[i] = convertDatanodeStorageReport(protos.get(i)); + } + return reports; + } + public static AdminStates convert(AdminState adminState) { switch(adminState) { case DECOMMISSION_INPROGRESS: @@ -1713,6 +1750,15 @@ public class PBHelper { return report; } + public static List convertStorageReports(StorageReport[] storages) { + final List protos = new ArrayList( + storages.length); + for(int i = 0; i < storages.length; i++) { + protos.add(convert(storages[i])); + } + return protos; + } + public static JournalInfo convert(JournalInfoProto info) { int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0; int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index a645d434985..fcc189d9f9b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -259,6 +259,15 @@ public class DatanodeDescriptor extends DatanodeInfo { } } + public StorageReport[] getStorageReports() { + final StorageReport[] reports = new StorageReport[storageMap.size()]; + final DatanodeStorageInfo[] infos = getStorageInfos(); + for(int i = 0; i < infos.length; i++) { + reports[i] = infos[i].toStorageReport(); + } + return reports; + } + boolean hasStaleStorages() { synchronized (storageMap) { for (DatanodeStorageInfo storage : storageMap.values()) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index f7bab3ced6a..fa4b0e533bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@ -291,6 +291,12 @@ public class DatanodeStorageInfo { public String toString() { return "[" + storageType + "]" + storageID + ":" + state; } + + StorageReport toStorageReport() { + return new StorageReport( + new DatanodeStorage(storageID, state, storageType), + false, capacity, dfsUsed, remaining, blockPoolUsed); + } /** @return the first {@link DatanodeStorageInfo} corresponding to * the given datanode diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index a705f17dd27..f12c4e2cfe1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -62,6 +62,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CAC import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPL_QUEUE_THRESHOLD_PCT_KEY; @@ -83,9 +85,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROU import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RANDOMIZE_BLOCK_LOCATIONS_PER_BLOCK_DEFAULT; - import static org.apache.hadoop.util.Time.now; import java.io.BufferedWriter; @@ -231,6 +230,7 @@ import org.apache.hadoop.hdfs.server.namenode.startupprogress.StepType; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat; import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; @@ -4916,6 +4916,28 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } + DatanodeStorageReport[] getDatanodeStorageReport(final DatanodeReportType type + ) throws AccessControlException, StandbyException { + checkSuperuserPrivilege(); + checkOperation(OperationCategory.UNCHECKED); + readLock(); + try { + checkOperation(OperationCategory.UNCHECKED); + final DatanodeManager dm = getBlockManager().getDatanodeManager(); + final List datanodes = dm.getDatanodeListForReport(type); + + DatanodeStorageReport[] reports = new DatanodeStorageReport[datanodes.size()]; + for (int i = 0; i < reports.length; i++) { + final DatanodeDescriptor d = datanodes.get(i); + reports[i] = new DatanodeStorageReport(new DatanodeInfo(d), + d.getStorageReports()); + } + return reports; + } finally { + readUnlock(); + } + } + /** * Save namespace image. * This will save current namespace into fsimage file and empty edits file. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index f1f67247c26..2c2cd4f2272 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -115,6 +115,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand; import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; import org.apache.hadoop.hdfs.server.protocol.FinalizeCommand; import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse; import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; @@ -830,11 +831,23 @@ class NameNodeRpcServer implements NamenodeProtocols { throws IOException { DatanodeInfo results[] = namesystem.datanodeReport(type); if (results == null ) { - throw new IOException("Cannot find datanode report"); + throw new IOException("Failed to get datanode report for " + type + + " datanodes."); } return results; } + @Override // ClientProtocol + public DatanodeStorageReport[] getDatanodeStorageReport( + DatanodeReportType type) throws IOException { + final DatanodeStorageReport[] reports = namesystem.getDatanodeStorageReport(type); + if (reports == null ) { + throw new IOException("Failed to get datanode storage report for " + type + + " datanodes."); + } + return reports; + } + @Override // ClientProtocol public boolean setSafeMode(SafeModeAction action, boolean isChecked) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorageReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorageReport.java new file mode 100644 index 00000000000..6a956a0fac1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeStorageReport.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.protocol; + +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; + +/** + * Class captures information of a datanode and its storages. + */ +public class DatanodeStorageReport { + final DatanodeInfo datanodeInfo; + final StorageReport[] storageReports; + + public DatanodeStorageReport(DatanodeInfo datanodeInfo, + StorageReport[] storageReports) { + this.datanodeInfo = datanodeInfo; + this.storageReports = storageReports; + } + + public DatanodeInfo getDatanodeInfo() { + return datanodeInfo; + } + + public StorageReport[] getStorageReports() { + return storageReports; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 5a75c41fb54..d2f92d64d0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -281,6 +281,19 @@ message GetDatanodeReportResponseProto { repeated DatanodeInfoProto di = 1; } +message GetDatanodeStorageReportRequestProto { + required DatanodeReportTypeProto type = 1; +} + +message DatanodeStorageReportProto { + required DatanodeInfoProto datanodeInfo = 1; + repeated StorageReportProto storageReports = 2; +} + +message GetDatanodeStorageReportResponseProto { + repeated DatanodeStorageReportProto datanodeStorageReports = 1; +} + message GetPreferredBlockSizeRequestProto { required string filename = 1; } @@ -672,6 +685,8 @@ service ClientNamenodeProtocol { rpc getFsStats(GetFsStatusRequestProto) returns(GetFsStatsResponseProto); rpc getDatanodeReport(GetDatanodeReportRequestProto) returns(GetDatanodeReportResponseProto); + rpc getDatanodeStorageReport(GetDatanodeStorageReportRequestProto) + returns(GetDatanodeStorageReportResponseProto); rpc getPreferredBlockSize(GetPreferredBlockSizeRequestProto) returns(GetPreferredBlockSizeResponseProto); rpc setSafeMode(SetSafeModeRequestProto) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index 2afcf057f70..187761a4502 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -44,20 +44,6 @@ message DatanodeRegistrationProto { required string softwareVersion = 4; // Software version of the DN, e.g. "2.0.0" } -/** - * Represents a storage available on the datanode - */ -message DatanodeStorageProto { - enum StorageState { - NORMAL = 0; - READ_ONLY_SHARED = 1; - } - - required string storageUuid = 1; - optional StorageState state = 2 [default = NORMAL]; - optional StorageTypeProto storageType = 3 [default = DISK]; -} - /** * Commands sent from namenode to the datanodes */ @@ -196,16 +182,6 @@ message HeartbeatRequestProto { optional uint64 cacheUsed = 7 [default = 0 ]; } -message StorageReportProto { - required string storageUuid = 1 [ deprecated = true ]; - optional bool failed = 2 [ default = false ]; - optional uint64 capacity = 3 [ default = 0 ]; - optional uint64 dfsUsed = 4 [ default = 0 ]; - optional uint64 remaining = 5 [ default = 0 ]; - optional uint64 blockPoolUsed = 6 [ default = 0 ]; - optional DatanodeStorageProto storage = 7; // supersedes StorageUuid -} - /** * state - State the NN is in when returning response to the DN * txid - Highest transaction ID this NN has seen diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 8f15be6e92e..04fcc500e84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -99,6 +99,30 @@ message DatanodeInfoProto { optional uint64 cacheUsed = 12 [default = 0]; } +/** + * Represents a storage available on the datanode + */ +message DatanodeStorageProto { + enum StorageState { + NORMAL = 0; + READ_ONLY_SHARED = 1; + } + + required string storageUuid = 1; + optional StorageState state = 2 [default = NORMAL]; + optional StorageTypeProto storageType = 3 [default = DISK]; +} + +message StorageReportProto { + required string storageUuid = 1 [ deprecated = true ]; + optional bool failed = 2 [ default = false ]; + optional uint64 capacity = 3 [ default = 0 ]; + optional uint64 dfsUsed = 4 [ default = 0 ]; + optional uint64 remaining = 5 [ default = 0 ]; + optional uint64 blockPoolUsed = 6 [ default = 0 ]; + optional DatanodeStorageProto storage = 7; // supersedes StorageUuid +} + /** * Summary of a file or directory */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java index d5802b0b54c..1e6db21d8af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDatanodeReport.java @@ -21,19 +21,26 @@ import static org.apache.hadoop.test.MetricsAsserts.assertGauge; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.junit.Assert.assertEquals; -import java.net.InetSocketAddress; -import java.util.ArrayList; +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport; +import org.apache.hadoop.hdfs.server.protocol.StorageReport; import org.junit.Test; /** * This test ensures the all types of data node report work correctly. */ public class TestDatanodeReport { + static final Log LOG = LogFactory.getLog(TestDatanodeReport.class); final static private Configuration conf = new HdfsConfiguration(); final static private int NUM_OF_DATANODES = 4; @@ -50,20 +57,18 @@ public class TestDatanodeReport { try { //wait until the cluster is up cluster.waitActive(); + final String bpid = cluster.getNamesystem().getBlockPoolId(); + final List datanodes = cluster.getDataNodes(); + final DFSClient client = cluster.getFileSystem().dfs; - InetSocketAddress addr = new InetSocketAddress("localhost", - cluster.getNameNodePort()); - DFSClient client = new DFSClient(addr, conf); - - assertEquals(client.datanodeReport(DatanodeReportType.ALL).length, - NUM_OF_DATANODES); - assertEquals(client.datanodeReport(DatanodeReportType.LIVE).length, - NUM_OF_DATANODES); - assertEquals(client.datanodeReport(DatanodeReportType.DEAD).length, 0); + assertReports(NUM_OF_DATANODES, DatanodeReportType.ALL, client, datanodes, bpid); + assertReports(NUM_OF_DATANODES, DatanodeReportType.LIVE, client, datanodes, bpid); + assertReports(0, DatanodeReportType.DEAD, client, datanodes, bpid); // bring down one datanode - ArrayList datanodes = cluster.getDataNodes(); - datanodes.remove(datanodes.size()-1).shutdown(); + final DataNode last = datanodes.get(datanodes.size() - 1); + LOG.info("XXX shutdown datanode " + last.getDatanodeUuid()); + last.shutdown(); DatanodeInfo[] nodeInfo = client.datanodeReport(DatanodeReportType.DEAD); while (nodeInfo.length != 1) { @@ -74,22 +79,59 @@ public class TestDatanodeReport { nodeInfo = client.datanodeReport(DatanodeReportType.DEAD); } - assertEquals(client.datanodeReport(DatanodeReportType.LIVE).length, - NUM_OF_DATANODES-1); - assertEquals(client.datanodeReport(DatanodeReportType.ALL).length, - NUM_OF_DATANODES); + assertReports(NUM_OF_DATANODES, DatanodeReportType.ALL, client, datanodes, null); + assertReports(NUM_OF_DATANODES - 1, DatanodeReportType.LIVE, client, datanodes, null); + assertReports(1, DatanodeReportType.DEAD, client, datanodes, null); Thread.sleep(5000); assertGauge("ExpiredHeartbeats", 1, getMetrics("FSNamesystem")); - }finally { + } finally { cluster.shutdown(); } } - - public static void main(String[] args) throws Exception { - new TestDatanodeReport().testDatanodeReport(); + + final static Comparator CMP = new Comparator() { + @Override + public int compare(StorageReport left, StorageReport right) { + return left.getStorage().getStorageID().compareTo( + right.getStorage().getStorageID()); + } + }; + + static void assertReports(int numDatanodes, DatanodeReportType type, + DFSClient client, List datanodes, String bpid) throws IOException { + final DatanodeInfo[] infos = client.datanodeReport(type); + assertEquals(numDatanodes, infos.length); + final DatanodeStorageReport[] reports = client.getDatanodeStorageReport(type); + assertEquals(numDatanodes, reports.length); + + for(int i = 0; i < infos.length; i++) { + assertEquals(infos[i], reports[i].getDatanodeInfo()); + + final DataNode d = findDatanode(infos[i].getDatanodeUuid(), datanodes); + if (bpid != null) { + //check storage + final StorageReport[] computed = reports[i].getStorageReports(); + Arrays.sort(computed, CMP); + final StorageReport[] expected = d.getFSDataset().getStorageReports(bpid); + Arrays.sort(expected, CMP); + + assertEquals(expected.length, computed.length); + for(int j = 0; j < expected.length; j++) { + assertEquals(expected[j].getStorage().getStorageID(), + computed[j].getStorage().getStorageID()); + } + } + } } -} - - + static DataNode findDatanode(String id, List datanodes) { + for(DataNode d : datanodes) { + if (d.getDatanodeUuid().equals(id)) { + return d; + } + } + throw new IllegalStateException("Datnode " + id + " not in datanode list: " + + datanodes); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index 6c8547ebf8d..440b4f3a6cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -31,25 +31,25 @@ import org.apache.hadoop.fs.permission.AclEntryScope; import org.apache.hadoop.fs.permission.AclEntryType; import org.apache.hadoop.fs.permission.AclStatus; import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlock; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeStorageProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeStorageProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExportedBlockKeysProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; @@ -67,9 +67,18 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; -import org.apache.hadoop.hdfs.server.protocol.*; +import org.apache.hadoop.hdfs.server.protocol.BlockCommand; +import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; +import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations; import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations; +import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; +import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.proto.SecurityProtos.TokenProto; import org.apache.hadoop.security.token.Token; From c6cddce752bf537ea6567101000d93cbbc18aa16 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 29 Jul 2014 00:44:12 +0000 Subject: [PATCH 37/37] HADOOP-10876. The constructor of Path should not take an empty URL as a parameter. Contributed by Zhihai Xu. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1614230 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/fs/Path.java | 16 ++++++++++++- .../java/org/apache/hadoop/fs/TestPath.java | 24 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 34bd95103cd..b13598f9aa8 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -491,6 +491,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10830. Missing lock in JavaKeyStoreProvider.createCredentialEntry. (Benoy Antony via umamahesh) + HADOOP-10876. The constructor of Path should not take an empty URL as a + parameter. (Zhihai Xu via wang) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index 54ddedaff1d..0e8db1df11e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -128,7 +128,20 @@ public class Path implements Comparable { "Can not create a Path from an empty string"); } } - + + /** check URI parameter of Path constructor. */ + private void checkPathArg(URI aUri) throws IllegalArgumentException { + // disallow construction of a Path from an empty URI + if (aUri == null) { + throw new IllegalArgumentException( + "Can not create a Path from a null URI"); + } + if (aUri.toString().isEmpty()) { + throw new IllegalArgumentException( + "Can not create a Path from an empty URI"); + } + } + /** Construct a path from a String. Path strings are URIs, but with * unescaped elements and some additional normalization. */ public Path(String pathString) throws IllegalArgumentException { @@ -176,6 +189,7 @@ public class Path implements Comparable { * Construct a path from a URI */ public Path(URI aUri) { + checkPathArg(aUri); uri = aUri.normalize(); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java index 94908da7a38..54d25c995bd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java @@ -26,11 +26,13 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.AvroTestUtil; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import com.google.common.base.Joiner; import junit.framework.TestCase; +import static org.junit.Assert.fail; public class TestPath extends TestCase { /** @@ -305,6 +307,28 @@ public class TestPath extends TestCase { // if the child uri is absolute path assertEquals("foo://bar/fud#boo", new Path(new Path(new URI( "foo://bar/baz#bud")), new Path(new URI("/fud#boo"))).toString()); + + // empty URI + URI uri3 = new URI(""); + assertEquals("", uri3.toString()); + try { + path = new Path(uri3); + fail("Expected exception for empty URI"); + } catch (IllegalArgumentException e) { + // expect to receive an IllegalArgumentException + GenericTestUtils.assertExceptionContains("Can not create a Path" + + " from an empty URI", e); + } + // null URI + uri3 = null; + try { + path = new Path(uri3); + fail("Expected exception for null URI"); + } catch (IllegalArgumentException e) { + // expect to receive an IllegalArgumentException + GenericTestUtils.assertExceptionContains("Can not create a Path" + + " from a null URI", e); + } } /** Test URIs created from Path objects */