From d35798314c2eebdb0cdc6bde5a6195a4a8f7dc27 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Mon, 25 Mar 2013 04:34:29 +0000 Subject: [PATCH 01/37] svn merge -c -1448745 and -1448285 for reverting HADOOP-9112: test-patch should -1 for @Tests without a timeout git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460520 13f79535-47bb-0310-9956-ffa450edef68 --- dev-support/test-patch.sh | 22 ++----------------- .../hadoop-common/CHANGES.txt | 3 --- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index c3d89aae01c..c4b2b8411b2 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -323,7 +323,7 @@ checkAuthor () { } ############################################################################### -### Check for tests and their timeout in the patch +### Check for tests in the patch checkTests () { echo "" echo "" @@ -357,25 +357,7 @@ checkTests () { JIRA_COMMENT="$JIRA_COMMENT {color:green}+1 tests included{color}. The patch appears to include $testReferences new or modified test files." - echo "" - echo "======================================================================" - echo "======================================================================" - echo " Checking if the tests have timeout assigned in this patch." - echo "======================================================================" - echo "======================================================================" - - nontimeoutTests=`cat $PATCH_DIR/patch | $AWK '{ printf "%s ", $0 }' | $GREP --extended-regex --count '[ ]*\+[ ]*((@Test[\+ ]*[A-Za-z]+)|([\+ ]*@Test[ \+]*\([ \+]*\)[\ ]*\+?[ ]*[A-Za-z]+)|([\+ ]*@Test[\+ ]*\(exception[ \+]*=[ \+]*[A-Z.a-z0-9A-Z ]*\)))'` - - if [[ $nontimeoutTests == 0 ]] ; then - JIRA_COMMENT="$JIRA_COMMENT - - {color:green}+1 tests included appear to have a timeout.{color}" - return 0 - fi - JIRA_COMMENT="$JIRA_COMMENT - - {color:red}-1 one of tests included doesn't have a timeout.{color}" - return 1 + return 0 } cleanUpXml () { diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index db4796fa763..e8ce4fa6a8b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -162,9 +162,6 @@ Trunk (Unreleased) HADOOP-9218 Document the Rpc-wrappers used internally (sanjay Radia) - HADOOP-9112. test-patch should -1 for @Tests without a timeout - (Surenkumar Nihalani via bobby) - BUG FIXES From 90b063c343bd03981a08f082bfcfc23bbf89edea Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 25 Mar 2013 12:23:48 +0000 Subject: [PATCH 02/37] HADOOP-9350 -add entry to BUILDING.TXT to say 'no java7 on OS/X git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460634 13f79535-47bb-0310-9956-ffa450edef68 --- BUILDING.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/BUILDING.txt b/BUILDING.txt index 40ad2564cdb..7e3d450953f 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -139,6 +139,15 @@ Create a local staging version of the website (in /tmp/hadoop-site) ---------------------------------------------------------------------------------- +Building on OS/X + +---------------------------------------------------------------------------------- + +Hadoop does not build on OS/X with Java 7. +see: https://issues.apache.org/jira/browse/HADOOP-9350 + +---------------------------------------------------------------------------------- + Building on Windows ---------------------------------------------------------------------------------- From a10055cf6de058d10dec54705a6de746ecca111f Mon Sep 17 00:00:00 2001 From: Steve Loughran Date: Mon, 25 Mar 2013 13:12:27 +0000 Subject: [PATCH 03/37] HADOOP-9258 Add stricter tests to FileSystemContractTestBase (includes fixes for production code HADOOP-9261 & HADOOP-9265 and test enhancements HADOOP-9228, HADOOP-9227 & HADOOP-9259) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460646 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 7 + .../hadoop/fs/s3/Jets3tFileSystemStore.java | 22 +- .../org/apache/hadoop/fs/s3/S3FileSystem.java | 117 ++++++- .../fs/s3native/NativeS3FileSystem.java | 60 +++- .../hadoop/fs/FileSystemContractBaseTest.java | 299 +++++++++++++++++- .../hadoop/fs/s3/InMemoryFileSystemStore.java | 1 + .../NativeS3FileSystemContractBaseTest.java | 2 +- 7 files changed, 463 insertions(+), 45 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e8ce4fa6a8b..e959bc8b593 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -162,6 +162,7 @@ Trunk (Unreleased) HADOOP-9218 Document the Rpc-wrappers used internally (sanjay Radia) + HADOOP-9258 Add stricter tests to FileSystemContractTestBase (stevel) BUG FIXES @@ -355,6 +356,12 @@ Trunk (Unreleased) HADOOP-9431 TestSecurityUtil#testLocalHostNameForNullOrWild on systems where hostname contains capital letters (Chris Nauroth via sanjay) + HADOOP-9261 S3n filesystem can move a directory under itself -and so lose data + (fixed in HADOOP-9258) (stevel) + + HADOOP-9265 S3 blockstore filesystem breaks part of the Filesystem contract + (fixed in HADOOP-9258) (stevel) + OPTIMIZATIONS HADOOP-7761. Improve the performance of raw comparisons. (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java index 4adc306633a..60a548eeefa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java @@ -137,9 +137,15 @@ class Jets3tFileSystemStore implements FileSystemStore { @Override public boolean inodeExists(Path path) throws IOException { - InputStream in = get(pathToKey(path), true); + String key = pathToKey(path); + InputStream in = get(key, true); if (in == null) { - return false; + if (isRoot(key)) { + storeINode(path, INode.DIRECTORY_INODE); + return true; + } else { + return false; + } } in.close(); return true; @@ -211,7 +217,13 @@ class Jets3tFileSystemStore implements FileSystemStore { @Override public INode retrieveINode(Path path) throws IOException { - return INode.deserialize(get(pathToKey(path), true)); + String key = pathToKey(path); + InputStream in = get(key, true); + if (in == null && isRoot(key)) { + storeINode(path, INode.DIRECTORY_INODE); + return INode.DIRECTORY_INODE; + } + return INode.deserialize(in); } @Override @@ -366,6 +378,10 @@ class Jets3tFileSystemStore implements FileSystemStore { return blockToKey(block.getId()); } + private boolean isRoot(String key) { + return key.isEmpty() || key.equals("/"); + } + @Override public void purge() throws IOException { try { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java index 81ef31446e5..e49eefa1115 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java @@ -252,32 +252,125 @@ public class S3FileSystem extends FileSystem { @Override public boolean rename(Path src, Path dst) throws IOException { Path absoluteSrc = makeAbsolute(src); + final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - "; INode srcINode = store.retrieveINode(absoluteSrc); + boolean debugEnabled = LOG.isDebugEnabled(); if (srcINode == null) { // src path doesn't exist + if (debugEnabled) { + LOG.debug(debugPreamble + "returning false as src does not exist"); + } return false; } + Path absoluteDst = makeAbsolute(dst); - INode dstINode = store.retrieveINode(absoluteDst); - if (dstINode != null && dstINode.isDirectory()) { - absoluteDst = new Path(absoluteDst, absoluteSrc.getName()); - dstINode = store.retrieveINode(absoluteDst); - } - if (dstINode != null) { - // dst path already exists - can't overwrite - return false; - } + + //validate the parent dir of the destination Path dstParent = absoluteDst.getParent(); if (dstParent != null) { + //if the dst parent is not root, make sure it exists INode dstParentINode = store.retrieveINode(dstParent); - if (dstParentINode == null || dstParentINode.isFile()) { - // dst parent doesn't exist or is a file + if (dstParentINode == null) { + // dst parent doesn't exist + if (debugEnabled) { + LOG.debug(debugPreamble + + "returning false as dst parent does not exist"); + } + return false; + } + if (dstParentINode.isFile()) { + // dst parent exists but is a file + if (debugEnabled) { + LOG.debug(debugPreamble + + "returning false as dst parent exists and is a file"); + } return false; } } + + //get status of source + boolean srcIsFile = srcINode.isFile(); + + INode dstINode = store.retrieveINode(absoluteDst); + boolean destExists = dstINode != null; + boolean destIsDir = destExists && !dstINode.isFile(); + if (srcIsFile) { + + //source is a simple file + if (destExists) { + if (destIsDir) { + //outcome #1 dest exists and is dir -filename to subdir of dest + if (debugEnabled) { + LOG.debug(debugPreamble + + "copying src file under dest dir to " + absoluteDst); + } + absoluteDst = new Path(absoluteDst, absoluteSrc.getName()); + } else { + //outcome #2 dest it's a file: fail iff different from src + boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst); + if (debugEnabled) { + LOG.debug(debugPreamble + + "copying file onto file, outcome is " + renamingOnToSelf); + } + return renamingOnToSelf; + } + } else { + // #3 dest does not exist: use dest as path for rename + if (debugEnabled) { + LOG.debug(debugPreamble + + "copying file onto file"); + } + } + } else { + //here the source exists and is a directory + // outcomes (given we know the parent dir exists if we get this far) + // #1 destination is a file: fail + // #2 destination is a directory: create a new dir under that one + // #3 destination doesn't exist: create a new dir with that name + // #3 and #4 are only allowed if the dest path is not == or under src + + if (destExists) { + if (!destIsDir) { + // #1 destination is a file: fail + if (debugEnabled) { + LOG.debug(debugPreamble + + "returning false as src is a directory, but not dest"); + } + return false; + } else { + // the destination dir exists + // destination for rename becomes a subdir of the target name + absoluteDst = new Path(absoluteDst, absoluteSrc.getName()); + if (debugEnabled) { + LOG.debug(debugPreamble + + "copying src dir under dest dir to " + absoluteDst); + } + } + } + //the final destination directory is now know, so validate it for + //illegal moves + + if (absoluteSrc.equals(absoluteDst)) { + //you can't rename a directory onto itself + if (debugEnabled) { + LOG.debug(debugPreamble + + "Dest==source && isDir -failing"); + } + return false; + } + if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) { + //you can't move a directory under itself + if (debugEnabled) { + LOG.debug(debugPreamble + + "dst is equal to or under src dir -failing"); + } + return false; + } + } + //here the dest path is set up -so rename return renameRecursive(absoluteSrc, absoluteDst); } - + private boolean renameRecursive(Path src, Path dst) throws IOException { INode srcINode = store.retrieveINode(src); store.storeINode(dst, srcINode); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java index efc415f89e3..276d6e5fdb8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java @@ -582,35 +582,58 @@ public class NativeS3FileSystem extends FileSystem { public boolean rename(Path src, Path dst) throws IOException { String srcKey = pathToKey(makeAbsolute(src)); + final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - "; if (srcKey.length() == 0) { // Cannot rename root of file system + if (LOG.isDebugEnabled()) { + LOG.debug(debugPreamble + + "returning false as cannot rename the root of a filesystem"); + } return false; } - final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - "; - + //get status of source + boolean srcIsFile; + try { + srcIsFile = getFileStatus(src).isFile(); + } catch (FileNotFoundException e) { + //bail out fast if the source does not exist + if (LOG.isDebugEnabled()) { + LOG.debug(debugPreamble + "returning false as src does not exist"); + } + return false; + } // Figure out the final destination - String dstKey; + String dstKey = pathToKey(makeAbsolute(dst)); + try { boolean dstIsFile = getFileStatus(dst).isFile(); if (dstIsFile) { + //destination is a file. + //you can't copy a file or a directory onto an existing file + //except for the special case of dest==src, which is a no-op if(LOG.isDebugEnabled()) { LOG.debug(debugPreamble + - "returning false as dst is an already existing file"); + "returning without rename as dst is an already existing file"); } - return false; + //exit, returning true iff the rename is onto self + return srcKey.equals(dstKey); } else { + //destination exists and is a directory if(LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "using dst as output directory"); } + //destination goes under the dst path, with the name of the + //source entry dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName()))); } } catch (FileNotFoundException e) { + //destination does not exist => the source file or directory + //is copied over with the name of the destination if(LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "using dst as output destination"); } - dstKey = pathToKey(makeAbsolute(dst)); try { if (getFileStatus(dst.getParent()).isFile()) { if(LOG.isDebugEnabled()) { @@ -628,16 +651,17 @@ public class NativeS3FileSystem extends FileSystem { } } - boolean srcIsFile; - try { - srcIsFile = getFileStatus(src).isFile(); - } catch (FileNotFoundException e) { - if(LOG.isDebugEnabled()) { - LOG.debug(debugPreamble + "returning false as src does not exist"); + //rename to self behavior follows Posix rules and is different + //for directories and files -the return code is driven by src type + if (srcKey.equals(dstKey)) { + //fully resolved destination key matches source: fail + if (LOG.isDebugEnabled()) { + LOG.debug(debugPreamble + "renamingToSelf; returning true"); } - return false; + return true; } if (srcIsFile) { + //source is a file; COPY then DELETE if(LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "src is file, so doing copy then delete in S3"); @@ -645,9 +669,19 @@ public class NativeS3FileSystem extends FileSystem { store.copy(srcKey, dstKey); store.delete(srcKey); } else { + //src is a directory if(LOG.isDebugEnabled()) { LOG.debug(debugPreamble + "src is directory, so copying contents"); } + //Verify dest is not a child of the parent + if (dstKey.startsWith(srcKey + "/")) { + if (LOG.isDebugEnabled()) { + LOG.debug( + debugPreamble + "cannot rename a directory to a subdirectory of self"); + } + return false; + } + //create the subdir under the destination store.storeEmptyFile(dstKey + FOLDER_SUFFIX); List keysToDelete = new ArrayList(); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java index ea309a768a8..95999cd9f69 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Locale; import junit.framework.TestCase; @@ -51,7 +52,13 @@ public abstract class FileSystemContractBaseTest extends TestCase { @Override protected void tearDown() throws Exception { - fs.delete(path("/test"), true); + try { + if (fs != null) { + fs.delete(path("/test"), true); + } + } catch (IOException e) { + LOG.error("Error deleting /test: " + e, e); + } } protected int getBlockSize() { @@ -62,10 +69,23 @@ public abstract class FileSystemContractBaseTest extends TestCase { return "/user/" + System.getProperty("user.name"); } + /** + * Override this if the filesystem does not support rename + * @return true if the FS supports rename -and rename related tests + * should be run + */ protected boolean renameSupported() { return true; } + /** + * Override this if the filesystem is not case sensitive + * @return true if the case detection/preservation tests should run + */ + protected boolean filesystemIsCaseSensitive() { + return true; + } + public void testFsStatus() throws Exception { FsStatus fsStatus = fs.getStatus(); assertNotNull(fsStatus); @@ -109,6 +129,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { assertTrue(fs.mkdirs(testDir)); assertTrue(fs.exists(testDir)); + assertTrue("Should be a directory", fs.isDirectory(testDir)); assertFalse(fs.isFile(testDir)); Path parentDir = testDir.getParent(); @@ -118,17 +139,17 @@ public abstract class FileSystemContractBaseTest extends TestCase { Path grandparentDir = parentDir.getParent(); assertTrue(fs.exists(grandparentDir)); assertFalse(fs.isFile(grandparentDir)); - + } - + public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { Path testDir = path("/test/hadoop"); assertFalse(fs.exists(testDir)); assertTrue(fs.mkdirs(testDir)); assertTrue(fs.exists(testDir)); - + createFile(path("/test/hadoop/file")); - + Path testSubDir = path("/test/hadoop/file/subdir"); try { fs.mkdirs(testSubDir); @@ -137,7 +158,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { // expected } assertFalse(fs.exists(testSubDir)); - + Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir"); try { fs.mkdirs(testDeepSubDir); @@ -146,7 +167,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { // expected } assertFalse(fs.exists(testDeepSubDir)); - + } public void testMkdirsWithUmask() throws Exception { @@ -177,7 +198,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { // expected } } - + public void testListStatusThrowsExceptionForNonExistentFile() throws Exception { try { fs.listStatus(path("/test/hadoop/file")); @@ -186,7 +207,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { // expected } } - + public void testListStatus() throws Exception { Path[] testDirs = { path("/test/hadoop/a"), path("/test/hadoop/b"), @@ -210,7 +231,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { paths = fs.listStatus(path("/test/hadoop/a")); assertEquals(0, paths.length); } - + public void testWriteReadAndDeleteEmptyFile() throws Exception { writeReadAndDelete(0); } @@ -222,7 +243,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { public void testWriteReadAndDeleteOneBlock() throws Exception { writeReadAndDelete(getBlockSize()); } - + public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception { writeReadAndDelete(getBlockSize() + (getBlockSize() / 2)); } @@ -365,8 +386,7 @@ public abstract class FileSystemContractBaseTest extends TestCase { Path dst = path("/test/new/newdir"); fs.mkdirs(dst); rename(src, dst, true, false, true); - assertTrue("Destination changed", - fs.exists(path("/test/new/newdir/file"))); + assertIsFile(path("/test/new/newdir/file")); } public void testRenameDirectoryMoveToNonExistentDirectory() @@ -466,9 +486,9 @@ public abstract class FileSystemContractBaseTest extends TestCase { private void rename(Path src, Path dst, boolean renameSucceeded, boolean srcExists, boolean dstExists) throws IOException { - assertEquals("Rename result", renameSucceeded, fs.rename(src, dst)); - assertEquals("Source exists", srcExists, fs.exists(src)); - assertEquals("Destination exists", dstExists, fs.exists(dst)); + assertEquals("mv " + src + " " + dst,renameSucceeded, fs.rename(src, dst)); + assertEquals("Source exists: " + src, srcExists, fs.exists(src)); + assertEquals("Destination exists" + dst, dstExists, fs.exists(dst)); } /** @@ -494,6 +514,253 @@ public abstract class FileSystemContractBaseTest extends TestCase { writeAndRead(path, filedata2, blockSize * 2, true, false); } + /** + * Assert that a filesystem is case sensitive. + * This is done by creating a mixed-case filename and asserting that + * its lower case version is not there. + * @throws Exception + */ + public void testFilesystemIsCaseSensitive() throws Exception { + if (!filesystemIsCaseSensitive()) { + LOG.info("Skipping test"); + return; + } + String mixedCaseFilename = "/test/UPPER.TXT"; + Path upper = path(mixedCaseFilename); + Path lower = path(mixedCaseFilename.toLowerCase(Locale.ENGLISH)); + assertFalse("File exists" + upper, fs.exists(upper)); + assertFalse("File exists" + lower, fs.exists(lower)); + FSDataOutputStream out = fs.create(upper); + out.writeUTF("UPPER"); + out.close(); + FileStatus upperStatus = fs.getFileStatus(upper); + assertTrue("File does not exist" + upper, fs.exists(upper)); + //verify the lower-case version of the filename doesn't exist + assertFalse("File exists" + lower, fs.exists(lower)); + //now overwrite the lower case version of the filename with a + //new version. + out = fs.create(lower); + out.writeUTF("l"); + out.close(); + assertTrue("File does not exist" + lower, fs.exists(lower)); + //verify the length of the upper file hasn't changed + FileStatus newStatus = fs.getFileStatus(upper); + assertEquals("Expected status:" + upperStatus + + " actual status " + newStatus, + upperStatus.getLen(), + newStatus.getLen()); } + + /** + * Asserts that a zero byte file has a status of file and not + * directory or symlink + * @throws Exception on failures + */ + public void testZeroByteFilesAreFiles() throws Exception { + Path src = path("/test/testZeroByteFilesAreFiles"); + //create a zero byte file + FSDataOutputStream out = fs.create(src); + out.close(); + assertIsFile(src); + } + + /** + * Asserts that a zero byte file has a status of file and not + * directory or symlink + * @throws Exception on failures + */ + public void testMultiByteFilesAreFiles() throws Exception { + Path src = path("/test/testMultiByteFilesAreFiles"); + FSDataOutputStream out = fs.create(src); + out.writeUTF("testMultiByteFilesAreFiles"); + out.close(); + assertIsFile(src); + } + + /** + * Assert that root directory renames are not allowed + * @throws Exception on failures + */ + public void testRootDirAlwaysExists() throws Exception { + //this will throw an exception if the path is not found + fs.getFileStatus(path("/")); + //this catches overrides of the base exists() method that don't + //use getFileStatus() as an existence probe + assertTrue("FileSystem.exists() fails for root", fs.exists(path("/"))); + } + + /** + * Assert that root directory renames are not allowed + * @throws Exception on failures + */ + public void testRenameRootDirForbidden() throws Exception { + if (!renameSupported()) return; + + rename(path("/"), + path("/test/newRootDir"), + false, true, false); + } + + /** + * Assert that renaming a parent directory to be a child + * of itself is forbidden + * @throws Exception on failures + */ + public void testRenameChildDirForbidden() throws Exception { + if (!renameSupported()) return; + LOG.info("testRenameChildDirForbidden"); + Path parentdir = path("/test/parentdir"); + fs.mkdirs(parentdir); + Path childFile = new Path(parentdir, "childfile"); + createFile(childFile); + //verify one level down + Path childdir = new Path(parentdir, "childdir"); + rename(parentdir, childdir, false, true, false); + //now another level + fs.mkdirs(childdir); + Path childchilddir = new Path(childdir, "childdir"); + rename(parentdir, childchilddir, false, true, false); + } + + /** + * This a sanity check to make sure that any filesystem's handling of + * renames doesn't cause any regressions + */ + public void testRenameToDirWithSamePrefixAllowed() throws Throwable { + if (!renameSupported()) return; + Path parentdir = path("test/parentdir"); + fs.mkdirs(parentdir); + Path dest = path("test/parentdirdest"); + rename(parentdir, dest, true, false, true); + } + + /** + * trying to rename a directory onto itself should fail, + * preserving everything underneath. + */ + public void testRenameDirToSelf() throws Throwable { + if (!renameSupported()) { + return; + } + Path parentdir = path("test/parentdir"); + fs.mkdirs(parentdir); + Path child = new Path(parentdir, "child"); + createFile(child); + + rename(parentdir, parentdir, false, true, true); + //verify the child is still there + assertIsFile(child); + } + + /** + * trying to rename a directory onto its parent dir will build + * a destination path of its original name, which should then fail. + * The source path and the destination path should still exist afterwards + */ + public void testMoveDirUnderParent() throws Throwable { + if (!renameSupported()) { + return; + } + Path testdir = path("test/dir"); + fs.mkdirs(testdir); + Path parent = testdir.getParent(); + //the outcome here is ambiguous, so is not checked + fs.rename(testdir, parent); + assertEquals("Source exists: " + testdir, true, fs.exists(testdir)); + assertEquals("Destination exists" + parent, true, fs.exists(parent)); + } + + /** + * trying to rename a file onto itself should succeed (it's a no-op) + * + */ + public void testRenameFileToSelf() throws Throwable { + if (!renameSupported()) return; + Path filepath = path("test/file"); + createFile(filepath); + //HDFS expects rename src, src -> true + rename(filepath, filepath, true, true, true); + //verify the file is still there + assertIsFile(filepath); + } + + /** + * trying to move a file into it's parent dir should succeed + * again: no-op + */ + public void testMoveFileUnderParent() throws Throwable { + if (!renameSupported()) return; + Path filepath = path("test/file"); + createFile(filepath); + //HDFS expects rename src, src -> true + rename(filepath, filepath, true, true, true); + //verify the file is still there + assertIsFile(filepath); + } + + public void testLSRootDir() throws Throwable { + Path dir = path("/"); + Path child = path("/test"); + createFile(child); + assertListFilesFinds(dir, child); + } + + public void testListStatusRootDir() throws Throwable { + Path dir = path("/"); + Path child = path("/test"); + createFile(child); + assertListStatusFinds(dir, child); + } + + private void assertListFilesFinds(Path dir, Path subdir) throws IOException { + RemoteIterator iterator = + fs.listFiles(dir, true); + boolean found = false; + StringBuilder builder = new StringBuilder(); + while (iterator.hasNext()) { + LocatedFileStatus next = iterator.next(); + builder.append(next.toString()).append('\n'); + if (next.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + private void assertListStatusFinds(Path dir, Path subdir) throws IOException { + FileStatus[] stats = fs.listStatus(dir); + boolean found = false; + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.toString()).append('\n'); + if (stat.getPath().equals(subdir)) { + found = true; + } + } + assertTrue("Path " + subdir + + " not found in directory " + dir + ":" + builder, + found); + } + + + /** + * Assert that a file exists and whose {@link FileStatus} entry + * declares that this is a file and not a symlink or directory. + * @param filename name of the file + * @throws IOException IO problems during file operations + */ + private void assertIsFile(Path filename) throws IOException { + assertTrue("Does not exist: " + filename, fs.exists(filename)); + FileStatus status = fs.getFileStatus(filename); + String fileInfo = filename + " " + status; + assertTrue("Not a file " + fileInfo, status.isFile()); + assertFalse("File claims to be a symlink " + fileInfo, + status.isSymlink()); + assertFalse("File claims to be a directory " + fileInfo, + status.isDirectory()); + } + /** * * Write a file and read it in, validating the result. Optional flags control diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java index 8024c6acc71..59a83977221 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java @@ -50,6 +50,7 @@ class InMemoryFileSystemStore implements FileSystemStore { @Override public void initialize(URI uri, Configuration conf) { this.conf = conf; + inodes.put(new Path("/"), INode.DIRECTORY_INODE); } @Override diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java index 60010e43644..220e0bd4730 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java @@ -51,7 +51,7 @@ public abstract class NativeS3FileSystemContractBaseTest public void testListStatusForRoot() throws Exception { FileStatus[] paths = fs.listStatus(path("/")); - assertEquals(0, paths.length); + assertEquals("Root directory is not empty; ", 0, paths.length); Path testDir = path("/test"); assertTrue(fs.mkdirs(testDir)); From c02dd9aa4b7e6dce4549f124ee636a99ff96ea67 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 25 Mar 2013 15:46:25 +0000 Subject: [PATCH 04/37] YARN-109. .tmp file is not deleted for localized archives (Mayank Bansal via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460723 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../apache/hadoop/yarn/util/FSDownload.java | 24 +- .../hadoop/yarn/util/TestFSDownload.java | 298 +++++++++++++++++- 3 files changed, 313 insertions(+), 12 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 36e57312d7f..96bff2f0f2c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -463,6 +463,9 @@ Release 0.23.7 - UNRELEASED YARN-345. Many InvalidStateTransitonException errors for ApplicationImpl in Node Manager (Robert Parker via jlowe) + YARN-109. .tmp file is not deleted for localized archives (Mayank Bansal + via bobby) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java index 627d565d4d0..5253f49e87d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java @@ -251,6 +251,12 @@ public class FSDownload implements Callable { } break; } + if(localrsrc.isFile()){ + try { + files.delete(new Path(localrsrc.toString()), false); + } catch (IOException ignore) { + } + } return 0; // TODO Should calculate here before returning //return FileUtil.getDU(destDir); @@ -264,41 +270,41 @@ public class FSDownload implements Callable { } catch (URISyntaxException e) { throw new IOException("Invalid resource", e); } - Path tmp; do { tmp = new Path(destDirPath, String.valueOf(rand.nextLong())); } while (files.util().exists(tmp)); destDirPath = tmp; - createDir(destDirPath, cachePerms); final Path dst_work = new Path(destDirPath + "_tmp"); createDir(dst_work, cachePerms); - Path dFinal = files.makeQualified(new Path(dst_work, sCopy.getName())); try { - Path dTmp = null == userUgi - ? files.makeQualified(copy(sCopy, dst_work)) - : userUgi.doAs(new PrivilegedExceptionAction() { + Path dTmp = null == userUgi ? files.makeQualified(copy(sCopy, dst_work)) + : userUgi.doAs(new PrivilegedExceptionAction() { public Path run() throws Exception { return files.makeQualified(copy(sCopy, dst_work)); }; }); Pattern pattern = null; String p = resource.getPattern(); - if(p != null) { + if (p != null) { pattern = Pattern.compile(p); } unpack(new File(dTmp.toUri()), new File(dFinal.toUri()), pattern); changePermissions(dFinal.getFileSystem(conf), dFinal); files.rename(dst_work, destDirPath, Rename.OVERWRITE); } catch (Exception e) { - try { files.delete(destDirPath, true); } catch (IOException ignore) { } + try { + files.delete(destDirPath, true); + } catch (IOException ignore) { + } throw e; } finally { try { files.delete(dst_work, true); - } catch (FileNotFoundException ignore) { } + } catch (FileNotFoundException ignore) { + } // clear ref to internal var rand = null; conf = null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java index c29b69f1148..912c7248ca5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.util.EnumSet; import java.util.HashMap; @@ -47,10 +48,12 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -113,7 +116,127 @@ public class TestFSDownload { return ret; } - @Test + static LocalResource createTarFile(FileContext files, Path p, int len, + Random r, LocalResourceVisibility vis) throws IOException, + URISyntaxException { + + FSDataOutputStream outFile = null; + try { + byte[] bytes = new byte[len]; + Path tarPath = new Path(p.toString()); + outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE)); + r.nextBytes(bytes); + outFile.write(bytes); + } finally { + if (outFile != null) + outFile.close(); + } + StringBuffer tarCommand = new StringBuffer(); + URI u = new URI(p.getParent().toString()); + tarCommand.append("cd '"); + tarCommand.append(FileUtil.makeShellPath(u.getPath().toString())); + tarCommand.append("' ; "); + tarCommand.append("tar -czf " + p.getName() + ".tar " + p.getName()); + String[] shellCmd = { "bash", "-c", tarCommand.toString() }; + ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); + shexec.execute(); + int exitcode = shexec.getExitCode(); + if (exitcode != 0) { + throw new IOException("Error untarring file " + p + + ". Tar process exited with exit code " + exitcode); + } + LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); + ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + + ".tar"))); + ret.setSize(len); + ret.setType(LocalResourceType.ARCHIVE); + ret.setVisibility(vis); + ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar")) + .getModificationTime()); + return ret; + } + + static LocalResource createJarFile(FileContext files, Path p, int len, + Random r, LocalResourceVisibility vis) throws IOException, + URISyntaxException { + + FSDataOutputStream outFile = null; + try { + byte[] bytes = new byte[len]; + Path tarPath = new Path(p.toString()); + outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE)); + r.nextBytes(bytes); + outFile.write(bytes); + } finally { + if (outFile != null) + outFile.close(); + } + StringBuffer tarCommand = new StringBuffer(); + URI u = new URI(p.getParent().toString()); + tarCommand.append("cd '"); + tarCommand.append(FileUtil.makeShellPath(u.getPath().toString())); + tarCommand.append("' ; "); + tarCommand.append("jar cf " + p.getName() + ".jar " + p.getName()); + String[] shellCmd = { "bash", "-c", tarCommand.toString() }; + ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); + shexec.execute(); + int exitcode = shexec.getExitCode(); + if (exitcode != 0) { + throw new IOException("Error untarring file " + p + + ". Tar process exited with exit code " + exitcode); + } + LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); + ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + + ".jar"))); + ret.setSize(len); + ret.setType(LocalResourceType.ARCHIVE); + ret.setVisibility(vis); + ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar")) + .getModificationTime()); + return ret; + } + + static LocalResource createZipFile(FileContext files, Path p, int len, + Random r, LocalResourceVisibility vis) throws IOException, + URISyntaxException { + + FSDataOutputStream outFile = null; + try { + byte[] bytes = new byte[len]; + Path tarPath = new Path(p.toString()); + outFile = files.create(tarPath, EnumSet.of(CREATE, OVERWRITE)); + r.nextBytes(bytes); + outFile.write(bytes); + } finally { + if (outFile != null) + outFile.close(); + } + StringBuffer zipCommand = new StringBuffer(); + URI u = new URI(p.getParent().toString()); + zipCommand.append("cd '"); + zipCommand.append(FileUtil.makeShellPath(u.getPath().toString())); + zipCommand.append("' ; "); + zipCommand.append("gzip " + p.getName()); + String[] shellCmd = { "bash", "-c", zipCommand.toString() }; + ShellCommandExecutor shexec = new ShellCommandExecutor(shellCmd); + shexec.execute(); + int exitcode = shexec.getExitCode(); + if (exitcode != 0) { + throw new IOException("Error untarring file " + p + + ". Tar process exited with exit code " + exitcode); + } + LocalResource ret = recordFactory.newRecordInstance(LocalResource.class); + ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString() + + ".zip"))); + ret.setSize(len); + ret.setType(LocalResourceType.ARCHIVE); + ret.setVisibility(vis); + ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".gz")) + .getModificationTime()); + return ret; + } + + @Test (timeout=10000) public void testDownloadBadPublic() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); @@ -161,7 +284,7 @@ public class TestFSDownload { } } - @Test + @Test (timeout=10000) public void testDownload() throws IOException, URISyntaxException, InterruptedException { Configuration conf = new Configuration(); @@ -229,6 +352,175 @@ public class TestFSDownload { } } + @SuppressWarnings("deprecation") + @Test (timeout=10000) + public void testDownloadArchive() throws IOException, URISyntaxException, + InterruptedException { + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + FileContext files = FileContext.getLocalFSFileContext(conf); + final Path basedir = files.makeQualified(new Path("target", + TestFSDownload.class.getSimpleName())); + files.mkdir(basedir, null, true); + conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); + + Random rand = new Random(); + long sharedSeed = rand.nextLong(); + rand.setSeed(sharedSeed); + System.out.println("SEED: " + sharedSeed); + + Map> pending = new HashMap>(); + ExecutorService exec = Executors.newSingleThreadExecutor(); + LocalDirAllocator dirs = new LocalDirAllocator( + TestFSDownload.class.getName()); + + int size = rand.nextInt(512) + 512; + LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; + + Path p = new Path(basedir, "" + 1); + LocalResource rsrc = createTarFile(files, p, size, rand, vis); + Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf); + FSDownload fsd = new FSDownload(files, + UserGroupInformation.getCurrentUser(), conf, destPath, rsrc, + new Random(sharedSeed)); + pending.put(rsrc, exec.submit(fsd)); + + try { + FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus( + basedir); + for (FileStatus filestatus : filesstatus) { + if (filestatus.isDir()) { + FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( + filestatus.getPath()); + for (FileStatus childfile : childFiles) { + if (childfile.getPath().getName().equalsIgnoreCase("1.tar.tmp")) { + Assert.fail("Tmp File should not have been there " + + childfile.getPath()); + } + } + } + } + }catch (Exception e) { + throw new IOException("Failed exec", e); + } + finally { + exec.shutdown(); + } + } + + @SuppressWarnings("deprecation") + @Test (timeout=10000) + public void testDownloadPatternJar() throws IOException, URISyntaxException, + InterruptedException { + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + FileContext files = FileContext.getLocalFSFileContext(conf); + final Path basedir = files.makeQualified(new Path("target", + TestFSDownload.class.getSimpleName())); + files.mkdir(basedir, null, true); + conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); + + Random rand = new Random(); + long sharedSeed = rand.nextLong(); + rand.setSeed(sharedSeed); + System.out.println("SEED: " + sharedSeed); + + Map> pending = new HashMap>(); + ExecutorService exec = Executors.newSingleThreadExecutor(); + LocalDirAllocator dirs = new LocalDirAllocator( + TestFSDownload.class.getName()); + + int size = rand.nextInt(512) + 512; + LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; + + Path p = new Path(basedir, "" + 1); + LocalResource rsrcjar = createJarFile(files, p, size, rand, vis); + rsrcjar.setType(LocalResourceType.PATTERN); + Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf); + FSDownload fsdjar = new FSDownload(files, + UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrcjar, + new Random(sharedSeed)); + pending.put(rsrcjar, exec.submit(fsdjar)); + + try { + FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus( + basedir); + for (FileStatus filestatus : filesstatus) { + if (filestatus.isDir()) { + FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( + filestatus.getPath()); + for (FileStatus childfile : childFiles) { + if (childfile.getPath().getName().equalsIgnoreCase("1.jar.tmp")) { + Assert.fail("Tmp File should not have been there " + + childfile.getPath()); + } + } + } + } + }catch (Exception e) { + throw new IOException("Failed exec", e); + } + finally { + exec.shutdown(); + } + } + + @SuppressWarnings("deprecation") + @Test (timeout=10000) + public void testDownloadArchiveZip() throws IOException, URISyntaxException, + InterruptedException { + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); + FileContext files = FileContext.getLocalFSFileContext(conf); + final Path basedir = files.makeQualified(new Path("target", + TestFSDownload.class.getSimpleName())); + files.mkdir(basedir, null, true); + conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); + + Random rand = new Random(); + long sharedSeed = rand.nextLong(); + rand.setSeed(sharedSeed); + System.out.println("SEED: " + sharedSeed); + + Map> pending = new HashMap>(); + ExecutorService exec = Executors.newSingleThreadExecutor(); + LocalDirAllocator dirs = new LocalDirAllocator( + TestFSDownload.class.getName()); + + int size = rand.nextInt(512) + 512; + LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE; + + Path p = new Path(basedir, "" + 1); + LocalResource rsrczip = createZipFile(files, p, size, rand, vis); + Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf); + FSDownload fsdzip = new FSDownload(files, + UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrczip, + new Random(sharedSeed)); + pending.put(rsrczip, exec.submit(fsdzip)); + + try { + FileStatus[] filesstatus = files.getDefaultFileSystem().listStatus( + basedir); + for (FileStatus filestatus : filesstatus) { + if (filestatus.isDir()) { + FileStatus[] childFiles = files.getDefaultFileSystem().listStatus( + filestatus.getPath()); + for (FileStatus childfile : childFiles) { + if (childfile.getPath().getName().equalsIgnoreCase("1.gz.tmp")) { + Assert.fail("Tmp File should not have been there " + + childfile.getPath()); + } + } + } + } + }catch (Exception e) { + throw new IOException("Failed exec", e); + } + finally { + exec.shutdown(); + } + } + private void verifyPermsRecursively(FileSystem fs, FileContext files, Path p, LocalResourceVisibility vis) throws IOException { @@ -261,7 +553,7 @@ public class TestFSDownload { } } - @Test + @Test (timeout=10000) public void testDirDownload() throws IOException, InterruptedException { Configuration conf = new Configuration(); FileContext files = FileContext.getLocalFSFileContext(conf); From 6a482a88b8f56a4c5590e71ce6713d7f63830e92 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Mon, 25 Mar 2013 18:23:59 +0000 Subject: [PATCH 05/37] YARN-71. Fix the NodeManager to clean up local-dirs on restart. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460808 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/server/nodemanager/NodeManager.java | 13 +- .../ResourceLocalizationService.java | 79 ++++- .../nodemanager/TestNodeManagerReboot.java | 297 ++++++++++++++++++ 4 files changed, 390 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 96bff2f0f2c..c3c13eea3ac 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -124,6 +124,9 @@ Release 2.0.5-beta - UNRELEASED YARN-470. Support a way to disable resource monitoring on the NodeManager. (Siddharth Seth via hitesh) + + YARN-71. Fix the NodeManager to clean up local-dirs on restart. + (Xuan Gong via sseth) Release 2.0.4-alpha - UNRELEASED diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 5aa947f32b8..7a53eb9033a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.service.CompositeService; import org.apache.hadoop.yarn.service.Service; import org.apache.hadoop.yarn.util.Records; +import com.google.common.annotations.VisibleForTesting; + public class NodeManager extends CompositeService implements EventHandler { @@ -113,6 +115,10 @@ public class NodeManager extends CompositeService return new WebServer(nmContext, resourceView, aclsManager, dirsHandler); } + protected DeletionService createDeletionService(ContainerExecutor exec) { + return new DeletionService(exec); + } + protected void doSecureLogin() throws IOException { SecurityUtil.login(getConfig(), YarnConfiguration.NM_KEYTAB, YarnConfiguration.NM_PRINCIPAL); @@ -143,7 +149,7 @@ public class NodeManager extends CompositeService } catch (IOException e) { throw new YarnException("Failed to initialize container executor", e); } - DeletionService del = new DeletionService(exec); + DeletionService del = createDeletionService(exec); addService(del); // NodeManager level dispatcher @@ -351,6 +357,11 @@ public class NodeManager extends CompositeService return containerManager; } + @VisibleForTesting + Context getNMContext() { + return this.context; + } + public static void main(String[] args) { Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler()); StringUtils.startupShutdownMessage(NodeManager.class, args, LOG); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 9ca812e1ddd..29971c5b65b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; import java.io.DataOutputStream; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.InetSocketAddress; import java.net.URISyntaxException; @@ -53,8 +54,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.Credentials; @@ -175,9 +178,11 @@ public class ResourceLocalizationService extends CompositeService this.recordFactory = RecordFactoryProvider.getRecordFactory(conf); try { - // TODO queue deletions here, rather than NM init? FileContext lfs = getLocalFileContext(conf); lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK)); + + cleanUpLocalDir(lfs,delService); + List localDirs = dirsHandler.getLocalDirs(); for (String localDir : localDirs) { // $local/usercache @@ -926,4 +931,76 @@ public class ResourceLocalizationService extends CompositeService } + private void cleanUpLocalDir(FileContext lfs, DeletionService del) { + long currentTimeStamp = System.currentTimeMillis(); + for (String localDir : dirsHandler.getLocalDirs()) { + renameLocalDir(lfs, localDir, ContainerLocalizer.USERCACHE, + currentTimeStamp); + renameLocalDir(lfs, localDir, ContainerLocalizer.FILECACHE, + currentTimeStamp); + renameLocalDir(lfs, localDir, ResourceLocalizationService.NM_PRIVATE_DIR, + currentTimeStamp); + try { + deleteLocalDir(lfs, del, localDir); + } catch (IOException e) { + // Do nothing, just give the warning + LOG.warn("Failed to delete localDir: " + localDir); + } + } + } + + private void renameLocalDir(FileContext lfs, String localDir, + String localSubDir, long currentTimeStamp) { + try { + lfs.rename(new Path(localDir, localSubDir), new Path( + localDir, localSubDir + "_DEL_" + currentTimeStamp)); + } catch (FileNotFoundException ex) { + // No need to handle this exception + // localSubDir may not be exist + } catch (Exception ex) { + // Do nothing, just give the warning + LOG.warn("Failed to rename the local file under " + + localDir + "/" + localSubDir); + } + } + + private void deleteLocalDir(FileContext lfs, DeletionService del, + String localDir) throws IOException { + RemoteIterator fileStatus = lfs.listStatus(new Path(localDir)); + if (fileStatus != null) { + while (fileStatus.hasNext()) { + FileStatus status = fileStatus.next(); + try { + if (status.getPath().getName().matches(".*" + + ContainerLocalizer.USERCACHE + "_DEL_.*")) { + cleanUpFilesFromSubDir(lfs, del, status.getPath()); + } else if (status.getPath().getName() + .matches(".*" + NM_PRIVATE_DIR + "_DEL_.*") + || + status.getPath().getName() + .matches(".*" + ContainerLocalizer.FILECACHE + "_DEL_.*")) { + del.delete(null, status.getPath(), new Path[] {}); + } + } catch (IOException ex) { + // Do nothing, just give the warning + LOG.warn("Failed to delete this local Directory: " + + status.getPath().getName()); + } + } + } + } + + private void cleanUpFilesFromSubDir(FileContext lfs, DeletionService del, + Path dirPath) throws IOException { + RemoteIterator fileStatus = lfs.listStatus(dirPath); + if (fileStatus != null) { + while (fileStatus.hasNext()) { + FileStatus status = fileStatus.next(); + String owner = status.getOwner(); + del.delete(owner, status.getPath(), new Path[] {}); + } + } + del.delete(null, dirPath, new Path[] {}); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java new file mode 100644 index 00000000000..10a85c74804 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java @@ -0,0 +1,297 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager; + +import static org.mockito.Mockito.*; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest; +import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatcher; + +public class TestNodeManagerReboot { + + static final File basedir = + new File("target", TestNodeManagerReboot.class.getName()); + static final File logsDir = new File(basedir, "logs"); + static final File nmLocalDir = new File(basedir, "nm0"); + static final File localResourceDir = new File(basedir, "resource"); + + static final String user = System.getProperty("user.name"); + private FileContext localFS; + + private MyNodeManager nm; + private DeletionService delService; + static final Log LOG = LogFactory.getLog(TestNodeManagerReboot.class); + + @Before + public void setup() throws UnsupportedFileSystemException { + localFS = FileContext.getLocalFSFileContext(); + } + + @After + public void tearDown() throws IOException, InterruptedException { + localFS.delete(new Path(basedir.getPath()), true); + if (nm != null) { + nm.stop(); + } + } + + @Test(timeout = 20000) + public void testClearLocalDirWhenNodeReboot() throws IOException { + nm = new MyNodeManager(); + nm.start(); + // create files under fileCache + createFiles(nmLocalDir.getAbsolutePath(), ContainerLocalizer.FILECACHE, 100); + localResourceDir.mkdirs(); + ContainerManagerImpl containerManager = nm.getContainerManager(); + + ContainerLaunchContext containerLaunchContext = + Records.newRecord(ContainerLaunchContext.class); + // Construct the Container-id + ContainerId cId = createContainerId(); + containerLaunchContext.setContainerId(cId); + + containerLaunchContext.setUser(user); + + URL localResourceUri = + ConverterUtils.getYarnUrlFromPath(localFS + .makeQualified(new Path(localResourceDir.getAbsolutePath()))); + + LocalResource localResource = + Records.newRecord(LocalResource.class); + localResource.setResource(localResourceUri); + localResource.setSize(-1); + localResource.setVisibility(LocalResourceVisibility.APPLICATION); + localResource.setType(LocalResourceType.FILE); + localResource.setTimestamp(localResourceDir.lastModified()); + String destinationFile = "dest_file"; + Map localResources = + new HashMap(); + localResources.put(destinationFile, localResource); + containerLaunchContext.setLocalResources(localResources); + containerLaunchContext.setUser(containerLaunchContext.getUser()); + List commands = new ArrayList(); + containerLaunchContext.setCommands(commands); + containerLaunchContext.setResource(Records + .newRecord(Resource.class)); + containerLaunchContext.getResource().setMemory(1024); + StartContainerRequest startRequest = + Records.newRecord(StartContainerRequest.class); + startRequest.setContainerLaunchContext(containerLaunchContext); + containerManager.startContainer(startRequest); + + GetContainerStatusRequest request = + Records.newRecord(GetContainerStatusRequest.class); + request.setContainerId(cId); + Container container = + nm.getNMContext().getContainers().get(request.getContainerId()); + + final int MAX_TRIES = 20; + int numTries = 0; + while (!container.getContainerState().equals(ContainerState.DONE) + && numTries <= MAX_TRIES) { + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + // Do nothing + } + numTries++; + } + + Assert.assertEquals(ContainerState.DONE, container.getContainerState()); + + Assert.assertTrue( + "The container should create a subDir named currentUser: " + user + + "under localDir/usercache", + numOfLocalDirs(nmLocalDir.getAbsolutePath(), + ContainerLocalizer.USERCACHE) > 0); + + Assert.assertTrue("There should be files or Dirs under nm_private when " + + "container is launched", numOfLocalDirs(nmLocalDir.getAbsolutePath(), + ResourceLocalizationService.NM_PRIVATE_DIR) > 0); + + nm.handle(new NodeManagerEvent(NodeManagerEventType.REBOOT)); + + numTries = 0; + while ((numOfLocalDirs(nmLocalDir.getAbsolutePath(), ContainerLocalizer + .USERCACHE) > 0 || numOfLocalDirs(nmLocalDir.getAbsolutePath(), + ContainerLocalizer.FILECACHE) > 0 || numOfLocalDirs(nmLocalDir + .getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) + > 0) && numTries < MAX_TRIES) { + try { + Thread.sleep(500); + } catch (InterruptedException ex) { + // Do nothing + } + numTries++; + } + + Assert.assertTrue("After NM reboots, all local files should be deleted", + numOfLocalDirs(nmLocalDir.getAbsolutePath(), ContainerLocalizer + .USERCACHE) == 0 && numOfLocalDirs(nmLocalDir.getAbsolutePath(), + ContainerLocalizer.FILECACHE) == 0 && numOfLocalDirs(nmLocalDir + .getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) + == 0); + verify(delService, times(1)).delete(eq(user), + argThat(new PathInclude(user))); + verify(delService, times(1)).delete( + (String) isNull(), + argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR + + "_DEL_"))); + verify(delService, times(1)).delete((String) isNull(), + argThat(new PathInclude(ContainerLocalizer.FILECACHE + "_DEL_"))); + verify(delService, times(1)).delete((String) isNull(), + argThat(new PathInclude(ContainerLocalizer.USERCACHE + "_DEL_"))); + + } + + private int numOfLocalDirs(String localDir, String localSubDir) { + File[] listOfFiles = new File(localDir, localSubDir).listFiles(); + if (listOfFiles == null) { + return 0; + } else { + return listOfFiles.length; + } + } + + private void createFiles(String dir, String subDir, int numOfFiles) { + for (int i = 0; i < numOfFiles; i++) { + File newFile = new File(dir + "/" + subDir, "file_" + (i + 1)); + try { + newFile.createNewFile(); + } catch (IOException e) { + // Do nothing + } + } + } + + private ContainerId createContainerId() { + ApplicationId appId = Records.newRecord(ApplicationId.class); + appId.setClusterTimestamp(0); + appId.setId(0); + ApplicationAttemptId appAttemptId = + Records.newRecord(ApplicationAttemptId.class); + appAttemptId.setApplicationId(appId); + appAttemptId.setAttemptId(1); + ContainerId containerId = + Records.newRecord(ContainerId.class); + containerId.setApplicationAttemptId(appAttemptId); + return containerId; + } + + private class MyNodeManager extends NodeManager { + + public MyNodeManager() { + super(); + this.init(createNMConfig()); + } + + @Override + protected NodeStatusUpdater createNodeStatusUpdater(Context context, + Dispatcher dispatcher, NodeHealthCheckerService healthChecker) { + MockNodeStatusUpdater myNodeStatusUpdater = new MockNodeStatusUpdater( + context, dispatcher, healthChecker, metrics); + return myNodeStatusUpdater; + } + + @Override + protected DeletionService createDeletionService(ContainerExecutor exec) { + delService = spy(new DeletionService(exec)); + return delService; + } + + // mimic part of reboot process + @Override + public void handle(NodeManagerEvent event) { + switch (event.getType()) { + case SHUTDOWN: + this.stop(); + break; + case REBOOT: + this.stop(); + this.createNewMyNodeManager().start(); + break; + default: + LOG.warn("Invalid shutdown event " + event.getType() + ". Ignoring."); + } + } + + private MyNodeManager createNewMyNodeManager() { + return new MyNodeManager(); + } + + private YarnConfiguration createNMConfig() { + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB + conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345"); + conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346"); + conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath()); + conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath()); + return conf; + } + } + + class PathInclude extends ArgumentMatcher { + + final String part; + + PathInclude(String part) { + this.part = part; + } + + @Override + public boolean matches(Object o) { + return ((Path) o).getName().indexOf(part) != -1; + } + } +} From 18e08a8f511652ed9f4ba386bb4f0cf8339e2729 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Mon, 25 Mar 2013 18:28:50 +0000 Subject: [PATCH 06/37] YARN-439. Flatten NodeHeartbeatResponse. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460811 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../NodeHeartbeatResponse.java | 24 +- .../impl/pb/NodeHeartbeatResponsePBImpl.java | 247 ++++++++++-- .../server/api/records/HeartbeatResponse.java | 52 --- .../impl/pb/HeartbeatResponsePBImpl.java | 350 ------------------ .../proto/yarn_server_common_protos.proto | 7 - .../yarn_server_common_service_protos.proto | 7 +- .../apache/hadoop/yarn/TestRecordFactory.java | 11 - .../nodemanager/NodeStatusUpdaterImpl.java | 10 +- .../nodemanager/MockNodeStatusUpdater.java | 7 +- .../nodemanager/TestNodeStatusUpdater.java | 26 +- .../ResourceTrackerService.java | 37 +- .../server/resourcemanager/rmnode/RMNode.java | 11 +- .../resourcemanager/rmnode/RMNodeImpl.java | 20 +- .../rmnode/RMNodeStatusEvent.java | 8 +- .../yarn/server/resourcemanager/MockNM.java | 14 +- .../server/resourcemanager/MockNodes.java | 6 +- .../server/resourcemanager/NodeManager.java | 6 +- .../TestApplicationCleanup.java | 22 +- .../TestRMNodeTransitions.java | 16 +- .../server/resourcemanager/TestRMRestart.java | 4 +- .../TestResourceTrackerService.java | 14 +- .../resourcetracker/TestNMExpiry.java | 2 +- .../TestRMNMRPCResponseId.java | 15 +- .../hadoop/yarn/server/MiniYARNCluster.java | 3 +- .../yarn/server/TestRMNMSecretKeys.java | 4 +- 26 files changed, 348 insertions(+), 577 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c3c13eea3ac..490b5e98ef0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -62,6 +62,8 @@ Release 2.0.5-beta - UNRELEASED YARN-396. Rationalize AllocateResponse in RM Scheduler API. (Zhijie Shen via hitesh) + YARN-439. Flatten NodeHeartbeatResponse. (Xuan Gong via sseth) + NEW FEATURES IMPROVEMENTS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java index 665f1f164c0..6aff34ae284 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java @@ -18,10 +18,28 @@ package org.apache.hadoop.yarn.server.api.protocolrecords; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import java.util.List; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.api.records.NodeAction; public interface NodeHeartbeatResponse { - public abstract HeartbeatResponse getHeartbeatResponse(); + int getResponseId(); + NodeAction getNodeAction(); + + List getContainersToCleanup(); + + List getApplicationsToCleanup(); + + void setResponseId(int responseId); + void setNodeAction(NodeAction action); + + MasterKey getMasterKey(); + void setMasterKey(MasterKey secretKey); + + void addAllContainersToCleanup(List containers); - public abstract void setHeartbeatResponse(HeartbeatResponse heartbeatResponse); + void addAllApplicationsToCleanup(List applications); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java index 49352a2afa6..bc51b241f3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java @@ -18,14 +18,25 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; +import java.util.ArrayList; +import java.util.List; +import java.util.Iterator; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProto; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.impl.pb.HeartbeatResponsePBImpl; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.api.records.NodeAction; +import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; @@ -34,8 +45,9 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase containersToCleanup = null; + private List applicationsToCleanup = null; + private MasterKey masterKey = null; public NodeHeartbeatResponsePBImpl() { builder = NodeHeartbeatResponseProto.newBuilder(); @@ -54,8 +66,14 @@ public class NodeHeartbeatResponsePBImpl extends ProtoBase getContainersToCleanup() { + initContainersToCleanup(); + return this.containersToCleanup; + } + + private void initContainersToCleanup() { + if (this.containersToCleanup != null) { + return; + } + NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getContainersToCleanupList(); + this.containersToCleanup = new ArrayList(); + + for (ContainerIdProto c : list) { + this.containersToCleanup.add(convertFromProtoFormat(c)); + } + } + + @Override + public void addAllContainersToCleanup( + final List containersToCleanup) { + if (containersToCleanup == null) + return; + initContainersToCleanup(); + this.containersToCleanup.addAll(containersToCleanup); + } + + private void addContainersToCleanupToProto() { + maybeInitBuilder(); + builder.clearContainersToCleanup(); + if (containersToCleanup == null) + return; + Iterable iterable = new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = containersToCleanup.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ContainerIdProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllContainersToCleanup(iterable); + } + + @Override + public List getApplicationsToCleanup() { + initApplicationsToCleanup(); + return this.applicationsToCleanup; + } + + private void initApplicationsToCleanup() { + if (this.applicationsToCleanup != null) { + return; + } + NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; + List list = p.getApplicationsToCleanupList(); + this.applicationsToCleanup = new ArrayList(); + + for (ApplicationIdProto c : list) { + this.applicationsToCleanup.add(convertFromProtoFormat(c)); + } + } + + @Override + public void addAllApplicationsToCleanup( + final List applicationsToCleanup) { + if (applicationsToCleanup == null) + return; + initApplicationsToCleanup(); + this.applicationsToCleanup.addAll(applicationsToCleanup); + } + + private void addApplicationsToCleanupToProto() { + maybeInitBuilder(); + builder.clearApplicationsToCleanup(); + if (applicationsToCleanup == null) + return; + Iterable iterable = new Iterable() { + + @Override + public Iterator iterator() { + return new Iterator() { + + Iterator iter = applicationsToCleanup.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public ApplicationIdProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + } + }; + builder.addAllApplicationsToCleanup(iterable); + } + + private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { + return new ContainerIdPBImpl(p); + } + + private ContainerIdProto convertToProtoFormat(ContainerId t) { + return ((ContainerIdPBImpl) t).getProto(); + } + + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { + return new ApplicationIdPBImpl(p); + } + + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { + return ((ApplicationIdPBImpl) t).getProto(); + } + + private NodeAction convertFromProtoFormat(NodeActionProto p) { + return NodeAction.valueOf(p.name()); + } + + private NodeActionProto convertToProtoFormat(NodeAction t) { + return NodeActionProto.valueOf(t.name()); + } + + private MasterKeyPBImpl convertFromProtoFormat(MasterKeyProto p) { + return new MasterKeyPBImpl(p); + } + + private MasterKeyProto convertToProtoFormat(MasterKey t) { + return ((MasterKeyPBImpl) t).getProto(); + } +} -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java deleted file mode 100644 index 4536934054f..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.api.records; - -import java.util.List; - -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; - -public interface HeartbeatResponse { - int getResponseId(); - NodeAction getNodeAction(); - - List getContainersToCleanupList(); - ContainerId getContainerToCleanup(int index); - int getContainersToCleanupCount(); - - List getApplicationsToCleanupList(); - ApplicationId getApplicationsToCleanup(int index); - int getApplicationsToCleanupCount(); - - void setResponseId(int responseId); - void setNodeAction(NodeAction action); - - MasterKey getMasterKey(); - void setMasterKey(MasterKey secretKey); - - void addAllContainersToCleanup(List containers); - void addContainerToCleanup(ContainerId container); - void removeContainerToCleanup(int index); - void clearContainersToCleanup(); - - void addAllApplicationsToCleanup(List applications); - void addApplicationToCleanup(ApplicationId applicationId); - void removeApplicationToCleanup(int index); - void clearApplicationsToCleanup(); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java deleted file mode 100644 index 8a7d890aebd..00000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.records.impl.pb; - - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.MasterKey; -import org.apache.hadoop.yarn.server.api.records.NodeAction; - -public class HeartbeatResponsePBImpl extends - ProtoBase implements HeartbeatResponse { - HeartbeatResponseProto proto = HeartbeatResponseProto.getDefaultInstance(); - HeartbeatResponseProto.Builder builder = null; - boolean viaProto = false; - - private List containersToCleanup = null; - private List applicationsToCleanup = null; - private MasterKey masterKey = null; - - public HeartbeatResponsePBImpl() { - builder = HeartbeatResponseProto.newBuilder(); - } - - public HeartbeatResponsePBImpl(HeartbeatResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public HeartbeatResponseProto getProto() { - - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToBuilder() { - if (this.containersToCleanup != null) { - addContainersToCleanupToProto(); - } - if (this.applicationsToCleanup != null) { - addApplicationsToCleanupToProto(); - } - if (this.masterKey != null) { - builder.setMasterKey(convertToProtoFormat(this.masterKey)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = HeartbeatResponseProto.newBuilder(proto); - } - viaProto = false; - } - - - @Override - public int getResponseId() { - HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - return (p.getResponseId()); - } - - @Override - public void setResponseId(int responseId) { - maybeInitBuilder(); - builder.setResponseId((responseId)); - } - - @Override - public MasterKey getMasterKey() { - HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.masterKey != null) { - return this.masterKey; - } - if (!p.hasMasterKey()) { - return null; - } - this.masterKey = convertFromProtoFormat(p.getMasterKey()); - return this.masterKey; - } - - @Override - public void setMasterKey(MasterKey masterKey) { - maybeInitBuilder(); - if (masterKey == null) - builder.clearMasterKey(); - this.masterKey = masterKey; - } - - @Override - public NodeAction getNodeAction() { - HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - if(!p.hasNodeAction()) { - return null; - } - return (convertFromProtoFormat(p.getNodeAction())); - } - - @Override - public void setNodeAction(NodeAction nodeAction) { - maybeInitBuilder(); - if (nodeAction == null) { - builder.clearNodeAction(); - return; - } - builder.setNodeAction(convertToProtoFormat(nodeAction)); - } - - @Override - public List getContainersToCleanupList() { - initContainersToCleanup(); - return this.containersToCleanup; - } - @Override - public ContainerId getContainerToCleanup(int index) { - initContainersToCleanup(); - return this.containersToCleanup.get(index); - } - @Override - public int getContainersToCleanupCount() { - initContainersToCleanup(); - return this.containersToCleanup.size(); - } - - private void initContainersToCleanup() { - if (this.containersToCleanup != null) { - return; - } - HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getContainersToCleanupList(); - this.containersToCleanup = new ArrayList(); - - for (ContainerIdProto c : list) { - this.containersToCleanup.add(convertFromProtoFormat(c)); - } - } - - @Override - public void addAllContainersToCleanup(final List containersToCleanup) { - if (containersToCleanup == null) - return; - initContainersToCleanup(); - this.containersToCleanup.addAll(containersToCleanup); - } - - private void addContainersToCleanupToProto() { - maybeInitBuilder(); - builder.clearContainersToCleanup(); - if (containersToCleanup == null) - return; - Iterable iterable = new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - - Iterator iter = containersToCleanup.iterator(); - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public ContainerIdProto next() { - return convertToProtoFormat(iter.next()); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - - } - }; - - } - }; - builder.addAllContainersToCleanup(iterable); - } - @Override - public void addContainerToCleanup(ContainerId containersToCleanup) { - initContainersToCleanup(); - this.containersToCleanup.add(containersToCleanup); - } - @Override - public void removeContainerToCleanup(int index) { - initContainersToCleanup(); - this.containersToCleanup.remove(index); - } - @Override - public void clearContainersToCleanup() { - initContainersToCleanup(); - this.containersToCleanup.clear(); - } - @Override - public List getApplicationsToCleanupList() { - initApplicationsToCleanup(); - return this.applicationsToCleanup; - } - @Override - public ApplicationId getApplicationsToCleanup(int index) { - initApplicationsToCleanup(); - return this.applicationsToCleanup.get(index); - } - @Override - public int getApplicationsToCleanupCount() { - initApplicationsToCleanup(); - return this.applicationsToCleanup.size(); - } - - private void initApplicationsToCleanup() { - if (this.applicationsToCleanup != null) { - return; - } - HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder; - List list = p.getApplicationsToCleanupList(); - this.applicationsToCleanup = new ArrayList(); - - for (ApplicationIdProto c : list) { - this.applicationsToCleanup.add(convertFromProtoFormat(c)); - } - } - - @Override - public void addAllApplicationsToCleanup(final List applicationsToCleanup) { - if (applicationsToCleanup == null) - return; - initApplicationsToCleanup(); - this.applicationsToCleanup.addAll(applicationsToCleanup); - } - - private void addApplicationsToCleanupToProto() { - maybeInitBuilder(); - builder.clearApplicationsToCleanup(); - if (applicationsToCleanup == null) - return; - Iterable iterable = new Iterable() { - @Override - public Iterator iterator() { - return new Iterator() { - - Iterator iter = applicationsToCleanup.iterator(); - - @Override - public boolean hasNext() { - return iter.hasNext(); - } - - @Override - public ApplicationIdProto next() { - return convertToProtoFormat(iter.next()); - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - - } - }; - - } - }; - builder.addAllApplicationsToCleanup(iterable); - } - @Override - public void addApplicationToCleanup(ApplicationId applicationsToCleanup) { - initApplicationsToCleanup(); - this.applicationsToCleanup.add(applicationsToCleanup); - } - @Override - public void removeApplicationToCleanup(int index) { - initApplicationsToCleanup(); - this.applicationsToCleanup.remove(index); - } - @Override - public void clearApplicationsToCleanup() { - initApplicationsToCleanup(); - this.applicationsToCleanup.clear(); - } - - private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) { - return new ContainerIdPBImpl(p); - } - - private ContainerIdProto convertToProtoFormat(ContainerId t) { - return ((ContainerIdPBImpl)t).getProto(); - } - - private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { - return new ApplicationIdPBImpl(p); - } - - private ApplicationIdProto convertToProtoFormat(ApplicationId t) { - return ((ApplicationIdPBImpl)t).getProto(); - } - - private NodeAction convertFromProtoFormat(NodeActionProto p) { - return NodeAction.valueOf(p.name()); - } - - private NodeActionProto convertToProtoFormat(NodeAction t) { - return NodeActionProto.valueOf(t.name()); - } - - private MasterKeyPBImpl convertFromProtoFormat(MasterKeyProto p) { - return new MasterKeyPBImpl(p); - } - - private MasterKeyProto convertToProtoFormat(MasterKey t) { - return ((MasterKeyPBImpl)t).getProto(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto index 71f5b1bc88f..4d1ce15fca8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto @@ -47,11 +47,4 @@ message RegistrationResponseProto { optional NodeActionProto nodeAction = 2; } -message HeartbeatResponseProto { - optional int32 response_id = 1; - optional MasterKeyProto master_key = 2; - optional NodeActionProto nodeAction = 3; - repeated ContainerIdProto containers_to_cleanup = 4; - repeated ApplicationIdProto applications_to_cleanup = 5; -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index e4d82c75d61..30d0cfeff33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -38,6 +38,11 @@ message NodeHeartbeatRequestProto { optional MasterKeyProto last_known_master_key = 2; } + message NodeHeartbeatResponseProto { - optional HeartbeatResponseProto heartbeat_response = 1; + optional int32 response_id = 1; + optional MasterKeyProto master_key = 2; + optional NodeActionProto nodeAction = 3; + repeated ContainerIdProto containers_to_cleanup = 4; + repeated ApplicationIdProto applications_to_cleanup = 5; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java index c77d21071ec..b833e6104bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRecordFactory.java @@ -25,8 +25,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeHeartbeatRequestPBImpl; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; -import org.apache.hadoop.yarn.server.api.records.impl.pb.HeartbeatResponsePBImpl; import org.junit.Test; public class TestRecordFactory { @@ -34,15 +32,6 @@ public class TestRecordFactory { @Test public void testPbRecordFactory() { RecordFactory pbRecordFactory = RecordFactoryPBImpl.get(); - - try { - HeartbeatResponse response = pbRecordFactory.newRecordInstance(HeartbeatResponse.class); - Assert.assertEquals(HeartbeatResponsePBImpl.class, response.getClass()); - } catch (YarnException e) { - e.printStackTrace(); - Assert.fail("Failed to crete record"); - } - try { NodeHeartbeatRequest request = pbRecordFactory.newRecordInstance(NodeHeartbeatRequest.class); Assert.assertEquals(NodeHeartbeatRequestPBImpl.class, request.getClass()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index e3dae41603a..3df4311b600 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -50,8 +50,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.api.ResourceTracker; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -408,8 +408,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements request.setLastKnownMasterKey(NodeStatusUpdaterImpl.this.context .getContainerTokenSecretManager().getCurrentKey()); } - HeartbeatResponse response = - resourceTracker.nodeHeartbeat(request).getHeartbeatResponse(); + NodeHeartbeatResponse response = + resourceTracker.nodeHeartbeat(request); // See if the master-key has rolled over if (isSecurityEnabled()) { @@ -439,14 +439,14 @@ public class NodeStatusUpdaterImpl extends AbstractService implements lastHeartBeatID = response.getResponseId(); List containersToCleanup = response - .getContainersToCleanupList(); + .getContainersToCleanup(); if (containersToCleanup.size() != 0) { dispatcher.getEventHandler().handle( new CMgrCompletedContainersEvent(containersToCleanup, CMgrCompletedContainersEvent.Reason.BY_RESOURCEMANAGER)); } List appsToCleanup = - response.getApplicationsToCleanupList(); + response.getApplicationsToCleanup(); //Only start tracking for keepAlive on FINISH_APP trackAppsForKeepAlive(appsToCleanup); if (appsToCleanup.size() != 0) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java index e2ea8212d88..bf46cde02f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java @@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; @@ -79,13 +78,9 @@ public class MockNodeStatusUpdater extends NodeStatusUpdaterImpl { LOG.info("Got heartbeat number " + heartBeatID); nodeStatus.setResponseId(heartBeatID++); - HeartbeatResponse response = recordFactory - .newRecordInstance(HeartbeatResponse.class); - response.setResponseId(heartBeatID); - NodeHeartbeatResponse nhResponse = recordFactory .newRecordInstance(NodeHeartbeatResponse.class); - nhResponse.setHeartbeatResponse(response); + nhResponse.setResponseId(heartBeatID); return nhResponse; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index ff9e0824efc..afeb0302422 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -56,7 +56,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; @@ -218,13 +217,10 @@ public class TestNodeStatusUpdater { this.context.getContainers(); Assert.assertEquals(2, activeContainers.size()); } - HeartbeatResponse response = recordFactory - .newRecordInstance(HeartbeatResponse.class); - response.setResponseId(heartBeatID); NodeHeartbeatResponse nhResponse = recordFactory .newRecordInstance(NodeHeartbeatResponse.class); - nhResponse.setHeartbeatResponse(response); + nhResponse.setResponseId(heartBeatID); return nhResponse; } } @@ -335,14 +331,11 @@ public class TestNodeStatusUpdater { throws YarnRemoteException { NodeStatus nodeStatus = request.getNodeStatus(); nodeStatus.setResponseId(heartBeatID++); - HeartbeatResponse response = recordFactory - .newRecordInstance(HeartbeatResponse.class); - response.setResponseId(heartBeatID); - response.setNodeAction(heartBeatNodeAction); NodeHeartbeatResponse nhResponse = recordFactory .newRecordInstance(NodeHeartbeatResponse.class); - nhResponse.setHeartbeatResponse(response); + nhResponse.setResponseId(heartBeatID); + nhResponse.setNodeAction(heartBeatNodeAction); return nhResponse; } } @@ -378,10 +371,10 @@ public class TestNodeStatusUpdater { LOG.info("Got heartBeatId: [" + heartBeatID +"]"); NodeStatus nodeStatus = request.getNodeStatus(); nodeStatus.setResponseId(heartBeatID++); - HeartbeatResponse response = - recordFactory.newRecordInstance(HeartbeatResponse.class); - response.setResponseId(heartBeatID); - response.setNodeAction(heartBeatNodeAction); + NodeHeartbeatResponse nhResponse = + recordFactory.newRecordInstance(NodeHeartbeatResponse.class); + nhResponse.setResponseId(heartBeatID); + nhResponse.setNodeAction(heartBeatNodeAction); if (nodeStatus.getKeepAliveApplications() != null && nodeStatus.getKeepAliveApplications().size() > 0) { @@ -397,11 +390,8 @@ public class TestNodeStatusUpdater { if (heartBeatID == 2) { LOG.info("Sending FINISH_APP for application: [" + appId + "]"); this.context.getApplications().put(appId, mock(Application.class)); - response.addAllApplicationsToCleanup(Collections.singletonList(appId)); + nhResponse.addAllApplicationsToCleanup(Collections.singletonList(appId)); } - NodeHeartbeatResponse nhResponse = - recordFactory.newRecordInstance(NodeHeartbeatResponse.class); - nhResponse.setHeartbeatResponse(response); return nhResponse; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 7aaa0668452..2ca1360eb9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -78,15 +77,9 @@ public class ResourceTrackerService extends AbstractService implements .newRecordInstance(NodeHeartbeatResponse.class); static { - HeartbeatResponse rebootResp = recordFactory - .newRecordInstance(HeartbeatResponse.class); - rebootResp.setNodeAction(NodeAction.REBOOT); - reboot.setHeartbeatResponse(rebootResp); - - HeartbeatResponse decommissionedResp = recordFactory - .newRecordInstance(HeartbeatResponse.class); - decommissionedResp.setNodeAction(NodeAction.SHUTDOWN); - shutDown.setHeartbeatResponse(decommissionedResp); + reboot.setNodeAction(NodeAction.REBOOT); + + shutDown.setNodeAction(NodeAction.SHUTDOWN); } public ResourceTrackerService(RMContext rmContext, @@ -240,17 +233,16 @@ public class ResourceTrackerService extends AbstractService implements .newRecordInstance(NodeHeartbeatResponse.class); // 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat - HeartbeatResponse lastHeartbeatResponse = rmNode.getLastHeartBeatResponse(); - if (remoteNodeStatus.getResponseId() + 1 == lastHeartbeatResponse + NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.getLastNodeHeartBeatResponse(); + if (remoteNodeStatus.getResponseId() + 1 == lastNodeHeartbeatResponse .getResponseId()) { LOG.info("Received duplicate heartbeat from node " + rmNode.getNodeAddress()); - nodeHeartBeatResponse.setHeartbeatResponse(lastHeartbeatResponse); - return nodeHeartBeatResponse; - } else if (remoteNodeStatus.getResponseId() + 1 < lastHeartbeatResponse + return lastNodeHeartbeatResponse; + } else if (remoteNodeStatus.getResponseId() + 1 < lastNodeHeartbeatResponse .getResponseId()) { LOG.info("Too far behind rm response id:" - + lastHeartbeatResponse.getResponseId() + " nm response id:" + + lastNodeHeartbeatResponse.getResponseId() + " nm response id:" + remoteNodeStatus.getResponseId()); // TODO: Just sending reboot is not enough. Think more. this.rmContext.getDispatcher().getEventHandler().handle( @@ -259,11 +251,9 @@ public class ResourceTrackerService extends AbstractService implements } // Heartbeat response - HeartbeatResponse latestResponse = recordFactory - .newRecordInstance(HeartbeatResponse.class); - latestResponse.setResponseId(lastHeartbeatResponse.getResponseId() + 1); - rmNode.updateHeartbeatResponseForCleanup(latestResponse); - latestResponse.setNodeAction(NodeAction.NORMAL); + nodeHeartBeatResponse.setResponseId(lastNodeHeartbeatResponse.getResponseId() + 1); + rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse); + nodeHeartBeatResponse.setNodeAction(NodeAction.NORMAL); // Check if node's masterKey needs to be updated and if the currentKey has // roller over, send it across @@ -282,7 +272,7 @@ public class ResourceTrackerService extends AbstractService implements } } if (shouldSendMasterKey) { - latestResponse.setMasterKey(nextMasterKeyForNode); + nodeHeartBeatResponse.setMasterKey(nextMasterKeyForNode); } } @@ -290,9 +280,8 @@ public class ResourceTrackerService extends AbstractService implements this.rmContext.getDispatcher().getEventHandler().handle( new RMNodeStatusEvent(nodeId, remoteNodeStatus.getNodeHealthStatus(), remoteNodeStatus.getContainersStatuses(), - remoteNodeStatus.getKeepAliveApplications(), latestResponse)); + remoteNodeStatus.getKeepAliveApplications(), nodeHeartBeatResponse)); - nodeHeartBeatResponse.setHeartbeatResponse(latestResponse); return nodeHeartBeatResponse; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index dc7295af085..9d1103d4fc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -27,7 +27,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; /** * Node managers information on available resources @@ -106,13 +106,13 @@ public interface RMNode { public List getAppsToCleanup(); /** - * Update a {@link HeartbeatResponse} with the list of containers and + * Update a {@link NodeHeartbeatResponse} with the list of containers and * applications to clean up for this node. - * @param response the {@link HeartbeatResponse} to update + * @param response the {@link NodeHeartbeatResponse} to update */ - public void updateHeartbeatResponseForCleanup(HeartbeatResponse response); + public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response); - public HeartbeatResponse getLastHeartBeatResponse(); + public NodeHeartbeatResponse getLastNodeHeartBeatResponse(); /** * Get and clear the list of containerUpdates accumulated across NM @@ -121,5 +121,4 @@ public interface RMNode { * @return containerUpdates accumulated across NM heartbeats. */ public List pullContainerUpdates(); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 23a45991390..255b4951396 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType; @@ -107,8 +107,8 @@ public class RMNodeImpl implements RMNode, EventHandler { /* the list of applications that have finished and need to be purged */ private final List finishedApplications = new ArrayList(); - private HeartbeatResponse latestHeartBeatResponse = recordFactory - .newRecordInstance(HeartbeatResponse.class); + private NodeHeartbeatResponse latestNodeHeartBeatResponse = recordFactory + .newRecordInstance(NodeHeartbeatResponse.class); private static final StateMachineFactory { this.nodeHealthStatus.setHealthReport("Healthy"); this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis()); - this.latestHeartBeatResponse.setResponseId(0); + this.latestNodeHeartBeatResponse.setResponseId(0); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -304,7 +304,7 @@ public class RMNodeImpl implements RMNode, EventHandler { }; @Override - public void updateHeartbeatResponseForCleanup(HeartbeatResponse response) { + public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response) { this.writeLock.lock(); try { @@ -319,12 +319,12 @@ public class RMNodeImpl implements RMNode, EventHandler { }; @Override - public HeartbeatResponse getLastHeartBeatResponse() { + public NodeHeartbeatResponse getLastNodeHeartBeatResponse() { this.readLock.lock(); try { - return this.latestHeartBeatResponse; + return this.latestNodeHeartBeatResponse; } finally { this.readLock.unlock(); } @@ -430,7 +430,7 @@ public class RMNodeImpl implements RMNode, EventHandler { if (rmNode.getTotalCapability().equals(newNode.getTotalCapability()) && rmNode.getHttpPort() == newNode.getHttpPort()) { // Reset heartbeat ID since node just restarted. - rmNode.getLastHeartBeatResponse().setResponseId(0); + rmNode.getLastNodeHeartBeatResponse().setResponseId(0); rmNode.context.getDispatcher().getEventHandler().handle( new NodeAddedSchedulerEvent(rmNode)); } else { @@ -507,7 +507,7 @@ public class RMNodeImpl implements RMNode, EventHandler { RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event; // Switch the last heartbeatresponse. - rmNode.latestHeartBeatResponse = statusEvent.getLatestResponse(); + rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse(); NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus(); @@ -591,7 +591,7 @@ public class RMNodeImpl implements RMNode, EventHandler { RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event; // Switch the last heartbeatresponse. - rmNode.latestHeartBeatResponse = statusEvent.getLatestResponse(); + rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse(); NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus(); rmNode.setNodeHealthStatus(remoteNodeHealthStatus); if (remoteNodeHealthStatus.getIsNodeHealthy()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java index 1285c2bed99..3f356817658 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java @@ -24,18 +24,18 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; public class RMNodeStatusEvent extends RMNodeEvent { private final NodeHealthStatus nodeHealthStatus; private final List containersCollection; - private final HeartbeatResponse latestResponse; + private final NodeHeartbeatResponse latestResponse; private final List keepAliveAppIds; public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus, List collection, List keepAliveAppIds, - HeartbeatResponse latestResponse) { + NodeHeartbeatResponse latestResponse) { super(nodeId, RMNodeEventType.STATUS_UPDATE); this.nodeHealthStatus = nodeHealthStatus; this.containersCollection = collection; @@ -51,7 +51,7 @@ public class RMNodeStatusEvent extends RMNodeEvent { return this.containersCollection; } - public HeartbeatResponse getLatestResponse() { + public NodeHeartbeatResponse getLatestResponse() { return this.latestResponse; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index 8f66bdba485..e22aa6ca7e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -33,8 +33,8 @@ import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; @@ -93,12 +93,12 @@ public class MockNM { return registrationResponse; } - public HeartbeatResponse nodeHeartbeat(boolean isHealthy) throws Exception { + public NodeHeartbeatResponse nodeHeartbeat(boolean isHealthy) throws Exception { return nodeHeartbeat(new HashMap>(), isHealthy, ++responseId); } - public HeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId, + public NodeHeartbeatResponse nodeHeartbeat(ApplicationAttemptId attemptId, int containerId, ContainerState containerState) throws Exception { HashMap> nodeUpdate = new HashMap>(1); @@ -112,12 +112,12 @@ public class MockNM { return nodeHeartbeat(nodeUpdate, true); } - public HeartbeatResponse nodeHeartbeat(Map> conts, boolean isHealthy) throws Exception { return nodeHeartbeat(conts, isHealthy, ++responseId); } - public HeartbeatResponse nodeHeartbeat(Map> conts, boolean isHealthy, int resId) throws Exception { NodeHeartbeatRequest req = Records.newRecord(NodeHeartbeatRequest.class); NodeStatus status = Records.newRecord(NodeStatus.class); @@ -133,8 +133,8 @@ public class MockNM { status.setNodeHealthStatus(healthStatus); req.setNodeStatus(status); req.setLastKnownMasterKey(this.currentMasterKey); - HeartbeatResponse heartbeatResponse = - resourceTracker.nodeHeartbeat(req).getHeartbeatResponse(); + NodeHeartbeatResponse heartbeatResponse = + resourceTracker.nodeHeartbeat(req); MasterKey masterKeyFromRM = heartbeatResponse.getMasterKey(); this.currentMasterKey = (masterKeyFromRM != null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 2c0fb6cf1d3..c3fe72d99cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; @@ -187,11 +187,11 @@ public class MockNodes { } @Override - public void updateHeartbeatResponseForCleanup(HeartbeatResponse response) { + public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response) { } @Override - public HeartbeatResponse getLastHeartBeatResponse() { + public NodeHeartbeatResponse getLastNodeHeartBeatResponse() { return null; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index e7ad0314e8f..610ec0b3f7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -51,8 +51,8 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; @@ -151,8 +151,8 @@ public class NodeManager implements ContainerManager { NodeHeartbeatRequest request = recordFactory .newRecordInstance(NodeHeartbeatRequest.class); request.setNodeStatus(nodeStatus); - HeartbeatResponse response = resourceTrackerService - .nodeHeartbeat(request).getHeartbeatResponse(); + NodeHeartbeatResponse response = resourceTrackerService + .nodeHeartbeat(request); responseID = response.getResponseId(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java index ca640b39e59..6705171de3c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -92,15 +92,15 @@ public class TestApplicationCleanup { Assert.assertEquals(request, contReceived); am.unregisterAppAttempt(); - HeartbeatResponse resp = nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, + NodeHeartbeatResponse resp = nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE); am.waitForState(RMAppAttemptState.FINISHED); //currently only containers are cleaned via this //AM container is cleaned via container launcher resp = nm1.nodeHeartbeat(true); - List contsToClean = resp.getContainersToCleanupList(); - List apps = resp.getApplicationsToCleanupList(); + List contsToClean = resp.getContainersToCleanup(); + List apps = resp.getApplicationsToCleanup(); int cleanedConts = contsToClean.size(); int cleanedApps = apps.size(); waitCount = 0; @@ -109,8 +109,8 @@ public class TestApplicationCleanup { + cleanedConts + " cleanedApps: " + cleanedApps); Thread.sleep(100); resp = nm1.nodeHeartbeat(true); - contsToClean = resp.getContainersToCleanupList(); - apps = resp.getApplicationsToCleanupList(); + contsToClean = resp.getContainersToCleanup(); + apps = resp.getApplicationsToCleanup(); cleanedConts += contsToClean.size(); cleanedApps += apps.size(); } @@ -198,9 +198,9 @@ public class TestApplicationCleanup { .getId(), ContainerState.RUNNING, "nothing", 0)); containerStatuses.put(app.getApplicationId(), containerStatusList); - HeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true); + NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true); dispatcher.await(); - List contsToClean = resp.getContainersToCleanupList(); + List contsToClean = resp.getContainersToCleanup(); int cleanedConts = contsToClean.size(); waitCount = 0; while (cleanedConts < 1 && waitCount++ < 200) { @@ -208,7 +208,7 @@ public class TestApplicationCleanup { Thread.sleep(100); resp = nm1.nodeHeartbeat(true); dispatcher.await(); - contsToClean = resp.getContainersToCleanupList(); + contsToClean = resp.getContainersToCleanup(); cleanedConts += contsToClean.size(); } LOG.info("Got cleanup for " + contsToClean.get(0)); @@ -226,7 +226,7 @@ public class TestApplicationCleanup { resp = nm1.nodeHeartbeat(containerStatuses, true); dispatcher.await(); - contsToClean = resp.getContainersToCleanupList(); + contsToClean = resp.getContainersToCleanup(); cleanedConts = contsToClean.size(); // The cleanup list won't be instantaneous as it is given out by scheduler // and not RMNodeImpl. @@ -236,7 +236,7 @@ public class TestApplicationCleanup { Thread.sleep(100); resp = nm1.nodeHeartbeat(true); dispatcher.await(); - contsToClean = resp.getContainersToCleanupList(); + contsToClean = resp.getContainersToCleanup(); cleanedConts += contsToClean.size(); } LOG.info("Got cleanup for " + contsToClean.get(0)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index e165e417cd9..cd5d2a23cbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; @@ -118,7 +118,7 @@ public class TestRMNodeTransitions { } private RMNodeStatusEvent getMockRMNodeStatusEvent() { - HeartbeatResponse response = mock(HeartbeatResponse.class); + NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class); NodeHealthStatus healthStatus = mock(NodeHealthStatus.class); Boolean yes = new Boolean(true); @@ -325,14 +325,14 @@ public class TestRMNodeTransitions { node.handle(statusEvent); Assert.assertEquals(1, node.getContainersToCleanUp().size()); Assert.assertEquals(1, node.getAppsToCleanup().size()); - HeartbeatResponse hbrsp = Records.newRecord(HeartbeatResponse.class); - node.updateHeartbeatResponseForCleanup(hbrsp); + NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class); + node.updateNodeHeartbeatResponseForCleanup(hbrsp); Assert.assertEquals(0, node.getContainersToCleanUp().size()); Assert.assertEquals(0, node.getAppsToCleanup().size()); - Assert.assertEquals(1, hbrsp.getContainersToCleanupCount()); - Assert.assertEquals(completedContainerId, hbrsp.getContainerToCleanup(0)); - Assert.assertEquals(1, hbrsp.getApplicationsToCleanupCount()); - Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup(0)); + Assert.assertEquals(1, hbrsp.getContainersToCleanup().size()); + Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0)); + Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size()); + Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0)); } private RMNodeImpl getRunningNode() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 63868b0c490..2057d8ab419 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -31,7 +31,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; @@ -219,7 +219,7 @@ public class TestRMRestart { Assert.assertTrue(allocResponse.getReboot()); // NM should be rebooted on heartbeat, even first heartbeat for nm2 - HeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse hbResponse = nm1.nodeHeartbeat(true); Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction()); hbResponse = nm2.nodeHeartbeat(true); Assert.assertEquals(NodeAction.REBOOT, hbResponse.getNodeAction()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 7e5b5da9356..cf679137f35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -36,9 +36,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; @@ -75,7 +75,7 @@ public class TestResourceTrackerService { assert(metrics != null); int metricCount = metrics.getNumDecommisionedNMs(); - HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); nodeHeartbeat = nm2.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); @@ -124,7 +124,7 @@ public class TestResourceTrackerService { int metricCount = ClusterMetrics.getMetrics().getNumDecommisionedNMs(); - HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); nodeHeartbeat = nm2.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); @@ -161,7 +161,7 @@ public class TestResourceTrackerService { ClusterMetrics metrics = ClusterMetrics.getMetrics(); assert(metrics != null); int initialMetricCount = metrics.getNumDecommisionedNMs(); - HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertEquals( NodeAction.NORMAL, nodeHeartbeat.getNodeAction()); @@ -198,7 +198,7 @@ public class TestResourceTrackerService { ClusterMetrics metrics = ClusterMetrics.getMetrics(); assert(metrics != null); int initialMetricCount = metrics.getNumDecommisionedNMs(); - HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertEquals( NodeAction.NORMAL, nodeHeartbeat.getNodeAction()); @@ -254,7 +254,7 @@ public class TestResourceTrackerService { MockNM nm2 = rm.registerNode("host2:1234", 2048); int initialMetricCount = ClusterMetrics.getMetrics().getNumRebootedNMs(); - HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction())); nodeHeartbeat = nm2.nodeHeartbeat( @@ -351,7 +351,7 @@ public class TestResourceTrackerService { // reconnect of healthy node nm1 = rm.registerNode("host1:1234", 5120); - HeartbeatResponse response = nm1.nodeHeartbeat(true); + NodeHeartbeatResponse response = nm1.nodeHeartbeat(true); Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction())); dispatcher.await(); Assert.assertEquals(expectedNMs, ClusterMetrics.getMetrics().getNumActiveNMs()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index 6ec3f5403b9..8ca0b43c35f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -109,7 +109,7 @@ public class TestNMExpiry { .newRecordInstance(NodeHeartbeatRequest.class); request.setNodeStatus(nodeStatus); lastResponseID = resourceTrackerService.nodeHeartbeat(request) - .getHeartbeatResponse().getResponseId(); + .getResponseId(); Thread.sleep(1000); } catch(Exception e) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index 7d7f99d054e..984d7cdfcf5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.resourcemanager.NMLivelinessMonitor; import org.apache.hadoop.yarn.server.resourcemanager.NodesListManager; @@ -116,23 +116,20 @@ public class TestRMNMRPCResponseId { nodeHeartBeatRequest.setNodeStatus(nodeStatus); nodeStatus.setResponseId(0); - HeartbeatResponse response = resourceTrackerService.nodeHeartbeat( - nodeHeartBeatRequest).getHeartbeatResponse(); + NodeHeartbeatResponse response = resourceTrackerService.nodeHeartbeat( + nodeHeartBeatRequest); Assert.assertTrue(response.getResponseId() == 1); nodeStatus.setResponseId(response.getResponseId()); - response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest) - .getHeartbeatResponse(); + response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest); Assert.assertTrue(response.getResponseId() == 2); /* try calling with less response id */ - response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest) - .getHeartbeatResponse(); + response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest); Assert.assertTrue(response.getResponseId() == 2); nodeStatus.setResponseId(0); - response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest) - .getHeartbeatResponse(); + response = resourceTrackerService.nodeHeartbeat(nodeHeartBeatRequest); Assert.assertTrue(NodeAction.REBOOT.equals(response.getNodeAction())); } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 452508ff4af..68a00362d3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -361,8 +361,7 @@ public class MiniYARNCluster extends CompositeService { NodeHeartbeatResponse response = recordFactory.newRecordInstance( NodeHeartbeatResponse.class); try { - response.setHeartbeatResponse(rt.nodeHeartbeat(request) - .getHeartbeatResponse()); + response = rt.nodeHeartbeat(request); } catch (IOException ioe) { LOG.info("Exception in heartbeat from node " + request.getNodeStatus().getNodeId(), ioe); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java index 7bd1ff2032d..5bc2f7ad0c9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java @@ -27,7 +27,7 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; -import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -67,7 +67,7 @@ public class TestRMNMSecretKeys { Assert.assertNotNull("Registration should cause a key-update!", masterKey); dispatcher.await(); - HeartbeatResponse response = nm.nodeHeartbeat(true); + NodeHeartbeatResponse response = nm.nodeHeartbeat(true); Assert.assertNull( "First heartbeat after registration shouldn't get any key updates!", response.getMasterKey()); From 1acb805058dad506543b8fcc50c5ec65ff99bd0b Mon Sep 17 00:00:00 2001 From: Bikas Saha Date: Mon, 25 Mar 2013 18:57:15 +0000 Subject: [PATCH 07/37] YARN-497. Yarn unmanaged-am launcher jar does not define a main class in its manifest (Hitesh Shah via bikas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460846 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../pom.xml | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 490b5e98ef0..a6ae28573b9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -93,6 +93,9 @@ Release 2.0.5-beta - UNRELEASED YARN-417. Create AMRMClient wrapper that provides asynchronous callbacks. (Sandy Ryza via bikas) + YARN-497. Yarn unmanaged-am launcher jar does not define a main class in + its manifest (Hitesh Shah via bikas) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml index 1694d9775a4..a43badec533 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml @@ -86,6 +86,16 @@ + + maven-jar-plugin + + + + org.apache.hadoop.yarn.applications.unmanagedamlauncher.UnmanagedAMLauncher + + + + org.apache.maven.plugins maven-surefire-plugin From 07573640147bb4cf3568078acae7b245e9299cf7 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 25 Mar 2013 21:11:33 +0000 Subject: [PATCH 08/37] YARN-378. Fix RM to make the AM max attempts/retries to be configurable per application by clients. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460895 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../hadoop/yarn/api/ApplicationConstants.java | 6 ++ .../records/ApplicationSubmissionContext.java | 18 ++++++ .../ApplicationSubmissionContextPBImpl.java | 12 ++++ .../src/main/proto/yarn_protos.proto | 1 + .../hadoop/yarn/conf/YarnConfiguration.java | 11 ++-- .../src/main/resources/yarn-default.xml | 8 ++- .../resourcemanager/ResourceManager.java | 11 ++++ .../amlauncher/AMLauncher.java | 3 + .../server/resourcemanager/rmapp/RMApp.java | 6 ++ .../resourcemanager/rmapp/RMAppImpl.java | 26 +++++++-- .../rmapp/attempt/RMAppAttempt.java | 2 +- .../resourcemanager/TestAppManager.java | 56 ++++++++++++++++++- .../TestApplicationMasterLauncher.java | 7 ++- .../resourcemanager/TestClientRMService.java | 4 +- .../resourcemanager/TestResourceManager.java | 14 +++++ .../applicationsmanager/MockAsm.java | 10 ++++ .../resourcemanager/rmapp/MockRMApp.java | 10 ++++ .../rmapp/TestRMAppTransitions.java | 14 ++--- .../webapp/TestRMWebServicesApps.java | 10 ++-- 20 files changed, 205 insertions(+), 27 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a6ae28573b9..5a6dda9aad3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -133,6 +133,9 @@ Release 2.0.5-beta - UNRELEASED YARN-71. Fix the NodeManager to clean up local-dirs on restart. (Xuan Gong via sseth) + YARN-378. Fix RM to make the AM max attempts/retries to be configurable + per application by clients. (Zhijie Shen via vinodkv) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 4934b0c8013..2fafffeb7ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -92,6 +92,12 @@ public interface ApplicationConstants { public static final String STDOUT = "stdout"; + /** + * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment + * only + */ + public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS"; + /** * Environment for Applications. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java index a9e3d5477a4..f057012777f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java @@ -189,4 +189,22 @@ public interface ApplicationSubmissionContext { @LimitedPrivate("mapreduce") @Unstable public void setCancelTokensWhenComplete(boolean cancel); + + /** + * @return the number of max attempts of the application to be submitted + */ + @Public + @Unstable + public int getMaxAppAttempts(); + + /** + * Set the number of max attempts of the application to be submitted. WARNING: + * it should be no larger than the global number of max attempts in the Yarn + * configuration. + * @param maxAppAttempts the number of max attempts of the application + * to be submitted. + */ + @Public + @Unstable + public void setMaxAppAttempts(int maxAppAttempts); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java index ef566834739..a6a890cc433 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java @@ -232,6 +232,18 @@ implements ApplicationSubmissionContext { builder.setCancelTokensWhenComplete(cancel); } + @Override + public int getMaxAppAttempts() { + ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder; + return p.getMaxAppAttempts(); + } + + @Override + public void setMaxAppAttempts(int maxAppAttempts) { + maybeInitBuilder(); + builder.setMaxAppAttempts(maxAppAttempts); + } + private PriorityPBImpl convertFromProtoFormat(PriorityProto p) { return new PriorityPBImpl(p); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 66fcca06039..4d0ca0f4d79 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -219,6 +219,7 @@ message ApplicationSubmissionContextProto { optional ContainerLaunchContextProto am_container_spec = 6; optional bool cancel_tokens_when_complete = 7 [default = true]; optional bool unmanaged_am = 8 [default = false]; + optional int32 maxAppAttempts = 9 [default = 0]; } enum ApplicationAccessTypeProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a95397a8f7d..6b8353a33d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -180,10 +180,13 @@ public class YarnConfiguration extends Configuration { RM_PREFIX + "admin.client.thread-count"; public static final int DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT = 1; - /** The maximum number of application master retries.*/ - public static final String RM_AM_MAX_RETRIES = - RM_PREFIX + "am.max-retries"; - public static final int DEFAULT_RM_AM_MAX_RETRIES = 1; + /** + * The maximum number of application attempts. + * It's a global setting for all application masters. + */ + public static final String RM_AM_MAX_ATTEMPTS = + RM_PREFIX + "am.max-attempts"; + public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 1; /** The keytab for the resource manager.*/ public static final String RM_KEYTAB = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index cfe97ca4eb1..8d391e2631d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -135,8 +135,12 @@ - The maximum number of application master retries. - yarn.resourcemanager.am.max-retries + The maximum number of application attempts. It's a global + setting for all application masters. Each application master can specify + its individual maximum number of application attempts via the API, but the + individual number cannot be more than the global upper bound. If it is, + the resourcemanager will override it. + yarn.resourcemanager.am.max-attempts 1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index aa96944fc6b..a16aa357758 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -139,6 +139,8 @@ public class ResourceManager extends CompositeService implements Recoverable { @Override public synchronized void init(Configuration conf) { + validateConfigs(conf); + this.conf = conf; this.conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); @@ -325,6 +327,15 @@ public class ResourceManager extends CompositeService implements Recoverable { this.applicationACLsManager, this.conf); } + protected static void validateConfigs(Configuration conf) { + int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + if (globalMaxAppAttempts <= 0) { + throw new YarnException( + "The global max attempts should be a positive integer."); + } + } + @Private public static class SchedulerEventDispatcher extends AbstractService implements EventHandler { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index de682f64f9d..e45e1dd8f81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -197,6 +197,9 @@ public class AMLauncher implements Runnable { String.valueOf(rmContext.getRMApps() .get(applicationId) .getSubmitTime())); + environment.put(ApplicationConstants.MAX_APP_ATTEMPTS_ENV, + String.valueOf(rmContext.getRMApps().get( + applicationId).getMaxAppAttempts())); if (UserGroupInformation.isSecurityEnabled()) { // TODO: Security enabled/disabled info should come from RM. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java index 92bc2b600f9..8629ae2f9dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java @@ -173,4 +173,10 @@ public interface RMApp extends EventHandler { * {@link FinishApplicationMasterRequest#setFinishApplicationStatus(FinalApplicationStatus)}. */ FinalApplicationStatus getFinalApplicationStatus(); + + /** + * The number of max attempts of the application. + * @return the number of max attempts of the application. + */ + int getMaxAppAttempts(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index fa8070929cc..62a3ba7218d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -87,7 +87,7 @@ public class RMAppImpl implements RMApp, Recoverable { private final YarnScheduler scheduler; private final ApplicationMasterService masterService; private final StringBuilder diagnostics = new StringBuilder(); - private final int maxRetries; + private final int maxAppAttempts; private final ReadLock readLock; private final WriteLock writeLock; private final Map attempts @@ -231,8 +231,19 @@ public class RMAppImpl implements RMApp, Recoverable { this.submitTime = submitTime; this.startTime = System.currentTimeMillis(); - this.maxRetries = conf.getInt(YarnConfiguration.RM_AM_MAX_RETRIES, - YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES); + int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + int individualMaxAppAttempts = submissionContext.getMaxAppAttempts(); + if (individualMaxAppAttempts <= 0 || + individualMaxAppAttempts > globalMaxAppAttempts) { + this.maxAppAttempts = globalMaxAppAttempts; + LOG.warn("The specific max attempts: " + individualMaxAppAttempts + + " for application: " + applicationId.getId() + + " is invalid, because it is out of the range [1, " + + globalMaxAppAttempts + "]. Use the global max attempts instead."); + } else { + this.maxAppAttempts = individualMaxAppAttempts; + } ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); this.readLock = lock.readLock(); @@ -493,6 +504,11 @@ public class RMAppImpl implements RMApp, Recoverable { } } + @Override + public int getMaxAppAttempts() { + return this.maxAppAttempts; + } + @Override public void handle(RMAppEvent event) { @@ -669,10 +685,10 @@ public class RMAppImpl implements RMApp, Recoverable { msg = "Unmanaged application " + app.getApplicationId() + " failed due to " + failedEvent.getDiagnostics() + ". Failing the application."; - } else if (app.attempts.size() >= app.maxRetries) { + } else if (app.attempts.size() >= app.maxAppAttempts) { retryApp = false; msg = "Application " + app.getApplicationId() + " failed " - + app.maxRetries + " times due to " + failedEvent.getDiagnostics() + + app.maxAppAttempts + " times due to " + failedEvent.getDiagnostics() + ". Failing the application."; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java index c43d9ecbb5b..510c4fea08b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java @@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; /** * Interface to an Application Attempt in the Resource Manager. * A {@link RMApp} can have multiple app attempts based on - * {@link YarnConfiguration#RM_AM_MAX_RETRIES}. For specific + * {@link YarnConfiguration#RM_AM_MAX_ATTEMPTS}. For specific * implementation take a look at {@link RMAppAttemptImpl}. */ public interface RMAppAttempt extends EventHandler { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index b85d482ede1..7e06fac573b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -377,7 +377,61 @@ public class TestAppManager{ ((Service)rmContext.getDispatcher()).stop(); } - @Test + @Test (timeout = 30000) + public void testRMAppSubmitMaxAppAttempts() throws Exception { + int[] globalMaxAppAttempts = new int[] { 10, 1 }; + int[][] individualMaxAppAttempts = new int[][]{ + new int[]{ 9, 10, 11, 0 }, + new int[]{ 1, 10, 0, -1 }}; + int[][] expectedNums = new int[][]{ + new int[]{ 9, 10, 10, 10 }, + new int[]{ 1, 1, 1, 1 }}; + for (int i = 0; i < globalMaxAppAttempts.length; ++i) { + for (int j = 0; j < individualMaxAppAttempts.length; ++j) { + long now = System.currentTimeMillis(); + + RMContext rmContext = mockRMContext(0, now - 10); + ResourceScheduler scheduler = new CapacityScheduler(); + Configuration conf = new Configuration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, globalMaxAppAttempts[i]); + ApplicationMasterService masterService = + new ApplicationMasterService(rmContext, scheduler); + TestRMAppManager appMonitor = new TestRMAppManager(rmContext, + new ClientToAMTokenSecretManagerInRM(), scheduler, masterService, + new ApplicationACLsManager(conf), conf); + + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + ApplicationSubmissionContext context = + recordFactory.newRecordInstance(ApplicationSubmissionContext.class); + ContainerLaunchContext amContainer = recordFactory + .newRecordInstance(ContainerLaunchContext.class); + amContainer.setApplicationACLs(new HashMap()); + context.setAMContainerSpec(amContainer); + setupDispatcher(rmContext, conf); + + ApplicationId appID = MockApps.newAppID(1); + context.setApplicationId(appID); + if (individualMaxAppAttempts[i][j] != 0) { + context.setMaxAppAttempts(individualMaxAppAttempts[i][j]); + } + appMonitor.submitApplication(context); + RMApp app = rmContext.getRMApps().get(appID); + Assert.assertEquals("max application attempts doesn't match", + expectedNums[i][j], app.getMaxAppAttempts()); + + // wait for event to be processed + int timeoutSecs = 0; + while ((getAppEventType() == RMAppEventType.KILL) && + timeoutSecs++ < 20) { + Thread.sleep(1000); + } + setAppEventType(RMAppEventType.KILL); + ((Service)rmContext.getDispatcher()).stop(); + } + } + } + + @Test (timeout = 3000) public void testRMAppSubmitWithQueueAndName() throws Exception { long now = System.currentTimeMillis(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java index 90620ed4822..84fd9a6dd51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -60,6 +61,7 @@ public class TestApplicationMasterLauncher { int nmPortAtContainerManager; int nmHttpPortAtContainerManager; long submitTimeAtContainerManager; + int maxAppAttempts; @Override public StartContainerResponse @@ -82,7 +84,8 @@ public class TestApplicationMasterLauncher { Integer.parseInt(env.get(ApplicationConstants.NM_HTTP_PORT_ENV)); submitTimeAtContainerManager = Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV)); - + maxAppAttempts = + Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV)); return null; } @@ -139,6 +142,8 @@ public class TestApplicationMasterLauncher { containerManager.nmPortAtContainerManager); Assert.assertEquals(nm1.getHttpPort(), containerManager.nmHttpPortAtContainerManager); + Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, + containerManager.maxAppAttempts); MockAM am = new MockAM(rm.getRMContext(), rm .getApplicationMasterService(), appAttemptId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 4dcf84a34e4..0d5ef6d6f35 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -374,8 +374,10 @@ public class TestClientRMService { private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, ApplicationId applicationId3, YarnConfiguration config, String queueName) { + ApplicationSubmissionContext asContext = mock(ApplicationSubmissionContext.class); + when(asContext.getMaxAppAttempts()).thenReturn(1); return new RMAppImpl(applicationId3, rmContext, config, null, null, - queueName, null, yarnScheduler, null , System + queueName, asContext, yarnScheduler, null , System .currentTimeMillis()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index c6ee0d57402..1596bfddd6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; import java.io.IOException; import java.util.Collection; @@ -27,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.net.NetworkTopology; +import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; @@ -183,4 +185,16 @@ public class TestResourceManager { } } + @Test (timeout = 30000) + public void testResourceManagerInitConfigValidation() throws Exception { + Configuration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, -1); + try { + resourceManager.init(conf); + fail("Exception is expected because the global max attempts is negative."); + } catch (YarnException e) { + // Exception is expected. + } + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index eea5c6558fc..afab1809203 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -216,6 +216,10 @@ public abstract class MockAsm extends MockApps { throw new UnsupportedOperationException("Not supported yet."); } @Override + public int getMaxAppAttempts() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public ApplicationReport createAndGetApplicationReport(boolean allowAccess) { throw new UnsupportedOperationException("Not supported yet."); } @@ -248,6 +252,7 @@ public abstract class MockAsm extends MockApps { final long finish = 234567 + i * 1000; RMAppState[] allStates = RMAppState.values(); final RMAppState state = allStates[i % allStates.length]; + final int maxAppAttempts = i % 1000; return new ApplicationBase() { @Override public ApplicationId getApplicationId() { @@ -302,6 +307,11 @@ public abstract class MockAsm extends MockApps { public RMAppAttempt getCurrentAppAttempt() { return null; } + + @Override + public int getMaxAppAttempts() { + return maxAppAttempts; + } }; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java index 142cc7e9e1e..c88cebce643 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java @@ -48,6 +48,7 @@ public class MockRMApp implements RMApp { String url = null; StringBuilder diagnostics = new StringBuilder(); RMAppAttempt attempt; + int maxAppAttempts = 1; public MockRMApp(int newid, long time, RMAppState newState) { finish = time; @@ -186,6 +187,15 @@ public class MockRMApp implements RMApp { this.diagnostics = new StringBuilder(diag); } + @Override + public int getMaxAppAttempts() { + return maxAppAttempts; + } + + public void setNumMaxRetries(int maxAppAttempts) { + this.maxAppAttempts = maxAppAttempts; + } + @Override public void handle(RMAppEvent event) { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 6d407dd4459..8e8e4859902 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -61,7 +61,7 @@ public class TestRMAppTransitions { static final Log LOG = LogFactory.getLog(TestRMAppTransitions.class); private RMContext rmContext; - private static int maxRetries = 4; + private static int maxAppAttempts = 4; private static int appId = 1; private DrainDispatcher rmDispatcher; @@ -167,8 +167,8 @@ public class TestRMAppTransitions { String name = MockApps.newAppName(); String queue = MockApps.newQueue(); Configuration conf = new YarnConfiguration(); - // ensure max retries set to known value - conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, maxRetries); + // ensure max application attempts set to known value + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, maxAppAttempts); YarnScheduler scheduler = mock(YarnScheduler.class); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); @@ -437,7 +437,7 @@ public class TestRMAppTransitions { RMApp application = testCreateAppAccepted(null); // ACCEPTED => ACCEPTED event RMAppEventType.RMAppEventType.ATTEMPT_FAILED - for (int i=1; i FAILED event RMAppEventType.RMAppEventType.ATTEMPT_FAILED - // after max retries + // after max application attempts String message = "Test fail"; RMAppEvent event = new RMAppFailedAttemptEvent(application.getApplicationId(), @@ -500,7 +500,7 @@ public class TestRMAppTransitions { Assert.assertEquals(expectedAttemptId, appAttempt.getAppAttemptId().getAttemptId()); // RUNNING => FAILED/RESTARTING event RMAppEventType.ATTEMPT_FAILED - for (int i=1; i FAILED/RESTARTING event RMAppEventType.ATTEMPT_FAILED - // after max retries + // after max application attempts RMAppEvent event = new RMAppFailedAttemptEvent(application.getApplicationId(), RMAppEventType.ATTEMPT_FAILED, ""); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index b81237d0b8c..5d0be9e630d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -83,7 +83,7 @@ public class TestRMWebServicesApps extends JerseyTest { bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration conf = new Configuration(); - conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 2); + conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rm = new MockRM(conf); @@ -871,9 +871,9 @@ public class TestRMWebServicesApps extends JerseyTest { MockNM amNodeManager = rm.registerNode("amNM:1234", 2048); RMApp app1 = rm.submitApp(1024, "testwordcount", "user1"); amNodeManager.nodeHeartbeat(true); - int maxRetries = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_RETRIES, - YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES); - int retriesLeft = maxRetries; + int maxAppAttempts = rm.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); + int retriesLeft = maxAppAttempts; while (--retriesLeft > 0) { RMAppEvent event = new RMAppFailedAttemptEvent(app1.getApplicationId(), @@ -882,7 +882,7 @@ public class TestRMWebServicesApps extends JerseyTest { rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED); amNodeManager.nodeHeartbeat(true); } - assertEquals("incorrect number of attempts", maxRetries, + assertEquals("incorrect number of attempts", maxAppAttempts, app1.getAppAttempts().values().size()); testAppAttemptsHelper(app1.getApplicationId().toString(), app1, MediaType.APPLICATION_JSON); From 5e325d456293c6f204ec748dafa073576e4f18df Mon Sep 17 00:00:00 2001 From: Sanjay Radia Date: Mon, 25 Mar 2013 22:30:30 +0000 Subject: [PATCH 09/37] HADOOP-9433 TestLocalFileSystem#testHasFileDescriptor leaks file handle (Chris Nauroth via sanjay) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460922 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 11 +++++++---- .../org/apache/hadoop/fs/TestLocalFileSystem.java | 12 +++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e959bc8b593..f25f91cf2e7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -356,11 +356,14 @@ Trunk (Unreleased) HADOOP-9431 TestSecurityUtil#testLocalHostNameForNullOrWild on systems where hostname contains capital letters (Chris Nauroth via sanjay) - HADOOP-9261 S3n filesystem can move a directory under itself -and so lose data - (fixed in HADOOP-9258) (stevel) + HADOOP-9261 S3n filesystem can move a directory under itself -and so lose data + (fixed in HADOOP-9258) (stevel) - HADOOP-9265 S3 blockstore filesystem breaks part of the Filesystem contract - (fixed in HADOOP-9258) (stevel) + HADOOP-9265 S3 blockstore filesystem breaks part of the Filesystem contract + (fixed in HADOOP-9258) (stevel) + + HADOOP-9433 TestLocalFileSystem#testHasFileDescriptor leaks file handle + (Chris Nauroth via sanjay) OPTIMIZATIONS diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index b843e9cf13e..a5c354266c9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Shell; import static org.apache.hadoop.fs.FileSystemTestHelper.*; @@ -266,9 +267,14 @@ public class TestLocalFileSystem { LocalFileSystem fs = FileSystem.getLocal(conf); Path path = new Path(TEST_ROOT_DIR, "test-file"); writeFile(fs, path, 1); - BufferedFSInputStream bis = new BufferedFSInputStream( - new RawLocalFileSystem().new LocalFSFileInputStream(path), 1024); - assertNotNull(bis.getFileDescriptor()); + BufferedFSInputStream bis = null; + try { + bis = new BufferedFSInputStream(new RawLocalFileSystem() + .new LocalFSFileInputStream(path), 1024); + assertNotNull(bis.getFileDescriptor()); + } finally { + IOUtils.cleanup(null, bis); + } } @Test From 46315a2d914058969c7234272420c063ce268bf5 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Mon, 25 Mar 2013 22:33:43 +0000 Subject: [PATCH 10/37] MAPREDUCE-5062. Fix MR AM to read max-retries from the RM. Contributed by *Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460923 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../hadoop/mapreduce/v2/app/MRAppMaster.java | 42 +++++---- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 2 +- .../mapreduce/v2/app/TestMRAppMaster.java | 89 +++++++++++++++---- .../mapreduce/v2/app/TestStagingCleanup.java | 21 +++-- .../apache/hadoop/mapreduce/MRJobConfig.java | 8 ++ .../src/main/resources/mapred-default.xml | 8 ++ .../org/apache/hadoop/mapred/YARNRunner.java | 3 + 8 files changed, 133 insertions(+), 43 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 771ea575828..5f8d73ff69f 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -253,6 +253,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5083. MiniMRCluster should use a random component when creating an actual cluster (Siddharth Seth via hitesh) + MAPREDUCE-5062. Fix MR AM to read max-retries from the RM. (Zhijie Shen via + vinodkv) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index 8598706dde4..fe7aa4cedbc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -161,6 +161,7 @@ public class MRAppMaster extends CompositeService { private final int nmPort; private final int nmHttpPort; protected final MRAppMetrics metrics; + private final int maxAppAttempts; private Map completedTasksFromPreviousRun; private List amInfos; private AppContext context; @@ -194,14 +195,14 @@ public class MRAppMaster extends CompositeService { public MRAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId, String nmHost, int nmPort, int nmHttpPort, - long appSubmitTime) { + long appSubmitTime, int maxAppAttempts) { this(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort, - new SystemClock(), appSubmitTime); + new SystemClock(), appSubmitTime, maxAppAttempts); } public MRAppMaster(ApplicationAttemptId applicationAttemptId, ContainerId containerId, String nmHost, int nmPort, int nmHttpPort, - Clock clock, long appSubmitTime) { + Clock clock, long appSubmitTime, int maxAppAttempts) { super(MRAppMaster.class.getName()); this.clock = clock; this.startTime = clock.getTime(); @@ -212,6 +213,7 @@ public class MRAppMaster extends CompositeService { this.nmPort = nmPort; this.nmHttpPort = nmHttpPort; this.metrics = MRAppMetrics.create(); + this.maxAppAttempts = maxAppAttempts; LOG.info("Created MRAppMaster for application " + applicationAttemptId); } @@ -220,18 +222,13 @@ public class MRAppMaster extends CompositeService { conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true); downloadTokensAndSetupUGI(conf); - - //TODO this is a hack, we really need the RM to inform us when we - // are the last one. This would allow us to configure retries on - // a per application basis. - int numAMRetries = conf.getInt(YarnConfiguration.RM_AM_MAX_RETRIES, - YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES); - isLastAMRetry = appAttemptID.getAttemptId() >= numAMRetries; - LOG.info("AM Retries: " + numAMRetries + - " attempt num: " + appAttemptID.getAttemptId() + + + isLastAMRetry = appAttemptID.getAttemptId() >= maxAppAttempts; + LOG.info("The specific max attempts: " + maxAppAttempts + + " for application: " + appAttemptID.getApplicationId().getId() + + ". Attempt num: " + appAttemptID.getAttemptId() + " is last retry: " + isLastAMRetry); - - + context = new RunningAppContext(conf); // Job name is the same as the app name util we support DAG of jobs @@ -266,6 +263,9 @@ public class MRAppMaster extends CompositeService { boolean commitFailure = fs.exists(endCommitFailureFile); if(!stagingExists) { isLastAMRetry = true; + LOG.info("Attempt num: " + appAttemptID.getAttemptId() + + " is last retry: " + isLastAMRetry + + " because the staging dir doesn't exist."); errorHappenedShutDown = true; forcedState = JobStateInternal.ERROR; shutDownMessage = "Staging dir does not exist " + stagingDir; @@ -275,6 +275,9 @@ public class MRAppMaster extends CompositeService { // what result we will use to notify, and how we will unregister errorHappenedShutDown = true; isLastAMRetry = true; + LOG.info("Attempt num: " + appAttemptID.getAttemptId() + + " is last retry: " + isLastAMRetry + + " because a commit was started."); copyHistory = true; if (commitSuccess) { shutDownMessage = "We crashed after successfully committing. Recovering."; @@ -777,6 +780,10 @@ public class MRAppMaster extends CompositeService { return taskAttemptListener; } + public Boolean isLastAMRetry() { + return isLastAMRetry; + } + /** * By the time life-cycle of this router starts, job-init would have already * happened. @@ -1206,6 +1213,8 @@ public class MRAppMaster extends CompositeService { System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV); String appSubmitTimeStr = System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV); + String maxAppAttempts = + System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV); validateInputParam(containerIdStr, ApplicationConstants.AM_CONTAINER_ID_ENV); @@ -1215,6 +1224,8 @@ public class MRAppMaster extends CompositeService { ApplicationConstants.NM_HTTP_PORT_ENV); validateInputParam(appSubmitTimeStr, ApplicationConstants.APP_SUBMIT_TIME_ENV); + validateInputParam(maxAppAttempts, + ApplicationConstants.MAX_APP_ATTEMPTS_ENV); ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); ApplicationAttemptId applicationAttemptId = @@ -1224,7 +1235,8 @@ public class MRAppMaster extends CompositeService { MRAppMaster appMaster = new MRAppMaster(applicationAttemptId, containerId, nodeHostString, Integer.parseInt(nodePortString), - Integer.parseInt(nodeHttpPortString), appSubmitTime); + Integer.parseInt(nodeHttpPortString), appSubmitTime, + Integer.parseInt(maxAppAttempts)); ShutdownHookManager.get().addShutdownHook( new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY); YarnConfiguration conf = new YarnConfiguration(new JobConf()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 0f24666e6f5..4a28ab00637 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -192,7 +192,7 @@ public class MRApp extends MRAppMaster { int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart, int startCount, Clock clock) { super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock, System - .currentTimeMillis()); + .currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); this.testWorkDir = new File("target", testName); testAbsPath = new Path(testWorkDir.getAbsolutePath()); LOG.info("PathUsed: " + testAbsPath); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java index 2ac20701879..e0c618b556b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java @@ -30,11 +30,15 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.OutputCommitter; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; +import org.apache.hadoop.mapreduce.v2.app.commit.CommitterEvent; +import org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler; import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; +import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.UserGroupInformation; @@ -42,6 +46,7 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Before; import org.junit.BeforeClass; @@ -80,7 +85,7 @@ public class TestMRAppMaster { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); MRAppMasterTest appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, - System.currentTimeMillis()); + System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); YarnConfiguration conf = new YarnConfiguration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); @@ -109,7 +114,8 @@ public class TestMRAppMaster { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, - System.currentTimeMillis(), false); + System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS, + false, false); boolean caught = false; try { MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); @@ -144,7 +150,8 @@ public class TestMRAppMaster { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, - System.currentTimeMillis(), false); + System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS, + false, false); boolean caught = false; try { MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); @@ -179,7 +186,8 @@ public class TestMRAppMaster { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, - System.currentTimeMillis(), false); + System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS, + false, false); boolean caught = false; try { MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); @@ -214,7 +222,8 @@ public class TestMRAppMaster { ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); MRAppMaster appMaster = new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, - System.currentTimeMillis(), false); + System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS, + false, false); boolean caught = false; try { MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); @@ -230,36 +239,73 @@ public class TestMRAppMaster { assertEquals(JobStateInternal.ERROR, appMaster.forcedState); appMaster.stop(); } + + @Test (timeout = 30000) + public void testMRAppMasterMaxAppAttempts() throws IOException, + InterruptedException { + int[] maxAppAttemtps = new int[] { 1, 2, 3 }; + Boolean[] expectedBools = new Boolean[]{ true, true, false }; + + String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002"; + String containerIdStr = "container_1317529182569_0004_000002_1"; + + String userName = "TestAppMasterUser"; + ApplicationAttemptId applicationAttemptId = ConverterUtils + .toApplicationAttemptId(applicationAttemptIdStr); + ContainerId containerId = ConverterUtils.toContainerId(containerIdStr); + YarnConfiguration conf = new YarnConfiguration(); + conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); + + File stagingDir = + new File(MRApps.getStagingAreaDir(conf, userName).toString()); + stagingDir.mkdirs(); + for (int i = 0; i < maxAppAttemtps.length; ++i) { + MRAppMasterTest appMaster = + new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1, + System.currentTimeMillis(), maxAppAttemtps[i], false, true); + MRAppMaster.initAndStartAppMaster(appMaster, conf, userName); + assertEquals("isLastAMRetry is correctly computed.", expectedBools[i], + appMaster.isLastAMRetry()); + } + } + } class MRAppMasterTest extends MRAppMaster { Path stagingDirPath; private Configuration conf; - private boolean overrideInitAndStart; + private boolean overrideInit; + private boolean overrideStart; ContainerAllocator mockContainerAllocator; + CommitterEventHandler mockCommitterEventHandler; + RMHeartbeatHandler mockRMHeartbeatHandler; public MRAppMasterTest(ApplicationAttemptId applicationAttemptId, ContainerId containerId, String host, int port, int httpPort, - long submitTime) { - this(applicationAttemptId, containerId, host, port, httpPort, submitTime, - true); + long submitTime, int maxAppAttempts) { + this(applicationAttemptId, containerId, host, port, httpPort, + submitTime, maxAppAttempts, true, true); } public MRAppMasterTest(ApplicationAttemptId applicationAttemptId, ContainerId containerId, String host, int port, int httpPort, - long submitTime, boolean overrideInitAndStart) { - super(applicationAttemptId, containerId, host, port, httpPort, submitTime); - this.overrideInitAndStart = overrideInitAndStart; + long submitTime, int maxAppAttempts, boolean overrideInit, + boolean overrideStart) { + super(applicationAttemptId, containerId, host, port, httpPort, submitTime, + maxAppAttempts); + this.overrideInit = overrideInit; + this.overrideStart = overrideStart; mockContainerAllocator = mock(ContainerAllocator.class); + mockCommitterEventHandler = mock(CommitterEventHandler.class); + mockRMHeartbeatHandler = mock(RMHeartbeatHandler.class); } @Override public void init(Configuration conf) { - if (overrideInitAndStart) { - this.conf = conf; - } else { + if (!overrideInit) { super.init(conf); } + this.conf = conf; } @Override @@ -277,9 +323,20 @@ class MRAppMasterTest extends MRAppMaster { return mockContainerAllocator; } + @Override + protected EventHandler createCommitterEventHandler( + AppContext context, OutputCommitter committer) { + return mockCommitterEventHandler; + } + + @Override + protected RMHeartbeatHandler getRMHeartbeatHandler() { + return mockRMHeartbeatHandler; + } + @Override public void start() { - if (overrideInitAndStart) { + if (overrideStart) { try { String user = UserGroupInformation.getCurrentUser().getShortUserName(); stagingDirPath = MRApps.getStagingAreaDir(conf, user); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java index 7dfc0b3f0f3..b278186766e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java @@ -49,7 +49,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.service.AbstractService; @@ -93,10 +92,9 @@ import org.junit.Test; verify(fs).delete(stagingJobPath, true); } - @Test + @Test (timeout = 30000) public void testDeletionofStagingOnKill() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); - conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 4); fs = mock(FileSystem.class); when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true); //Staging Dir exists @@ -113,7 +111,7 @@ import org.junit.Test; JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); - MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); + MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 4); appMaster.init(conf); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = @@ -122,10 +120,9 @@ import org.junit.Test; verify(fs, times(0)).delete(stagingJobPath, true); } - @Test + @Test (timeout = 30000) public void testDeletionofStagingOnKillLastTry() throws IOException { conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir); - conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 1); fs = mock(FileSystem.class); when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true); //Staging Dir exists @@ -142,7 +139,8 @@ import org.junit.Test; JobId jobid = recordFactory.newRecordInstance(JobId.class); jobid.setAppId(appId); ContainerAllocator mockAlloc = mock(ContainerAllocator.class); - MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); + MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, + MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); appMaster.init(conf); //simulate the process being killed MRAppMaster.MRAppMasterShutdownHook hook = @@ -155,15 +153,16 @@ import org.junit.Test; ContainerAllocator allocator; public TestMRApp(ApplicationAttemptId applicationAttemptId, - ContainerAllocator allocator) { + ContainerAllocator allocator, int maxAppAttempts) { super(applicationAttemptId, BuilderUtils.newContainerId( - applicationAttemptId, 1), "testhost", 2222, 3333, System - .currentTimeMillis()); + applicationAttemptId, 1), "testhost", 2222, 3333, + System.currentTimeMillis(), maxAppAttempts); this.allocator = allocator; } public TestMRApp(ApplicationAttemptId applicationAttemptId) { - this(applicationAttemptId, null); + this(applicationAttemptId, null, + MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index becdef853fa..3f80065a82c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -663,5 +663,13 @@ public interface MRJobConfig { public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN = "^mapreduce\\.workflow\\.adjacency\\..+"; + + /** + * The maximum number of application attempts. + * It is a application-specific setting. + */ + public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts"; + + public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 1; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 83131e7a798..74db2e8b43a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -806,6 +806,14 @@ + + mapreduce.am.max-attempts + 1 + The maximum number of application attempts. It is a + application-specific setting. It should not be larger than the global number + set by resourcemanager. Otherwise, it will be override. + + mapreduce.job.end-notification.url diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index c401b93c964..e8fd18a4c8a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -481,6 +481,9 @@ public class YARNRunner implements ClientProtocol { appContext.setCancelTokensWhenComplete( conf.getBoolean(MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN, true)); appContext.setAMContainerSpec(amContainer); // AM Container + appContext.setMaxAppAttempts( + conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, + MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS)); return appContext; } From 2aed48a67f408c290e90c83af9f76165d695f91a Mon Sep 17 00:00:00 2001 From: Bikas Saha Date: Tue, 26 Mar 2013 02:34:27 +0000 Subject: [PATCH 11/37] YARN-498. Unmanaged AM launcher does not set various constants in env for an AM, also does not handle failed AMs properly (Hitesh Shah via bikas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460954 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../distributedshell/ApplicationMaster.java | 22 ++++++-- .../UnmanagedAMLauncher.java | 56 +++++++++++++++++-- .../TestUnmanagedAMLauncher.java | 40 ++++++++++++- 4 files changed, 111 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5a6dda9aad3..f8a4937922a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -136,6 +136,9 @@ Release 2.0.5-beta - UNRELEASED YARN-378. Fix RM to make the AM max attempts/retries to be configurable per application by clients. (Zhijie Shen via vinodkv) + YARN-498. Unmanaged AM launcher does not set various constants in env for + an AM, also does not handle failed AMs properly. (Hitesh Shah via bikas) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 851bad584cf..1e65a9a73bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -319,10 +319,7 @@ public class ApplicationMaster { Map envs = System.getenv(); - if (envs.containsKey(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)) { - appAttemptID = ConverterUtils.toApplicationAttemptId(envs - .get(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV)); - } else if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { + if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) { if (cliParser.hasOption("app_attempt_id")) { String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); @@ -336,6 +333,23 @@ public class ApplicationMaster { appAttemptID = containerId.getApplicationAttemptId(); } + if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { + throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV + + " not set in the environment"); + } + if (!envs.containsKey(ApplicationConstants.NM_HOST_ENV)) { + throw new RuntimeException(ApplicationConstants.NM_HOST_ENV + + " not set in the environment"); + } + if (!envs.containsKey(ApplicationConstants.NM_HTTP_PORT_ENV)) { + throw new RuntimeException(ApplicationConstants.NM_HTTP_PORT_ENV + + " not set in the environment"); + } + if (!envs.containsKey(ApplicationConstants.NM_PORT_ENV)) { + throw new RuntimeException(ApplicationConstants.NM_PORT_ENV + + " not set in the environment"); + } + LOG.info("Application master for app" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java index b9e06256a82..62ac67fc170 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java @@ -22,6 +22,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.net.InetAddress; import java.util.ArrayList; import java.util.EnumSet; import java.util.Map; @@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; @@ -81,6 +83,8 @@ public class UnmanagedAMLauncher { // set the classpath explicitly private String classpath = null; + private volatile boolean amCompleted = false; + /** * @param args * Command line arguments @@ -179,8 +183,18 @@ public class UnmanagedAMLauncher { if(!setClasspath && classpath!=null) { envAMList.add("CLASSPATH="+classpath); } - - envAMList.add(ApplicationConstants.AM_APP_ATTEMPT_ID_ENV + "=" + attemptId); + + ContainerId containerId = Records.newRecord(ContainerId.class); + containerId.setApplicationAttemptId(attemptId); + containerId.setId(0); + + String hostname = InetAddress.getLocalHost().getHostName(); + envAMList.add(ApplicationConstants.AM_CONTAINER_ID_ENV + "=" + containerId); + envAMList.add(ApplicationConstants.NM_HOST_ENV + "=" + hostname); + envAMList.add(ApplicationConstants.NM_HTTP_PORT_ENV + "=0"); + envAMList.add(ApplicationConstants.NM_PORT_ENV + "=0"); + envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "=" + + System.currentTimeMillis()); String[] envAM = new String[envAMList.size()]; Process amProc = Runtime.getRuntime().exec(amCmd, envAMList.toArray(envAM)); @@ -233,8 +247,10 @@ public class UnmanagedAMLauncher { LOG.info("AM process exited with value: " + exitCode); } catch (InterruptedException e) { e.printStackTrace(); + } finally { + amCompleted = true; } - + try { // make sure that the error thread exits // on Windows these threads sometimes get stuck and hang the execution @@ -306,6 +322,7 @@ public class UnmanagedAMLauncher { appReport = monitorApplication(appId, EnumSet.of( YarnApplicationState.KILLED, YarnApplicationState.FAILED, YarnApplicationState.FINISHED)); + YarnApplicationState appState = appReport.getYarnApplicationState(); FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus(); @@ -341,6 +358,19 @@ public class UnmanagedAMLauncher { private ApplicationReport monitorApplication(ApplicationId appId, Set finalState) throws YarnRemoteException { + long foundAMCompletedTime = 0; + final int timeToWaitMS = 10000; + StringBuilder expectedFinalState = new StringBuilder(); + boolean first = true; + for (YarnApplicationState state : finalState) { + if (first) { + first = false; + expectedFinalState.append(state.name()); + } else { + expectedFinalState.append("," + state.name()); + } + } + while (true) { // Check app status every 1 second. @@ -370,8 +400,24 @@ public class UnmanagedAMLauncher { return report; } + // wait for 10 seconds after process has completed for app report to + // come back + if (amCompleted) { + if (foundAMCompletedTime == 0) { + foundAMCompletedTime = System.currentTimeMillis(); + } else if ((System.currentTimeMillis() - foundAMCompletedTime) + > timeToWaitMS) { + LOG.warn("Waited " + timeToWaitMS/1000 + + " seconds after process completed for AppReport" + + " to reach desired final state. Not waiting anymore." + + "CurrentState = " + state + + ", ExpectedStates = " + expectedFinalState.toString()); + throw new RuntimeException("Failed to receive final expected state" + + " in ApplicationReport" + + ", CurrentState=" + state + + ", ExpectedStates=" + expectedFinalState.toString()); + } + } } - } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java index 3e6e126da1e..43fec24afca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java @@ -18,6 +18,8 @@ package org.apache.hadoop.yarn.applications.unmanagedamlauncher; +import static org.junit.Assert.fail; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -99,7 +101,7 @@ public class TestUnmanagedAMLauncher { LOG.fatal("JAVA_HOME not defined. Test not running."); return; } - // start dist-shell with 0 containers because container launch will fail if + // start dist-shell with 0 containers because container launch will fail if // there are no dist cache resources. String[] args = { "--classpath", @@ -125,4 +127,40 @@ public class TestUnmanagedAMLauncher { } + @Test(timeout=30000) + public void testDSShellError() throws Exception { + String classpath = getTestRuntimeClasspath(); + String javaHome = System.getenv("JAVA_HOME"); + if (javaHome == null) { + LOG.fatal("JAVA_HOME not defined. Test not running."); + return; + } + + // remove shell command to make dist-shell fail in initialization itself + String[] args = { + "--classpath", + classpath, + "--queue", + "default", + "--cmd", + javaHome + + "/bin/java -Xmx512m " + + "org.apache.hadoop.yarn.applications.distributedshell.ApplicationMaster " + + "--container_memory 128 --num_containers 1 --priority 0" }; + + LOG.info("Initializing Launcher"); + UnmanagedAMLauncher launcher = new UnmanagedAMLauncher(new Configuration( + yarnCluster.getConfig())); + boolean initSuccess = launcher.init(args); + Assert.assertTrue(initSuccess); + LOG.info("Running Launcher"); + + try { + launcher.run(); + fail("Expected an exception to occur as launch should have failed"); + } catch (RuntimeException e) { + // Expected + } + } + } From e74d1f0435c2bcdfae2c26f6c340a5a487d20aa3 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Tue, 26 Mar 2013 03:25:26 +0000 Subject: [PATCH 12/37] YARN-469. Make scheduling mode in FS pluggable. (kkambatl via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1460961 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../scheduler/fair/FSLeafQueue.java | 20 +-- .../scheduler/fair/FSParentQueue.java | 2 +- .../scheduler/fair/FairScheduler.java | 2 +- .../scheduler/fair/QueueManager.java | 22 +-- .../scheduler/fair/Schedulable.java | 2 +- .../scheduler/fair/SchedulingMode.java | 105 +++++++++++-- .../FairSchedulingMode.java} | 142 +++++++++--------- .../fair/modes/FifoSchedulingMode.java | 76 ++++++++++ .../scheduler/fair/TestComputeFairShares.java | 31 ++-- .../scheduler/fair/TestFairScheduler.java | 3 +- .../scheduler/fair/TestSchedulingMode.java | 59 ++++++++ 12 files changed, 335 insertions(+), 131 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/{SchedulingAlgorithms.java => modes/FairSchedulingMode.java} (71%) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FifoSchedulingMode.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingMode.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f8a4937922a..c6d4b36d894 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -96,6 +96,8 @@ Release 2.0.5-beta - UNRELEASED YARN-497. Yarn unmanaged-am launcher jar does not define a main class in its manifest (Hitesh Shah via bikas) + YARN-469. Make scheduling mode in FS pluggable. (kkambatl via tucu) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index ff51e9506a2..2c8555005bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -92,13 +92,7 @@ public class FSLeafQueue extends FSQueue { @Override public void recomputeFairShares() { - if (schedulingMode == SchedulingMode.FAIR) { - SchedulingAlgorithms.computeFairShares(appScheds, getFairShare()); - } else { - for (AppSchedulable sched: appScheds) { - sched.setFairShare(Resources.createResource(0)); - } - } + schedulingMode.computeShares(getAppSchedulables(), getFairShare()); } @Override @@ -162,17 +156,9 @@ public class FSLeafQueue extends FSQueue { return Resources.none(); // We should never get here } - // Otherwise, chose app to schedule based on given policy (fair vs fifo). + // Otherwise, chose app to schedule based on given policy. else { - Comparator comparator; - if (schedulingMode == SchedulingMode.FIFO) { - comparator = new SchedulingAlgorithms.FifoComparator(); - } else if (schedulingMode == SchedulingMode.FAIR) { - comparator = new SchedulingAlgorithms.FairShareComparator(); - } else { - throw new RuntimeException("Unsupported queue scheduling mode " + - schedulingMode); - } + Comparator comparator = schedulingMode.getComparator(); Collections.sort(appScheds, comparator); for (AppSchedulable sched: appScheds) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java index e6b5a8f4940..dec5d888bd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java @@ -51,7 +51,7 @@ public class FSParentQueue extends FSQueue { @Override public void recomputeFairShares() { - SchedulingAlgorithms.computeFairShares(childQueues, getFairShare()); + SchedulingMode.getDefault().computeShares(childQueues, getFairShare()); for (FSQueue childQueue : childQueues) { childQueue.getMetrics().setAvailableResourcesToQueue(childQueue.getFairShare()); childQueue.recomputeFairShares(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 45cad65738d..53cc45a382e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -803,7 +803,7 @@ public class FairScheduler implements ResourceScheduler { // At most one task is scheduled each iteration of this loop List scheds = new ArrayList( queueMgr.getLeafQueues()); - Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator()); + Collections.sort(scheds, SchedulingMode.getDefault().getComparator()); boolean assignedContainer = false; for (FSLeafQueue sched : scheds) { Resource assigned = sched.assignContainer(node, false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java index b10bfa0569c..fc76d02c5fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java @@ -310,7 +310,7 @@ public class QueueManager { int queueMaxAppsDefault = Integer.MAX_VALUE; long fairSharePreemptionTimeout = Long.MAX_VALUE; long defaultMinSharePreemptionTimeout = Long.MAX_VALUE; - SchedulingMode defaultSchedulingMode = SchedulingMode.FAIR; + SchedulingMode defaultSchedulingMode = SchedulingMode.getDefault(); // Remember all queue names so we can display them on web UI, etc. List queueNamesInAllocFile = new ArrayList(); @@ -373,7 +373,8 @@ public class QueueManager { queueMaxAppsDefault = val;} else if ("defaultQueueSchedulingMode".equals(element.getTagName())) { String text = ((Text)element.getFirstChild()).getData().trim(); - defaultSchedulingMode = parseSchedulingMode(text); + SchedulingMode.setDefault(text); + defaultSchedulingMode = SchedulingMode.getDefault(); } else { LOG.warn("Bad element in allocations file: " + element.getTagName()); } @@ -449,7 +450,7 @@ public class QueueManager { minSharePreemptionTimeouts.put(queueName, val); } else if ("schedulingMode".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); - queueModes.put(queueName, parseSchedulingMode(text)); + queueModes.put(queueName, SchedulingMode.parse(text)); } else if ("aclSubmitApps".equals(field.getTagName())) { String text = ((Text)field.getFirstChild()).getData().trim(); acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text)); @@ -476,19 +477,6 @@ public class QueueManager { } } - private SchedulingMode parseSchedulingMode(String text) - throws AllocationConfigurationException { - text = text.toLowerCase(); - if (text.equals("fair")) { - return SchedulingMode.FAIR; - } else if (text.equals("fifo")) { - return SchedulingMode.FIFO; - } else { - throw new AllocationConfigurationException( - "Unknown scheduling mode : " + text + "; expected 'fifo' or 'fair'"); - } - } - /** * Get the minimum resource allocation for the given queue. * @return the cap set on this queue, or 0 if not set. @@ -663,7 +651,7 @@ public class QueueManager { minSharePreemptionTimeouts = new HashMap(); defaultMinSharePreemptionTimeout = Long.MAX_VALUE; fairSharePreemptionTimeout = Long.MAX_VALUE; - defaultSchedulingMode = SchedulingMode.FAIR; + defaultSchedulingMode = SchedulingMode.getDefault(); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java index cbcbd46f5f2..1dc0630dd7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java @@ -55,7 +55,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; */ @Private @Unstable -abstract class Schedulable { +public abstract class Schedulable { /** Fair share assigned to this Schedulable */ private Resource fairShare = Resources.createResource(0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingMode.java index b84d824f6cb..96948524a12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingMode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingMode.java @@ -15,17 +15,104 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; +import java.util.Collection; +import java.util.Comparator; +import java.util.concurrent.ConcurrentHashMap; -/** - * Internal scheduling modes for queues. - */ -@Private +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FairSchedulingMode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode; + +@Public @Unstable -public enum SchedulingMode { - FAIR, FIFO +public abstract class SchedulingMode { + private static final ConcurrentHashMap, SchedulingMode> instances = + new ConcurrentHashMap, SchedulingMode>(); + + private static SchedulingMode DEFAULT_MODE = + getInstance(FairSchedulingMode.class); + + public static SchedulingMode getDefault() { + return DEFAULT_MODE; + } + + public static void setDefault(String className) + throws AllocationConfigurationException { + DEFAULT_MODE = parse(className); + } + + /** + * Returns a {@link SchedulingMode} instance corresponding to the passed clazz + */ + public static SchedulingMode getInstance(Class clazz) { + SchedulingMode mode = instances.get(clazz); + if (mode == null) { + mode = ReflectionUtils.newInstance(clazz, null); + instances.put(clazz, mode); + } + return mode; + } + + /** + * Returns {@link SchedulingMode} instance corresponding to the + * {@link SchedulingMode} passed as a string. The mode can be "fair" for + * FairSchedulingMode of "fifo" for FifoSchedulingMode. For custom + * {@link SchedulingMode}s in the RM classpath, the mode should be canonical + * class name of the {@link SchedulingMode}. + * + * @param mode canonical class name or "fair" or "fifo" + * @throws AllocationConfigurationException + */ + @SuppressWarnings("unchecked") + public static SchedulingMode parse(String mode) + throws AllocationConfigurationException { + @SuppressWarnings("rawtypes") + Class clazz; + String text = mode.toLowerCase(); + if (text.equals("fair")) { + clazz = FairSchedulingMode.class; + } else if (text.equals("fifo")) { + clazz = FifoSchedulingMode.class; + } else { + try { + clazz = Class.forName(mode); + } catch (ClassNotFoundException cnfe) { + throw new AllocationConfigurationException(mode + + " SchedulingMode class not found!"); + } + } + if (!SchedulingMode.class.isAssignableFrom(clazz)) { + throw new AllocationConfigurationException(mode + + " does not extend SchedulingMode"); + } + return getInstance(clazz); + } + + /** + * @return returns the name of SchedulingMode + */ + public abstract String getName(); + + /** + * The comparator returned by this method is to be used for sorting the + * {@link Schedulable}s in that queue. + * + * @return the comparator to sort by + */ + public abstract Comparator getComparator(); + + /** + * Computes and updates the shares of {@link Schedulable}s as per the + * SchedulingMode, to be used later at schedule time. + * + * @param schedulables {@link Schedulable}s whose shares are to be updated + * @param totalResources Total {@link Resource}s in the cluster + */ + public abstract void computeShares( + Collection schedulables, Resource totalResources); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingAlgorithms.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FairSchedulingMode.java similarity index 71% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingAlgorithms.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FairSchedulingMode.java index a7a321c5f98..56a37f48899 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingAlgorithms.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FairSchedulingMode.java @@ -15,65 +15,44 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes; import java.io.Serializable; import java.util.Collection; import java.util.Comparator; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingMode; -/** - * Utility class containing scheduling algorithms used in the fair scheduler. - */ -@Private -@Unstable -class SchedulingAlgorithms { - public static final Log LOG = LogFactory.getLog( - SchedulingAlgorithms.class.getName()); +import com.google.common.annotations.VisibleForTesting; - /** - * Compare Schedulables in order of priority and then submission time, as in - * the default FIFO scheduler in Hadoop. - */ - public static class FifoComparator implements Comparator, Serializable { - private static final long serialVersionUID = -5905036205491177060L; +public class FairSchedulingMode extends SchedulingMode { + @VisibleForTesting + public static final String NAME = "FairShare"; + private FairShareComparator comparator = new FairShareComparator(); - @Override - public int compare(Schedulable s1, Schedulable s2) { - int res = s1.getPriority().compareTo(s2.getPriority()); - if (res == 0) { - res = (int) Math.signum(s1.getStartTime() - s2.getStartTime()); - } - if (res == 0) { - // In the rare case where jobs were submitted at the exact same time, - // compare them by name (which will be the JobID) to get a deterministic - // ordering, so we don't alternately launch tasks from different jobs. - res = s1.getName().compareTo(s2.getName()); - } - return res; - } + @Override + public String getName() { + return NAME; } /** * Compare Schedulables via weighted fair sharing. In addition, Schedulables * below their min share get priority over those whose min share is met. - * + * * Schedulables below their min share are compared by how far below it they * are as a ratio. For example, if job A has 8 out of a min share of 10 tasks * and job B has 50 out of a min share of 100, then job B is scheduled next, * because B is at 50% of its min share and A is at 80% of its min share. - * + * * Schedulables above their min share are compared by (runningTasks / weight). * If all weights are equal, slots are given to the job with the fewest tasks; * otherwise, jobs with more weight get proportionally more slots. */ - public static class FairShareComparator implements Comparator, Serializable { + private static class FairShareComparator implements Comparator, + Serializable { private static final long serialVersionUID = 5564969375856699313L; @Override @@ -85,10 +64,10 @@ class SchedulingAlgorithms { boolean s1Needy = Resources.lessThan(s1.getResourceUsage(), minShare1); boolean s2Needy = Resources.lessThan(s2.getResourceUsage(), minShare2); Resource one = Resources.createResource(1); - minShareRatio1 = (double) s1.getResourceUsage().getMemory() / - Resources.max(minShare1, one).getMemory(); - minShareRatio2 = (double) s2.getResourceUsage().getMemory() / - Resources.max(minShare2, one).getMemory(); + minShareRatio1 = (double) s1.getResourceUsage().getMemory() + / Resources.max(minShare1, one).getMemory(); + minShareRatio2 = (double) s2.getResourceUsage().getMemory() + / Resources.max(minShare2, one).getMemory(); useToWeightRatio1 = s1.getResourceUsage().getMemory() / s1.getWeight(); useToWeightRatio2 = s2.getResourceUsage().getMemory() / s2.getWeight(); int res = 0; @@ -98,7 +77,8 @@ class SchedulingAlgorithms { res = 1; else if (s1Needy && s2Needy) res = (int) Math.signum(minShareRatio1 - minShareRatio2); - else // Neither schedulable is needy + else + // Neither schedulable is needy res = (int) Math.signum(useToWeightRatio1 - useToWeightRatio2); if (res == 0) { // Apps are tied in fairness ratio. Break the tie by submit time and job @@ -111,6 +91,17 @@ class SchedulingAlgorithms { } } + @Override + public Comparator getComparator() { + return comparator; + } + + @Override + public void computeShares(Collection schedulables, + Resource totalResources) { + computeFairShares(schedulables, totalResources); + } + /** * Number of iterations for the binary search in computeFairShares. This is * equivalent to the number of bits of precision in the output. 25 iterations @@ -121,43 +112,42 @@ class SchedulingAlgorithms { /** * Given a set of Schedulables and a number of slots, compute their weighted * fair shares. The min shares and demands of the Schedulables are assumed to - * be set beforehand. We compute the fairest possible allocation of shares - * to the Schedulables that respects their min shares and demands. - * + * be set beforehand. We compute the fairest possible allocation of shares to + * the Schedulables that respects their min shares and demands. + * * To understand what this method does, we must first define what weighted * fair sharing means in the presence of minimum shares and demands. If there * were no minimum shares and every Schedulable had an infinite demand (i.e. * could launch infinitely many tasks), then weighted fair sharing would be * achieved if the ratio of slotsAssigned / weight was equal for each - * Schedulable and all slots were assigned. Minimum shares and demands add - * two further twists: - * - Some Schedulables may not have enough tasks to fill all their share. - * - Some Schedulables may have a min share higher than their assigned share. - * - * To deal with these possibilities, we define an assignment of slots as - * being fair if there exists a ratio R such that: - * - Schedulables S where S.demand < R * S.weight are assigned share S.demand - * - Schedulables S where S.minShare > R * S.weight are given share S.minShare - * - All other Schedulables S are assigned share R * S.weight - * - The sum of all the shares is totalSlots. - * + * Schedulable and all slots were assigned. Minimum shares and demands add two + * further twists: - Some Schedulables may not have enough tasks to fill all + * their share. - Some Schedulables may have a min share higher than their + * assigned share. + * + * To deal with these possibilities, we define an assignment of slots as being + * fair if there exists a ratio R such that: - Schedulables S where S.demand < + * R * S.weight are assigned share S.demand - Schedulables S where S.minShare + * > R * S.weight are given share S.minShare - All other Schedulables S are + * assigned share R * S.weight - The sum of all the shares is totalSlots. + * * We call R the weight-to-slots ratio because it converts a Schedulable's * weight to the number of slots it is assigned. - * + * * We compute a fair allocation by finding a suitable weight-to-slot ratio R. - * To do this, we use binary search. Given a ratio R, we compute the number - * of slots that would be used in total with this ratio (the sum of the shares + * To do this, we use binary search. Given a ratio R, we compute the number of + * slots that would be used in total with this ratio (the sum of the shares * computed using the conditions above). If this number of slots is less than * totalSlots, then R is too small and more slots could be assigned. If the * number of slots is more than totalSlots, then R is too large. - * + * * We begin the binary search with a lower bound on R of 0 (which means that * all Schedulables are only given their minShare) and an upper bound computed * to be large enough that too many slots are given (by doubling R until we - * either use more than totalSlots slots or we fulfill all jobs' demands). - * The helper method slotsUsedWithWeightToSlotRatio computes the total number - * of slots used with a given value of R. - * + * either use more than totalSlots slots or we fulfill all jobs' demands). The + * helper method slotsUsedWithWeightToSlotRatio computes the total number of + * slots used with a given value of R. + * * The running time of this algorithm is linear in the number of Schedulables, * because slotsUsedWithWeightToSlotRatio is linear-time and the number of * iterations of binary search is a constant (dependent on desired precision). @@ -168,12 +158,13 @@ class SchedulingAlgorithms { // at R = 1 and double it until we have either used totalSlots slots or we // have met all Schedulables' demands (if total demand < totalSlots). Resource totalDemand = Resources.createResource(0); - for (Schedulable sched: schedulables) { + for (Schedulable sched : schedulables) { Resources.addTo(totalDemand, sched.getDemand()); } Resource cap = Resources.min(totalDemand, totalResources); double rMax = 1.0; - while (Resources.lessThan(resUsedWithWeightToResRatio(rMax, schedulables), cap)) { + while (Resources.lessThan(resUsedWithWeightToResRatio(rMax, schedulables), + cap)) { rMax *= 2.0; } // Perform the binary search for up to COMPUTE_FAIR_SHARES_ITERATIONS steps @@ -181,27 +172,28 @@ class SchedulingAlgorithms { double right = rMax; for (int i = 0; i < COMPUTE_FAIR_SHARES_ITERATIONS; i++) { double mid = (left + right) / 2.0; - if (Resources.lessThan(resUsedWithWeightToResRatio(mid, schedulables), cap)) { + if (Resources.lessThan(resUsedWithWeightToResRatio(mid, schedulables), + cap)) { left = mid; } else { right = mid; } } // Set the fair shares based on the value of R we've converged to - for (Schedulable sched: schedulables) { + for (Schedulable sched : schedulables) { sched.setFairShare(computeShare(sched, right)); } } /** - * Compute the number of slots that would be used given a weight-to-slot - * ratio w2sRatio, for use in the computeFairShares algorithm as described - * in #{@link SchedulingAlgorithms#computeFairShares(Collection, double)}. + * Compute the number of slots that would be used given a weight-to-slot ratio + * w2sRatio, for use in the computeFairShares algorithm as described in # + * {@link SchedulingAlgorithms#computeFairShares(Collection, double)}. */ private static Resource resUsedWithWeightToResRatio(double w2sRatio, Collection schedulables) { Resource slotsTaken = Resources.createResource(0); - for (Schedulable sched: schedulables) { + for (Schedulable sched : schedulables) { Resource share = computeShare(sched, w2sRatio); Resources.addTo(slotsTaken, share); } @@ -210,8 +202,8 @@ class SchedulingAlgorithms { /** * Compute the resources assigned to a Schedulable given a particular - * res-to-slot ratio r2sRatio, for use in computeFairShares as described - * in #{@link SchedulingAlgorithms#computeFairShares(Collection, double)}. + * res-to-slot ratio r2sRatio, for use in computeFairShares as described in # + * {@link SchedulingAlgorithms#computeFairShares(Collection, double)}. */ private static Resource computeShare(Schedulable sched, double r2sRatio) { double share = sched.getWeight() * r2sRatio; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FifoSchedulingMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FifoSchedulingMode.java new file mode 100644 index 00000000000..f3c5e368dd6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FifoSchedulingMode.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes; + +import java.io.Serializable; +import java.util.Collection; +import java.util.Comparator; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingMode; + +import com.google.common.annotations.VisibleForTesting; + +public class FifoSchedulingMode extends SchedulingMode { + @VisibleForTesting + public static final String NAME = "FIFO"; + private FifoComparator comparator = new FifoComparator(); + + @Override + public String getName() { + return NAME; + } + + /** + * Compare Schedulables in order of priority and then submission time, as in + * the default FIFO scheduler in Hadoop. + */ + static class FifoComparator implements Comparator, Serializable { + private static final long serialVersionUID = -5905036205491177060L; + + @Override + public int compare(Schedulable s1, Schedulable s2) { + int res = s1.getPriority().compareTo(s2.getPriority()); + if (res == 0) { + res = (int) Math.signum(s1.getStartTime() - s2.getStartTime()); + } + if (res == 0) { + // In the rare case where jobs were submitted at the exact same time, + // compare them by name (which will be the JobID) to get a deterministic + // ordering, so we don't alternately launch tasks from different jobs. + res = s1.getName().compareTo(s2.getName()); + } + return res; + } + } + + @Override + public Comparator getComparator() { + return comparator; + } + + @Override + public void computeShares(Collection schedulables, + Resource totalResources) { + for (Schedulable sched : schedulables) { + sched.setFairShare(Resources.createResource(0)); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java index 05054b332e8..3a36fc15eaa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java @@ -24,6 +24,7 @@ import java.util.List; import junit.framework.Assert; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FairSchedulingMode; import org.junit.Before; import org.junit.Test; @@ -32,10 +33,12 @@ import org.junit.Test; */ public class TestComputeFairShares { private List scheds; + private SchedulingMode schedulingMode; @Before public void setUp() throws Exception { scheds = new ArrayList(); + schedulingMode = new FairSchedulingMode(); } /** @@ -48,7 +51,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50)); scheds.add(new FakeSchedulable(30)); scheds.add(new FakeSchedulable(20)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(10, 10, 10, 10); } @@ -65,7 +69,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50)); scheds.add(new FakeSchedulable(11)); scheds.add(new FakeSchedulable(3)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(13, 13, 11, 3); } @@ -83,7 +88,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(10, 20)); scheds.add(new FakeSchedulable(10, 0)); scheds.add(new FakeSchedulable(3, 2)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(20, 10, 7, 3); } @@ -97,7 +103,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50, 0, 1.0)); scheds.add(new FakeSchedulable(30, 0, 1.0)); scheds.add(new FakeSchedulable(20, 0, 0.5)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + schedulingMode.computeShares(scheds, + Resources.createResource(45)); verifyShares(20, 10, 10, 5); } @@ -114,7 +121,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(11, 0, 1.0)); scheds.add(new FakeSchedulable(30, 0, 1.0)); scheds.add(new FakeSchedulable(20, 0, 0.5)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + schedulingMode.computeShares(scheds, + Resources.createResource(45)); verifyShares(10, 11, 16, 8); } @@ -131,7 +139,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(11, 0, 1.0)); scheds.add(new FakeSchedulable(30, 5, 1.0)); scheds.add(new FakeSchedulable(20, 15, 0.5)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(45)); + schedulingMode.computeShares(scheds, + Resources.createResource(45)); verifyShares(10, 10, 10, 15); } @@ -146,7 +155,9 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50 * million)); scheds.add(new FakeSchedulable(30 * million)); scheds.add(new FakeSchedulable(20 * million)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40 * million)); + schedulingMode + .computeShares(scheds, + Resources.createResource(40 * million)); verifyShares(10 * million, 10 * million, 10 * million, 10 * million); } @@ -159,7 +170,8 @@ public class TestComputeFairShares { scheds.add(new FakeSchedulable(50)); scheds.add(new FakeSchedulable(30)); scheds.add(new FakeSchedulable(0)); - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(30)); + schedulingMode.computeShares(scheds, + Resources.createResource(30)); verifyShares(10, 10, 10, 0); } @@ -168,7 +180,8 @@ public class TestComputeFairShares { */ @Test public void testEmptyList() { - SchedulingAlgorithms.computeFairShares(scheds, Resources.createResource(40)); + schedulingMode.computeShares(scheds, + Resources.createResource(40)); verifyShares(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index a5cfadd2558..d7b32ec532c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedS import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -1325,7 +1326,7 @@ public class TestFairScheduler { FSSchedulerApp app2 = scheduler.applications.get(attId2); FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1"); - queue1.setSchedulingMode(SchedulingMode.FIFO); + queue1.setSchedulingMode(new FifoSchedulingMode()); scheduler.update(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingMode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingMode.java new file mode 100644 index 00000000000..71d43a317d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingMode.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FairSchedulingMode; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FifoSchedulingMode; +import org.junit.Test; + +public class TestSchedulingMode { + + @Test(timeout = 1000) + public void testParseSchedulingMode() throws AllocationConfigurationException { + + // Class name + SchedulingMode sm = SchedulingMode + .parse(FairSchedulingMode.class.getName()); + assertTrue("Invalid scheduler name", + sm.getName().equals(FairSchedulingMode.NAME)); + + // Canonical name + sm = SchedulingMode.parse(FairSchedulingMode.class + .getCanonicalName()); + assertTrue("Invalid scheduler name", + sm.getName().equals(FairSchedulingMode.NAME)); + + // Class + sm = SchedulingMode.getInstance(FairSchedulingMode.class); + assertTrue("Invalid scheduler name", + sm.getName().equals(FairSchedulingMode.NAME)); + + // Shortname - fair + sm = SchedulingMode.parse("fair"); + assertTrue("Invalid scheduler name", + sm.getName().equals(FairSchedulingMode.NAME)); + + // Shortname - fifo + sm = SchedulingMode.parse("fifo"); + assertTrue("Invalid scheduler name", + sm.getName().equals(FifoSchedulingMode.NAME)); + } +} From ca3e09bab5df35a1360deea1746fdd914c043814 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 26 Mar 2013 09:54:48 +0000 Subject: [PATCH 13/37] HDFS-4598. Fix the default value of ConcatSourcesParam and the WebHDFS doc. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461040 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/DistributedFileSystem.java | 7 ++-- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 39 +++++++++++++------ .../web/resources/ConcatSourcesParam.java | 19 ++++++++- .../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 15 ++----- .../hadoop/hdfs/web/resources/TestParam.java | 26 +++++++++++-- 6 files changed, 80 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8253761f9c7..aff2f516f22 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -448,6 +448,9 @@ Release 2.0.5-beta - UNRELEASED HDFS-4584. Skip TestNNWithQJM.testNewNamenodeTakesOverWriter() on Windows. (Arpit Agarwal via szetszwo) + HDFS-4598. Fix the default value of ConcatSourcesParam and the WebHDFS doc. + (szetszwo) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index d5d5d626ea8..3476f678708 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -312,13 +312,14 @@ public class DistributedFileSystem extends FileSystem { } /** - * Move blocks from srcs to trg - * and delete srcs afterwards - * RESTRICTION: all blocks should be the same size + * Move blocks from srcs to trg and delete srcs afterwards. + * The file block sizes must be the same. + * * @param trg existing file to append to * @param psrcs list of files (same block size, same replication) * @throws IOException */ + @Override public void concat(Path trg, Path [] psrcs) throws IOException { String [] srcs = new String [psrcs.length]; for(int i=0; i strPaths = new ArrayList(psrcs.length); - for(Path psrc : psrcs) { - strPaths.add(psrc.toUri().getPath()); - } - - String srcs = StringUtils.join(",", strPaths); - ConcatSourcesParam param = new ConcatSourcesParam(srcs); run(op, trg, param); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java index e6afbe3e4ec..b68c5f5b58e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java @@ -18,15 +18,28 @@ package org.apache.hadoop.hdfs.web.resources; +import org.apache.hadoop.fs.Path; + /** The concat source paths parameter. */ public class ConcatSourcesParam extends StringParam { /** Parameter name. */ public static final String NAME = "sources"; - public static final String DEFAULT = NULL; + public static final String DEFAULT = ""; private static final Domain DOMAIN = new Domain(NAME, null); + private static String paths2String(Path[] paths) { + if (paths == null || paths.length == 0) { + return ""; + } + final StringBuilder b = new StringBuilder(paths[0].toUri().getPath()); + for(int i = 1; i < paths.length; i++) { + b.append(',').append(paths[i].toUri().getPath()); + } + return b.toString(); + } + /** * Constructor. * @param str a string representation of the parameter value. @@ -35,6 +48,10 @@ public class ConcatSourcesParam extends StringParam { super(DOMAIN, str); } + public ConcatSourcesParam(Path[] paths) { + this(paths2String(paths)); + } + @Override public String getName() { return NAME; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm index 90f8dabce71..7735f8dafed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm @@ -109,7 +109,7 @@ WebHDFS REST API * {{{Append to a File}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append) - * {{{Concat File(s)}<<>>}} + * {{{Concatenate Files}<<>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat) * HTTP DELETE @@ -307,7 +307,7 @@ Content-Length: 0 * Submit a HTTP POST request. +--------------------------------- -curl -i -X POST "http://:/webhdfs/v1/?op=CONCAT&sources=" +curl -i -X POST "http://:/webhdfs/v1/?op=CONCAT&sources=" +--------------------------------- The client receives a response with zero content length: @@ -319,10 +319,6 @@ Content-Length: 0 [] - This REST API call is available as of Hadoop version 2.0.3. - Please note that is a comma seperated list of absolute paths. - (Example: sources=/test/file1,/test/file2,/test/file3) - See also: {{{Sources}<<>>}}, {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat @@ -1761,7 +1757,7 @@ var tokenProperties = *----------------+-------------------------------------------------------------------+ || Name | <<>> | *----------------+-------------------------------------------------------------------+ -|| Description | The comma seperated absolute paths used for concatenation. | +|| Description | A list of source paths. | *----------------+-------------------------------------------------------------------+ || Type | String | *----------------+-------------------------------------------------------------------+ @@ -1769,12 +1765,9 @@ var tokenProperties = *----------------+-------------------------------------------------------------------+ || Valid Values | A list of comma seperated absolute FileSystem paths without scheme and authority. | *----------------+-------------------------------------------------------------------+ -|| Syntax | See the note in {{Delegation}}. | +|| Syntax | Any string. | *----------------+-------------------------------------------------------------------+ - <> that sources are absolute FileSystem paths. - - See also: {{{Concat File(s)}<<>>}} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java index c228c1f2989..97223ab2fb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java @@ -17,18 +17,22 @@ */ package org.apache.hadoop.hdfs.web.resources; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.util.Arrays; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - public class TestParam { public static final Log LOG = LogFactory.getLog(TestParam.class); @@ -265,4 +269,20 @@ public class TestParam { UserParam userParam = new UserParam("a$"); assertNotNull(userParam.getValue()); } + + @Test + public void testConcatSourcesParam() { + final String[] strings = {"/", "/foo", "/bar"}; + for(int n = 0; n < strings.length; n++) { + final String[] sub = new String[n]; + final Path[] paths = new Path[n]; + for(int i = 0; i < paths.length; i++) { + paths[i] = new Path(sub[i] = strings[i]); + } + + final String expected = StringUtils.join(",", Arrays.asList(sub)); + final ConcatSourcesParam computed = new ConcatSourcesParam(paths); + Assert.assertEquals(expected, computed.getValue()); + } + } } From a0a844222dc1362cc46771841ab9036cd7373675 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 26 Mar 2013 14:16:13 +0000 Subject: [PATCH 14/37] MAPREDUCE-4875. coverage fixing for org.apache.hadoop.mapred (Aleksey Gorshkov via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461146 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../mapred/TestTaskAttemptListenerImpl.java | 29 +- .../apache/hadoop/mapred/JobEndNotifier.java | 75 +---- .../apache/hadoop/mapred/QueueManager.java | 94 +----- .../org/apache/hadoop/mapred/TaskLog.java | 86 ----- .../org/apache/hadoop/mapred/TaskStatus.java | 12 - .../org/apache/hadoop/mapred/TestClock.java | 38 +++ .../org/apache/hadoop/mapred/TestJobConf.java | 155 +++++++++ .../org/apache/hadoop/mapred/TestJobInfo.java | 56 ++++ .../hadoop/mapred/TestOldMethodsJobID.java | 168 ++++++++++ .../org/apache/hadoop/mapred/TestQueue.java | 238 ++++++++++++++ .../hadoop/mapred/TestSkipBadRecords.java | 61 ++++ .../org/apache/hadoop/mapred/TestTaskLog.java | 134 ++++++++ .../hadoop/mapred/TestTaskLogAppender.java | 71 +++++ .../src/test/resources/mapred-queues.xml | 69 ++++ .../org/apache/hadoop/mapred/TestIFile.java | 10 +- .../hadoop/mapred/TestMultiFileSplit.java | 31 +- .../hadoop/mapred/TestNetworkedJob.java | 298 +++++++++++++++++- .../mapred/TestQueueConfigurationParser.java | 74 +++++ .../mapred/TestStatisticsCollector.java | 25 ++ .../hadoop/mapred/TestTextInputFormat.java | 28 +- .../hadoop/mapred/TestTextOutputFormat.java | 64 +++- 22 files changed, 1529 insertions(+), 290 deletions(-) create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobInfo.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestOldMethodsJobID.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSkipBadRecords.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLogAppender.java create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/mapred-queues.xml create mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 5f8d73ff69f..e81da764d07 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -203,6 +203,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-4990. Construct debug strings conditionally in ShuffleHandler.Shuffle#sendMapOutput(). (kkambatl via tucu) + MAPREDUCE-4875. coverage fixing for org.apache.hadoop.mapred + (Aleksey Gorshkov via bobby) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java index b58ad347e67..d979d1e435d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java @@ -32,6 +32,8 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.Arrays; +import junit.framework.Assert; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.TypeConverter; @@ -78,7 +80,7 @@ public class TestTaskAttemptListenerImpl { } } - @Test + @Test (timeout=5000) public void testGetTask() throws IOException { AppContext appCtx = mock(AppContext.class); JobTokenSecretManager secret = mock(JobTokenSecretManager.class); @@ -136,9 +138,30 @@ public class TestTaskAttemptListenerImpl { assertTrue(result.shouldDie); listener.stop(); + + // test JVMID + JVMId jvmid = JVMId.forName("jvm_001_002_m_004"); + assertNotNull(jvmid); + try { + JVMId.forName("jvm_001_002_m_004_006"); + Assert.fail(); + } catch (IllegalArgumentException e) { + assertEquals(e.getMessage(), + "TaskId string : jvm_001_002_m_004_006 is not properly formed"); + } + } - @Test + @Test (timeout=5000) + public void testJVMId() { + + JVMId jvmid = new JVMId("test", 1, true, 2); + JVMId jvmid1 = JVMId.forName("jvm_test_0001_m_000002"); + // test compare methot should be the same + assertEquals(0, jvmid.compareTo(jvmid1)); + } + + @Test (timeout=10000) public void testGetMapCompletionEvents() throws IOException { TaskAttemptCompletionEvent[] empty = {}; TaskAttemptCompletionEvent[] taskEvents = { @@ -205,7 +228,7 @@ public class TestTaskAttemptListenerImpl { return tce; } - @Test + @Test (timeout=1000) public void testCommitWindow() throws IOException { SystemClock clock = new SystemClock(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java index c5fd9ca2289..5840730e54a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java @@ -19,8 +19,6 @@ package org.apache.hadoop.mapred; import java.io.IOException; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.DelayQueue; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; @@ -39,62 +37,7 @@ public class JobEndNotifier { private static final Log LOG = LogFactory.getLog(JobEndNotifier.class.getName()); - private static Thread thread; - private static volatile boolean running; - private static BlockingQueue queue = - new DelayQueue(); - - public static void startNotifier() { - running = true; - thread = new Thread( - new Runnable() { - public void run() { - try { - while (running) { - sendNotification(queue.take()); - } - } - catch (InterruptedException irex) { - if (running) { - LOG.error("Thread has ended unexpectedly", irex); - } - } - } - - private void sendNotification(JobEndStatusInfo notification) { - try { - int code = httpNotification(notification.getUri()); - if (code != 200) { - throw new IOException("Invalid response status code: " + code); - } - } - catch (IOException ioex) { - LOG.error("Notification failure [" + notification + "]", ioex); - if (notification.configureForRetry()) { - try { - queue.put(notification); - } - catch (InterruptedException iex) { - LOG.error("Notification queuing error [" + notification + "]", - iex); - } - } - } - catch (Exception ex) { - LOG.error("Notification failure [" + notification + "]", ex); - } - } - - } - - ); - thread.start(); - } - - public static void stopNotifier() { - running = false; - thread.interrupt(); - } + private static JobEndStatusInfo createNotification(JobConf conf, JobStatus status) { @@ -118,18 +61,6 @@ public class JobEndNotifier { return notification; } - public static void registerNotification(JobConf jobConf, JobStatus status) { - JobEndStatusInfo notification = createNotification(jobConf, status); - if (notification != null) { - try { - queue.put(notification); - } - catch (InterruptedException iex) { - LOG.error("Notification queuing failure [" + notification + "]", iex); - } - } - } - private static int httpNotification(String uri) throws IOException { URI url = new URI(uri, false); HttpClient m_client = new HttpClient(); @@ -194,10 +125,6 @@ public class JobEndNotifier { return retryInterval; } - public long getDelayTime() { - return delayTime; - } - public boolean configureForRetry() { boolean retry = false; if (getRetryAttempts() > 0) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java index 04ca5244be5..14934b92ef8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java @@ -40,7 +40,6 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.List; import java.net.URL; @@ -487,73 +486,8 @@ public class QueueManager { new QueueAclsInfo[queueAclsInfolist.size()]); } - /** - * ONLY FOR TESTING - Do not use in production code. - * This method is used for setting up of leafQueues only. - * We are not setting the hierarchy here. - * - * @param queues - */ - synchronized void setQueues(Queue[] queues) { - root.getChildren().clear(); - leafQueues.clear(); - allQueues.clear(); - - for (Queue queue : queues) { - root.addChild(queue); - } - //At this point we have root populated - //update data structures leafNodes. - leafQueues = getRoot().getLeafQueues(); - allQueues.putAll(getRoot().getInnerQueues()); - allQueues.putAll(leafQueues); - } - - /** - * Return an array of {@link JobQueueInfo} objects for the root - * queues configured in the system. - *

- * Root queues are queues that are at the top-most level in the - * hierarchy of queues in mapred-queues.xml, or they are the queues - * configured in the mapred.queue.names key in mapred-site.xml. - * - * @return array of JobQueueInfo objects for root level queues. - */ - - JobQueueInfo[] getRootQueues() { - List list = getRoot().getJobQueueInfo().getChildren(); - return list.toArray(new JobQueueInfo[list.size()]); - } - - /** - * Get the complete hierarchy of children for queue - * queueName - * - * @param queueName - * @return - */ - JobQueueInfo[] getChildQueues(String queueName) { - List list = - allQueues.get(queueName).getJobQueueInfo().getChildren(); - if (list != null) { - return list.toArray(new JobQueueInfo[list.size()]); - } else { - return new JobQueueInfo[0]; - } - } - - /** - * Used only for testing purposes . - * This method is unstable as refreshQueues would leave this - * data structure in unstable state. - * - * @param queueName - * @return - */ - Queue getQueue(String queueName) { - return this.allQueues.get(queueName); - } - + + /** * Return if ACLs are enabled for the Map/Reduce system @@ -573,29 +507,7 @@ public class QueueManager { return root; } - /** - * Returns the specific queue ACL for the given queue. - * Returns null if the given queue does not exist or the acl is not - * configured for that queue. - * If acls are disabled(mapreduce.cluster.acls.enabled set to false), returns - * ACL with all users. - */ - synchronized AccessControlList getQueueACL(String queueName, - QueueACL qACL) { - if (areAclsEnabled) { - Queue q = leafQueues.get(queueName); - if (q != null) { - return q.getAcls().get(toFullPropertyName( - queueName, qACL.getAclName())); - } - else { - LOG.warn("Queue " + queueName + " is not present."); - return null; - } - } - return new AccessControlList("*"); - } - + /** * Dumps the configuration of hierarchy of queues * @param out the writer object to which dump is written diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java index 12d686733fa..082d2172d8e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java @@ -386,73 +386,6 @@ public class TaskLog { return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024; } - /** - * Wrap a command in a shell to capture stdout and stderr to files. - * If the tailLength is 0, the entire output will be saved. - * @param cmd The command and the arguments that should be run - * @param stdoutFilename The filename that stdout should be saved to - * @param stderrFilename The filename that stderr should be saved to - * @param tailLength The length of the tail to be saved. - * @return the modified command that should be run - */ - public static List captureOutAndError(List cmd, - File stdoutFilename, - File stderrFilename, - long tailLength - ) throws IOException { - return captureOutAndError(null, cmd, stdoutFilename, - stderrFilename, tailLength, false); - } - - /** - * Wrap a command in a shell to capture stdout and stderr to files. - * Setup commands such as setting memory limit can be passed which - * will be executed before exec. - * If the tailLength is 0, the entire output will be saved. - * @param setup The setup commands for the execed process. - * @param cmd The command and the arguments that should be run - * @param stdoutFilename The filename that stdout should be saved to - * @param stderrFilename The filename that stderr should be saved to - * @param tailLength The length of the tail to be saved. - * @return the modified command that should be run - */ - public static List captureOutAndError(List setup, - List cmd, - File stdoutFilename, - File stderrFilename, - long tailLength - ) throws IOException { - return captureOutAndError(setup, cmd, stdoutFilename, stderrFilename, - tailLength, false); - } - - /** - * Wrap a command in a shell to capture stdout and stderr to files. - * Setup commands such as setting memory limit can be passed which - * will be executed before exec. - * If the tailLength is 0, the entire output will be saved. - * @param setup The setup commands for the execed process. - * @param cmd The command and the arguments that should be run - * @param stdoutFilename The filename that stdout should be saved to - * @param stderrFilename The filename that stderr should be saved to - * @param tailLength The length of the tail to be saved. - * @param pidFileName The name of the pid-file. pid-file's usage is deprecated - * @return the modified command that should be run - * - * @deprecated pidFiles are no more used. Instead pid is exported to - * env variable JVM_PID. - */ - @Deprecated - public static List captureOutAndError(List setup, - List cmd, - File stdoutFilename, - File stderrFilename, - long tailLength, - String pidFileName - ) throws IOException { - return captureOutAndError(setup, cmd, stdoutFilename, stderrFilename, - tailLength, false); - } /** * Wrap a command in a shell to capture stdout and stderr to files. @@ -607,25 +540,6 @@ public class TaskLog { return command.toString(); } - /** - * Wrap a command in a shell to capture debug script's - * stdout and stderr to debugout. - * @param cmd The command and the arguments that should be run - * @param debugoutFilename The filename that stdout and stderr - * should be saved to. - * @return the modified command that should be run - * @throws IOException - */ - public static List captureDebugOut(List cmd, - File debugoutFilename - ) throws IOException { - String debugout = FileUtil.makeShellPath(debugoutFilename); - List result = new ArrayList(3); - result.add(bashCommand); - result.add("-c"); - result.add(buildDebugScriptCommandLine(cmd, debugout)); - return result; - } /** * Method to return the location of user log directory. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskStatus.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskStatus.java index 3fd87021ce7..7bd5eb90616 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskStatus.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskStatus.java @@ -523,17 +523,5 @@ public abstract class TaskStatus implements Writable, Cloneable { return (isMap) ? new MapTaskStatus() : new ReduceTaskStatus(); } - static TaskStatus readTaskStatus(DataInput in) throws IOException { - boolean isMap = in.readBoolean(); - TaskStatus taskStatus = createTaskStatus(isMap); - taskStatus.readFields(in); - return taskStatus; - } - - static void writeTaskStatus(DataOutput out, TaskStatus taskStatus) - throws IOException { - out.writeBoolean(taskStatus.getIsMap()); - taskStatus.write(out); - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java new file mode 100644 index 00000000000..7081cd5a154 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java @@ -0,0 +1,38 @@ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import org.junit.Test; + +import static org.junit.Assert.*; +/** + * test Clock class + * + */ +public class TestClock { + + @Test (timeout=1000) + public void testClock(){ + Clock clock= new Clock(); + long templateTime=System.currentTimeMillis(); + long time=clock.getTime(); + assertEquals(templateTime, time,30); + + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java new file mode 100644 index 00000000000..646a747aa5d --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.util.regex.Pattern; +import static org.junit.Assert.*; + +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +/** + * test JobConf + * + */ +public class TestJobConf { + + /** + * test getters and setters of JobConf + */ + @SuppressWarnings("deprecation") + @Test (timeout=5000) + public void testJobConf() { + JobConf conf = new JobConf(); + // test default value + Pattern pattern = conf.getJarUnpackPattern(); + assertEquals(Pattern.compile("(?:classes/|lib/).*").toString(), + pattern.toString()); + // default value + assertFalse(conf.getKeepFailedTaskFiles()); + conf.setKeepFailedTaskFiles(true); + assertTrue(conf.getKeepFailedTaskFiles()); + + // default value + assertNull(conf.getKeepTaskFilesPattern()); + conf.setKeepTaskFilesPattern("123454"); + assertEquals("123454", conf.getKeepTaskFilesPattern()); + + // default value + assertNotNull(conf.getWorkingDirectory()); + conf.setWorkingDirectory(new Path("test")); + assertTrue(conf.getWorkingDirectory().toString().endsWith("test")); + + // default value + assertEquals(1, conf.getNumTasksToExecutePerJvm()); + + // default value + assertNull(conf.getKeyFieldComparatorOption()); + conf.setKeyFieldComparatorOptions("keySpec"); + assertEquals("keySpec", conf.getKeyFieldComparatorOption()); + + // default value + assertFalse(conf.getUseNewReducer()); + conf.setUseNewReducer(true); + assertTrue(conf.getUseNewReducer()); + + // default + assertTrue(conf.getMapSpeculativeExecution()); + assertTrue(conf.getReduceSpeculativeExecution()); + assertTrue(conf.getSpeculativeExecution()); + conf.setReduceSpeculativeExecution(false); + assertTrue(conf.getSpeculativeExecution()); + + conf.setMapSpeculativeExecution(false); + assertFalse(conf.getSpeculativeExecution()); + assertFalse(conf.getMapSpeculativeExecution()); + assertFalse(conf.getReduceSpeculativeExecution()); + + conf.setSessionId("ses"); + assertEquals("ses", conf.getSessionId()); + + assertEquals(3, conf.getMaxTaskFailuresPerTracker()); + conf.setMaxTaskFailuresPerTracker(2); + assertEquals(2, conf.getMaxTaskFailuresPerTracker()); + + assertEquals(0, conf.getMaxMapTaskFailuresPercent()); + conf.setMaxMapTaskFailuresPercent(50); + assertEquals(50, conf.getMaxMapTaskFailuresPercent()); + + assertEquals(0, conf.getMaxReduceTaskFailuresPercent()); + conf.setMaxReduceTaskFailuresPercent(70); + assertEquals(70, conf.getMaxReduceTaskFailuresPercent()); + + // by default + assertEquals(JobPriority.NORMAL.name(), conf.getJobPriority().name()); + conf.setJobPriority(JobPriority.HIGH); + assertEquals(JobPriority.HIGH.name(), conf.getJobPriority().name()); + + assertNull(conf.getJobSubmitHostName()); + conf.setJobSubmitHostName("hostname"); + assertEquals("hostname", conf.getJobSubmitHostName()); + + // default + assertNull(conf.getJobSubmitHostAddress()); + conf.setJobSubmitHostAddress("ww"); + assertEquals("ww", conf.getJobSubmitHostAddress()); + + // default value + assertFalse(conf.getProfileEnabled()); + conf.setProfileEnabled(true); + assertTrue(conf.getProfileEnabled()); + + // default value + assertEquals(conf.getProfileTaskRange(true).toString(), "0-2"); + assertEquals(conf.getProfileTaskRange(false).toString(), "0-2"); + conf.setProfileTaskRange(true, "0-3"); + assertEquals(conf.getProfileTaskRange(false).toString(), "0-2"); + assertEquals(conf.getProfileTaskRange(true).toString(), "0-3"); + + // default value + assertNull(conf.getMapDebugScript()); + conf.setMapDebugScript("mDbgScript"); + assertEquals("mDbgScript", conf.getMapDebugScript()); + + // default value + assertNull(conf.getReduceDebugScript()); + conf.setReduceDebugScript("rDbgScript"); + assertEquals("rDbgScript", conf.getReduceDebugScript()); + + // default value + assertNull(conf.getJobLocalDir()); + + assertEquals("default", conf.getQueueName()); + conf.setQueueName("qname"); + assertEquals("qname", conf.getQueueName()); + + assertEquals(1, conf.computeNumSlotsPerMap(100L)); + assertEquals(1, conf.computeNumSlotsPerReduce(100L)); + + conf.setMemoryForMapTask(100 * 1000); + assertEquals(1000, conf.computeNumSlotsPerMap(100L)); + conf.setMemoryForReduceTask(1000 * 1000); + assertEquals(1000, conf.computeNumSlotsPerReduce(1000L)); + + assertEquals(-1, conf.getMaxPhysicalMemoryForTask()); + assertEquals("The variable key is no longer used.", + JobConf.deprecatedString("key")); + + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobInfo.java new file mode 100644 index 00000000000..84c5850d179 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobInfo.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * test class JobInfo + * + * + */ +public class TestJobInfo { + @Test (timeout=5000) + public void testJobInfo() throws IOException { + JobID jid = new JobID("001", 1); + Text user = new Text("User"); + Path path = new Path("/tmp/test"); + JobInfo info = new JobInfo(jid, user, path); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + info.write(new DataOutputStream(out)); + + JobInfo copyinfo = new JobInfo(); + copyinfo.readFields(new DataInputStream(new ByteArrayInputStream(out + .toByteArray()))); + assertEquals(info.getJobID().toString(), copyinfo.getJobID().toString()); + assertEquals(info.getJobSubmitDir().getName(), copyinfo.getJobSubmitDir() + .getName()); + assertEquals(info.getUser().toString(), copyinfo.getUser().toString()); + + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestOldMethodsJobID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestOldMethodsJobID.java new file mode 100644 index 00000000000..3b0311d336a --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestOldMethodsJobID.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import org.apache.hadoop.mapred.TaskCompletionEvent.Status; +import org.apache.hadoop.mapreduce.TaskType; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Test deprecated methods + * + */ +public class TestOldMethodsJobID { + + /** + * test deprecated methods of TaskID + * @throws IOException + */ + @SuppressWarnings("deprecation") + @Test (timeout=5000) + public void testDepricatedMethods() throws IOException { + JobID jid = new JobID(); + TaskID test = new TaskID(jid, true, 1); + assertEquals(test.getTaskType(), TaskType.MAP); + test = new TaskID(jid, false, 1); + assertEquals(test.getTaskType(), TaskType.REDUCE); + + test = new TaskID("001", 1, false, 1); + assertEquals(test.getTaskType(), TaskType.REDUCE); + test = new TaskID("001", 1, true, 1); + assertEquals(test.getTaskType(), TaskType.MAP); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + test.write(new DataOutputStream(out)); + TaskID ti = TaskID.read(new DataInputStream(new ByteArrayInputStream(out + .toByteArray()))); + assertEquals(ti.toString(), test.toString()); + assertEquals("task_001_0001_m_000002", + TaskID.getTaskIDsPattern("001", 1, true, 2)); + assertEquals("task_003_0001_m_000004", + TaskID.getTaskIDsPattern("003", 1, TaskType.MAP, 4)); + assertEquals("003_0001_m_000004", + TaskID.getTaskIDsPatternWOPrefix("003", 1, TaskType.MAP, 4).toString()); + + } + + /** + * test JobID + * @throws IOException + */ + @SuppressWarnings("deprecation") + @Test (timeout=5000) + public void testJobID() throws IOException{ + JobID jid = new JobID("001",2); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + jid.write(new DataOutputStream(out)); + assertEquals(jid,JobID.read(new DataInputStream(new ByteArrayInputStream(out.toByteArray())))); + assertEquals("job_001_0001",JobID.getJobIDsPattern("001",1)); + } + /** + * test deprecated methods of TaskCompletionEvent + */ + @SuppressWarnings("deprecation") + @Test (timeout=5000) + public void testTaskCompletionEvent() { + TaskAttemptID taid = new TaskAttemptID("001", 1, TaskType.REDUCE, 2, 3); + TaskCompletionEvent template = new TaskCompletionEvent(12, taid, 13, true, + Status.SUCCEEDED, "httptracker"); + TaskCompletionEvent testEl = TaskCompletionEvent.downgrade(template); + testEl.setTaskAttemptId(taid); + testEl.setTaskTrackerHttp("httpTracker"); + + testEl.setTaskId("attempt_001_0001_m_000002_04"); + assertEquals("attempt_001_0001_m_000002_4",testEl.getTaskId()); + + testEl.setTaskStatus(Status.OBSOLETE); + assertEquals(Status.OBSOLETE.toString(), testEl.getStatus().toString()); + + testEl.setTaskRunTime(20); + assertEquals(testEl.getTaskRunTime(), 20); + testEl.setEventId(16); + assertEquals(testEl.getEventId(), 16); + + } + + /** + * test depricated methods of JobProfile + * @throws IOException + */ + @SuppressWarnings("deprecation") + @Test (timeout=5000) + public void testJobProfile() throws IOException { + + JobProfile profile = new JobProfile("user", "job_001_03", "jobFile", "uri", + "name"); + assertEquals("job_001_0003", profile.getJobId()); + assertEquals("default", profile.getQueueName()); + // serialization test + ByteArrayOutputStream out = new ByteArrayOutputStream(); + profile.write(new DataOutputStream(out)); + + JobProfile profile2 = new JobProfile(); + profile2.readFields(new DataInputStream(new ByteArrayInputStream(out + .toByteArray()))); + + assertEquals(profile2.name, profile.name); + assertEquals(profile2.jobFile, profile.jobFile); + assertEquals(profile2.queueName, profile.queueName); + assertEquals(profile2.url, profile.url); + assertEquals(profile2.user, profile.user); + } + /** + * test TaskAttemptID + */ + @SuppressWarnings( "deprecation" ) + @Test (timeout=5000) + public void testTaskAttemptID (){ + TaskAttemptID task = new TaskAttemptID("001",2,true,3,4); + assertEquals("attempt_001_0002_m_000003_4", TaskAttemptID.getTaskAttemptIDsPattern("001", 2, true, 3, 4)); + assertEquals("task_001_0002_m_000003", task.getTaskID().toString()); + assertEquals("attempt_001_0001_r_000002_3",TaskAttemptID.getTaskAttemptIDsPattern("001", 1, TaskType.REDUCE, 2, 3)); + assertEquals("001_0001_m_000001_2", TaskAttemptID.getTaskAttemptIDsPatternWOPrefix("001",1, TaskType.MAP, 1, 2).toString()); + + } + + /** + * test Reporter.NULL + * + */ + + @Test (timeout=5000) + public void testReporter(){ + Reporter nullReporter=Reporter.NULL; + assertNull(nullReporter.getCounter(null)); + assertNull(nullReporter.getCounter("group", "name")); + // getInputSplit method removed + try{ + assertNull(nullReporter.getInputSplit()); + }catch(UnsupportedOperationException e){ + assertEquals( "NULL reporter has no input",e.getMessage()); + } + assertEquals(0,nullReporter.getProgress(),0.01); + + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java new file mode 100644 index 00000000000..5fe5c3ac0cd --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java @@ -0,0 +1,238 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.MRConfig; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Test; + +import static junit.framework.Assert.*; +import static org.mockito.Mockito.*; + +/** + * TestCounters checks the sanity and recoverability of Queue + */ +public class TestQueue { + + /** + * test QueueManager + * configuration from file + * + * @throws IOException + */ + @Test (timeout=5000) + public void testQueue() throws IOException { + File f = null; + try { + f = writeFile(); + + QueueManager manager = new QueueManager(f.getCanonicalPath(), true); + manager.setSchedulerInfo("first", "queueInfo"); + manager.setSchedulerInfo("second", "queueInfoqueueInfo"); + Queue root = manager.getRoot(); + assertTrue(root.getChildren().size() == 2); + Iterator iterator = root.getChildren().iterator(); + Queue firstSubQueue = iterator.next(); + assertTrue(firstSubQueue.getName().equals("first")); + assertEquals( + firstSubQueue.getAcls().get("mapred.queue.first.acl-submit-job") + .toString(), + "Users [user1, user2] and members of the groups [group1, group2] are allowed"); + Queue secondSubQueue = iterator.next(); + assertTrue(secondSubQueue.getName().equals("second")); + assertEquals(secondSubQueue.getProperties().getProperty("key"), "value"); + assertEquals(secondSubQueue.getProperties().getProperty("key1"), "value1"); + // test status + assertEquals(firstSubQueue.getState().getStateName(), "running"); + assertEquals(secondSubQueue.getState().getStateName(), "stopped"); + + Set template = new HashSet(); + template.add("first"); + template.add("second"); + assertEquals(manager.getLeafQueueNames(), template); + + // test user access + + UserGroupInformation mockUGI = mock(UserGroupInformation.class); + when(mockUGI.getShortUserName()).thenReturn("user1"); + String[] groups = { "group1" }; + when(mockUGI.getGroupNames()).thenReturn(groups); + assertTrue(manager.hasAccess("first", QueueACL.SUBMIT_JOB, mockUGI)); + assertFalse(manager.hasAccess("second", QueueACL.SUBMIT_JOB, mockUGI)); + assertFalse(manager.hasAccess("first", QueueACL.ADMINISTER_JOBS, mockUGI)); + when(mockUGI.getShortUserName()).thenReturn("user3"); + assertTrue(manager.hasAccess("first", QueueACL.ADMINISTER_JOBS, mockUGI)); + + QueueAclsInfo[] qai = manager.getQueueAcls(mockUGI); + assertEquals(qai.length, 1); + // test refresh queue + manager.refreshQueues(getConfiguration(), null); + + iterator = root.getChildren().iterator(); + Queue firstSubQueue1 = iterator.next(); + Queue secondSubQueue1 = iterator.next(); + // tets equal method + assertTrue(firstSubQueue.equals(firstSubQueue1)); + assertEquals(firstSubQueue1.getState().getStateName(), "running"); + assertEquals(secondSubQueue1.getState().getStateName(), "stopped"); + + assertEquals(firstSubQueue1.getSchedulingInfo(), "queueInfo"); + assertEquals(secondSubQueue1.getSchedulingInfo(), "queueInfoqueueInfo"); + + // test JobQueueInfo + assertEquals(firstSubQueue.getJobQueueInfo().getQueueName(), "first"); + assertEquals(firstSubQueue.getJobQueueInfo().getQueueState(), "running"); + assertEquals(firstSubQueue.getJobQueueInfo().getSchedulingInfo(), + "queueInfo"); + assertEquals(secondSubQueue.getJobQueueInfo().getChildren().size(), 0); + // test + assertEquals(manager.getSchedulerInfo("first"), "queueInfo"); + assertEquals(manager.getJobQueueInfos()[0].getQueueName(), secondSubQueue + .getJobQueueInfo().getQueueName()); + assertEquals(manager.getJobQueueInfos()[1].getQueueName(), firstSubQueue + .getJobQueueInfo().getQueueName()); + // test getJobQueueInfoMapping + assertEquals( + manager.getJobQueueInfoMapping().get("first").getQueueName(), "first"); + // test dumpConfiguration + Writer writer = new StringWriter(); + + Configuration conf = getConfiguration(); + conf.unset(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY); + QueueManager.dumpConfiguration(writer, f.getAbsolutePath(), conf); + String result = writer.toString(); + assertTrue(result + .indexOf("\"name\":\"first\",\"state\":\"running\",\"acl_submit_job\":\"user1,user2 group1,group2\",\"acl_administer_jobs\":\"user3,user4 group3,group4\",\"properties\":[],\"children\":[]") > 0); + + writer = new StringWriter(); + QueueManager.dumpConfiguration(writer, conf); + result = writer.toString(); + assertEquals( + "{\"queues\":[{\"name\":\"default\",\"state\":\"running\",\"acl_submit_job\":\"*\",\"acl_administer_jobs\":\"*\",\"properties\":[],\"children\":[]},{\"name\":\"q1\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[],\"children\":[{\"name\":\"q1:q2\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[{\"key\":\"capacity\",\"value\":\"20\"},{\"key\":\"user-limit\",\"value\":\"30\"}],\"children\":[]}]}]}", + result); + // test constructor QueueAclsInfo + QueueAclsInfo qi = new QueueAclsInfo(); + assertNull(qi.getQueueName()); + + } finally { + if (f != null) { + f.delete(); + } + } + } + + private Configuration getConfiguration() { + Configuration conf = new Configuration(); + conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY, + "first,second"); + conf.set(QueueManager.QUEUE_CONF_PROPERTY_NAME_PREFIX + + "first.acl-submit-job", "user1,user2 group1,group2"); + conf.set(MRConfig.MR_ACLS_ENABLED, "true"); + conf.set(QueueManager.QUEUE_CONF_PROPERTY_NAME_PREFIX + "first.state", + "running"); + conf.set(QueueManager.QUEUE_CONF_PROPERTY_NAME_PREFIX + "second.state", + "stopped"); + return conf; + } + + @Test (timeout=5000) + public void testDefaultConfig() { + QueueManager manager = new QueueManager(true); + assertEquals(manager.getRoot().getChildren().size(), 2); + } + + /** + * test for Qmanager with empty configuration + * + * @throws IOException + */ + + @Test (timeout=5000) + public void test2Queue() throws IOException { + Configuration conf = getConfiguration(); + + QueueManager manager = new QueueManager(conf); + manager.setSchedulerInfo("first", "queueInfo"); + manager.setSchedulerInfo("second", "queueInfoqueueInfo"); + + Queue root = manager.getRoot(); + + // test children queues + assertTrue(root.getChildren().size() == 2); + Iterator iterator = root.getChildren().iterator(); + Queue firstSubQueue = iterator.next(); + assertTrue(firstSubQueue.getName().equals("first")); + assertEquals( + firstSubQueue.getAcls().get("mapred.queue.first.acl-submit-job") + .toString(), + "Users [user1, user2] and members of the groups [group1, group2] are allowed"); + Queue secondSubQueue = iterator.next(); + assertTrue(secondSubQueue.getName().equals("second")); + + assertEquals(firstSubQueue.getState().getStateName(), "running"); + assertEquals(secondSubQueue.getState().getStateName(), "stopped"); + assertTrue(manager.isRunning("first")); + assertFalse(manager.isRunning("second")); + + assertEquals(firstSubQueue.getSchedulingInfo(), "queueInfo"); + assertEquals(secondSubQueue.getSchedulingInfo(), "queueInfoqueueInfo"); +// test leaf queue + Set template = new HashSet(); + template.add("first"); + template.add("second"); + assertEquals(manager.getLeafQueueNames(), template); + + + } +/** + * write cofiguration + * @return + * @throws IOException + */ + private File writeFile() throws IOException { + + File f = null; + f = File.createTempFile("tst", "xml"); + BufferedWriter out = new BufferedWriter(new FileWriter(f)); + String properties = ""; + out.write(""); + out.newLine(); + out.write("firstuser1,user2 group1,group2user3,user4 group3,group4running"); + out.newLine(); + out.write("secondu1,u2 g1,g2" + + properties + "stopped"); + out.newLine(); + out.write(""); + out.flush(); + out.close(); + return f; + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSkipBadRecords.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSkipBadRecords.java new file mode 100644 index 00000000000..01dca6f37db --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSkipBadRecords.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * test SkipBadRecords + * + * + */ +public class TestSkipBadRecords { + @Test (timeout=5000) + public void testSkipBadRecords() { + // test default values + Configuration conf = new Configuration(); + assertEquals(2, SkipBadRecords.getAttemptsToStartSkipping(conf)); + assertTrue(SkipBadRecords.getAutoIncrMapperProcCount(conf)); + assertTrue(SkipBadRecords.getAutoIncrReducerProcCount(conf)); + assertEquals(0, SkipBadRecords.getMapperMaxSkipRecords(conf)); + assertEquals(0, SkipBadRecords.getReducerMaxSkipGroups(conf), 0); + assertNull(SkipBadRecords.getSkipOutputPath(conf)); + + // test setters + SkipBadRecords.setAttemptsToStartSkipping(conf, 5); + SkipBadRecords.setAutoIncrMapperProcCount(conf, false); + SkipBadRecords.setAutoIncrReducerProcCount(conf, false); + SkipBadRecords.setMapperMaxSkipRecords(conf, 6L); + SkipBadRecords.setReducerMaxSkipGroups(conf, 7L); + JobConf jc= new JobConf(); + SkipBadRecords.setSkipOutputPath(jc, new Path("test")); + + // test getters + assertEquals(5, SkipBadRecords.getAttemptsToStartSkipping(conf)); + assertFalse(SkipBadRecords.getAutoIncrMapperProcCount(conf)); + assertFalse(SkipBadRecords.getAutoIncrReducerProcCount(conf)); + assertEquals(6L, SkipBadRecords.getMapperMaxSkipRecords(conf)); + assertEquals(7L, SkipBadRecords.getReducerMaxSkipGroups(conf), 0); + assertEquals("test",SkipBadRecords.getSkipOutputPath(jc).toString()); + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java new file mode 100644 index 00000000000..328fc64048e --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.mapred; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.mapred.TaskLog.LogName; +import org.apache.hadoop.mapreduce.MRJobConfig; +import org.junit.Test; + +import static junit.framework.Assert.*; +import static org.mockito.Mockito.*; + +/** + * TestCounters checks the sanity and recoverability of Queue + */ +public class TestTaskLog { + + /** + * test TaskAttemptID + * + * @throws IOException + */ + @Test (timeout=50000) + public void testTaskLog() throws IOException { + // test TaskLog + System.setProperty(MRJobConfig.TASK_LOG_DIR, "testString"); + assertEquals(TaskLog.getMRv2LogDir(), "testString"); + TaskAttemptID taid = mock(TaskAttemptID.class); + JobID jid = new JobID("job", 1); + + when(taid.getJobID()).thenReturn(jid); + when(taid.toString()).thenReturn("JobId"); + + File f = TaskLog.getTaskLogFile(taid, true, LogName.STDOUT); + assertTrue(f.getAbsolutePath().endsWith("testString/stdout")); + + // test getRealTaskLogFileLocation + + File indexFile = TaskLog.getIndexFile(taid, true); + if (!indexFile.getParentFile().exists()) { + indexFile.getParentFile().mkdirs(); + } + indexFile.delete(); + indexFile.createNewFile(); + + TaskLog.syncLogs("location", taid, true); + + assertTrue(indexFile.getAbsolutePath().endsWith( + "userlogs/job_job_0001/JobId.cleanup/log.index")); + + f = TaskLog.getRealTaskLogFileLocation(taid, true, LogName.DEBUGOUT); + if (f != null) { + assertTrue(f.getAbsolutePath().endsWith("location/debugout")); + FileUtils.copyFile(indexFile, f); + } + // test obtainLogDirOwner + assertTrue(TaskLog.obtainLogDirOwner(taid).length() > 0); + // test TaskLog.Reader + assertTrue(readTaskLog(TaskLog.LogName.DEBUGOUT, taid, true).length() > 0); + + } + + public String readTaskLog(TaskLog.LogName filter, + org.apache.hadoop.mapred.TaskAttemptID taskId, boolean isCleanup) + throws IOException { + // string buffer to store task log + StringBuffer result = new StringBuffer(); + int res; + + // reads the whole tasklog into inputstream + InputStream taskLogReader = new TaskLog.Reader(taskId, filter, 0, -1, + isCleanup); + // construct string log from inputstream. + byte[] b = new byte[65536]; + while (true) { + res = taskLogReader.read(b); + if (res > 0) { + result.append(new String(b)); + } else { + break; + } + } + taskLogReader.close(); + + // trim the string and return it + String str = result.toString(); + str = str.trim(); + return str; + } + + /** + * test without TASK_LOG_DIR + * + * @throws IOException + */ + @Test (timeout=50000) + public void testTaskLogWithoutTaskLogDir() throws IOException { + // TaskLog tasklog= new TaskLog(); + System.clearProperty(MRJobConfig.TASK_LOG_DIR); + + // test TaskLog + + assertEquals(TaskLog.getMRv2LogDir(), null); + TaskAttemptID taid = mock(TaskAttemptID.class); + JobID jid = new JobID("job", 1); + + when(taid.getJobID()).thenReturn(jid); + when(taid.toString()).thenReturn("JobId"); + + File f = TaskLog.getTaskLogFile(taid, true, LogName.STDOUT); + assertTrue(f.getAbsolutePath().endsWith("stdout")); + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLogAppender.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLogAppender.java new file mode 100644 index 00000000000..89cf07c6824 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLogAppender.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.mapred; + +import java.io.StringWriter; +import java.io.Writer; + +import org.apache.log4j.Category; +import org.apache.log4j.Layout; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.apache.log4j.Priority; +import org.apache.log4j.spi.LoggingEvent; +import org.junit.Test; +import static org.junit.Assert.*; + +public class TestTaskLogAppender { +/** + * test TaskLogAppender + */ + @SuppressWarnings("deprecation") + @Test (timeout=5000) + public void testTaskLogAppender(){ + TaskLogAppender appender= new TaskLogAppender(); + + System.setProperty(TaskLogAppender.TASKID_PROPERTY,"attempt_01_02_m03_04_001"); + System.setProperty(TaskLogAppender.LOGSIZE_PROPERTY, "1003"); + appender.activateOptions(); + assertEquals(appender.getTaskId(), "attempt_01_02_m03_04_001"); + assertEquals(appender.getTotalLogFileSize(),1000); + assertEquals(appender.getIsCleanup(),false); + + // test writer + Writer writer= new StringWriter(); + appender.setWriter(writer); + Layout layout = new PatternLayout("%-5p [%t]: %m%n"); + appender.setLayout(layout); + Category logger= Logger.getLogger(getClass().getName()); + LoggingEvent event = new LoggingEvent("fqnOfCategoryClass", logger, Priority.INFO, "message", new Throwable()); + appender.append(event); + appender.flush() ; + appender.close(); + assertTrue(writer.toString().length()>0); + + // test cleanup should not changed + appender= new TaskLogAppender(); + appender.setIsCleanup(true); + appender.activateOptions(); + assertEquals(appender.getIsCleanup(),true); + + + } + +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/mapred-queues.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/mapred-queues.xml new file mode 100644 index 00000000000..2d8a9d04a0c --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/mapred-queues.xml @@ -0,0 +1,69 @@ + + + + + + + + + + default + + + + + + + running + + + * + + + * + + + + + q1 + + q2 + + + + + + + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java index 3514b15becf..e3c7253afcb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java @@ -26,12 +26,12 @@ import org.apache.hadoop.io.compress.DefaultCodec; import org.apache.hadoop.io.compress.GzipCodec; import org.junit.Test; - +import static org.junit.Assert.*; public class TestIFile { @Test /** - * Create an IFile.Writer using GzipCodec since this codec does not + * Create an IFile.Writer using GzipCodec since this code does not * have a compressor when run via the tests (ie no native libraries). */ public void testIFileWriterWithCodec() throws Exception { @@ -63,5 +63,11 @@ public class TestIFile { IFile.Reader reader = new IFile.Reader(conf, rfs, path, codec, null); reader.close(); + + // test check sum + byte[] ab= new byte[100]; + int readed= reader.checksumIn.readWithChecksum(ab, 0, ab.length); + assertEquals( readed,reader.checksumIn.getChecksum().length); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java index af0f399549a..16ff6af9271 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java @@ -21,13 +21,20 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.Arrays; import junit.framework.TestCase; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; - +/** + * + * test MultiFileSplit class + */ public class TestMultiFileSplit extends TestCase{ public void testReadWrite() throws Exception { @@ -58,4 +65,26 @@ public class TestMultiFileSplit extends TestCase{ assertTrue(Arrays.equals(split.getLengths(), readSplit.getLengths())); System.out.println(split.toString()); } + + /** + * test method getLocations + * @throws IOException + */ + public void testgetLocations() throws IOException{ + JobConf job= new JobConf(); + + File tmpFile = File.createTempFile("test","txt"); + tmpFile.createNewFile(); + OutputStream out=new FileOutputStream(tmpFile); + out.write("tempfile".getBytes()); + out.flush(); + out.close(); + Path[] path= {new Path(tmpFile.getAbsolutePath())}; + long[] lengths = {100}; + + MultiFileSplit split = new MultiFileSplit(job,path,lengths); + String [] locations= split.getLocations(); + assertTrue(locations.length==1); + assertEquals(locations[0], "localhost"); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java index 37aa7b24a3c..8d89fb262f3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java @@ -18,24 +18,37 @@ package org.apache.hadoop.mapred; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.*; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutput; +import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.io.PrintStream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo; +import org.apache.hadoop.mapred.JobClient.NetworkedJob; +import org.apache.hadoop.mapred.JobClient.TaskStatusFilter; import org.apache.hadoop.mapred.lib.IdentityMapper; import org.apache.hadoop.mapred.lib.IdentityReducer; +import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.yarn.YarnException; import org.junit.Test; - +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; public class TestNetworkedJob { private static String TEST_ROOT_DIR = new File(System.getProperty( @@ -44,8 +57,7 @@ public class TestNetworkedJob { private static Path inFile = new Path(testDir, "in"); private static Path outDir = new Path(testDir, "out"); - @SuppressWarnings("deprecation") - @Test + @Test (timeout=5000) public void testGetNullCounters() throws Exception { //mock creation Job mockJob = mock(Job.class); @@ -57,7 +69,7 @@ public class TestNetworkedJob { verify(mockJob).getCounters(); } - @Test + @Test (timeout=500000) public void testGetJobStatus() throws IOException, InterruptedException, ClassNotFoundException { MiniMRClientCluster mr = null; @@ -105,4 +117,278 @@ public class TestNetworkedJob { } } } +/** + * test JobConf + * @throws Exception + */ + @SuppressWarnings( "deprecation" ) + @Test (timeout=500000) + public void testNetworkedJob() throws Exception { + // mock creation + MiniMRClientCluster mr = null; + FileSystem fileSys = null; + + try { + Configuration conf = new Configuration(); + mr = MiniMRClientClusterFactory.create(this.getClass(), 2, conf); + + JobConf job = new JobConf(mr.getConfig()); + + fileSys = FileSystem.get(job); + fileSys.delete(testDir, true); + FSDataOutputStream out = fileSys.create(inFile, true); + out.writeBytes("This is a test file"); + out.close(); + + FileInputFormat.setInputPaths(job, inFile); + FileOutputFormat.setOutputPath(job, outDir); + + job.setInputFormat(TextInputFormat.class); + job.setOutputFormat(TextOutputFormat.class); + + job.setMapperClass(IdentityMapper.class); + job.setReducerClass(IdentityReducer.class); + job.setNumReduceTasks(0); + + JobClient client = new JobClient(mr.getConfig()); + + RunningJob rj = client.submitJob(job); + JobID jobId = rj.getID(); + NetworkedJob runningJob = (NetworkedJob) client.getJob(jobId); + runningJob.setJobPriority(JobPriority.HIGH.name()); + // test getters + assertTrue(runningJob.getConfiguration().toString() + .endsWith("0001/job.xml")); + assertEquals(runningJob.getID(), jobId); + assertEquals(runningJob.getJobID(), jobId.toString()); + assertEquals(runningJob.getJobName(), "N/A"); + assertTrue(runningJob.getJobFile().endsWith( + ".staging/" + runningJob.getJobID() + "/job.xml")); + assertTrue(runningJob.getTrackingURL().length() > 0); + assertTrue(runningJob.mapProgress() == 0.0f); + assertTrue(runningJob.reduceProgress() == 0.0f); + assertTrue(runningJob.cleanupProgress() == 0.0f); + assertTrue(runningJob.setupProgress() == 0.0f); + + TaskCompletionEvent[] tce = runningJob.getTaskCompletionEvents(0); + assertEquals(tce.length, 0); + + assertEquals(runningJob.getHistoryUrl(),""); + assertFalse(runningJob.isRetired()); + assertEquals( runningJob.getFailureInfo(),""); + assertEquals(runningJob.getJobStatus().getJobName(), "N/A"); + assertEquals(client.getMapTaskReports(jobId).length, 0); + + try { + client.getSetupTaskReports(jobId); + } catch (YarnException e) { + assertEquals(e.getMessage(), "Unrecognized task type: JOB_SETUP"); + } + try { + client.getCleanupTaskReports(jobId); + } catch (YarnException e) { + assertEquals(e.getMessage(), "Unrecognized task type: JOB_CLEANUP"); + } + assertEquals(client.getReduceTaskReports(jobId).length, 0); + // test ClusterStatus + ClusterStatus status = client.getClusterStatus(true); + assertEquals(status.getActiveTrackerNames().size(), 2); + // it method does not implemented and always return empty array or null; + assertEquals(status.getBlacklistedTrackers(), 0); + assertEquals(status.getBlacklistedTrackerNames().size(), 0); + assertEquals(status.getBlackListedTrackersInfo().size(), 0); + assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING); + assertEquals(status.getMapTasks(), 1); + assertEquals(status.getMaxMapTasks(), 20); + assertEquals(status.getMaxReduceTasks(), 4); + assertEquals(status.getNumExcludedNodes(), 0); + assertEquals(status.getReduceTasks(), 1); + assertEquals(status.getTaskTrackers(), 2); + assertEquals(status.getTTExpiryInterval(), 0); + assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING); + + // test read and write + ByteArrayOutputStream dataOut = new ByteArrayOutputStream(); + status.write(new DataOutputStream(dataOut)); + ClusterStatus status2 = new ClusterStatus(); + + status2.readFields(new DataInputStream(new ByteArrayInputStream(dataOut + .toByteArray()))); + assertEquals(status.getActiveTrackerNames(), + status2.getActiveTrackerNames()); + assertEquals(status.getBlackListedTrackersInfo(), + status2.getBlackListedTrackersInfo()); + assertEquals(status.getMapTasks(), status2.getMapTasks()); + + try { + } catch (RuntimeException e) { + assertTrue(e.getMessage().endsWith("not found on CLASSPATH")); + } + + // test taskStatusfilter + JobClient.setTaskOutputFilter(job, TaskStatusFilter.ALL); + assertEquals(JobClient.getTaskOutputFilter(job), TaskStatusFilter.ALL); + + // runningJob.setJobPriority(JobPriority.HIGH.name()); + + // test default map + assertEquals(client.getDefaultMaps(), 20); + assertEquals(client.getDefaultReduces(), 4); + assertEquals(client.getSystemDir().getName(), "jobSubmitDir"); + // test queue information + JobQueueInfo[] rootQueueInfo = client.getRootQueues(); + assertEquals(rootQueueInfo.length, 1); + assertEquals(rootQueueInfo[0].getQueueName(), "default"); + JobQueueInfo[] qinfo = client.getQueues(); + assertEquals(qinfo.length, 1); + assertEquals(qinfo[0].getQueueName(), "default"); + assertEquals(client.getChildQueues("default").length, 0); + assertEquals(client.getJobsFromQueue("default").length, 1); + assertTrue(client.getJobsFromQueue("default")[0].getJobFile().endsWith( + "/job.xml")); + + JobQueueInfo qi = client.getQueueInfo("default"); + assertEquals(qi.getQueueName(), "default"); + assertEquals(qi.getQueueState(), "running"); + + QueueAclsInfo[] aai = client.getQueueAclsForCurrentUser(); + assertEquals(aai.length, 2); + assertEquals(aai[0].getQueueName(), "root"); + assertEquals(aai[1].getQueueName(), "default"); + // test token + Token token = client + .getDelegationToken(new Text(UserGroupInformation.getCurrentUser() + .getShortUserName())); + assertEquals(token.getKind().toString(), "RM_DELEGATION_TOKEN"); + + // test JobClient + + + // The following asserts read JobStatus twice and ensure the returned + // JobStatus objects correspond to the same Job. + assertEquals("Expected matching JobIDs", jobId, client.getJob(jobId) + .getJobStatus().getJobID()); + assertEquals("Expected matching startTimes", rj.getJobStatus() + .getStartTime(), client.getJob(jobId).getJobStatus().getStartTime()); + } finally { + if (fileSys != null) { + fileSys.delete(testDir, true); + } + if (mr != null) { + mr.stop(); + } + } + } + + /** + * test BlackListInfo class + * + * @throws IOException + */ + @Test (timeout=5000) + public void testBlackListInfo() throws IOException { + BlackListInfo info = new BlackListInfo(); + info.setBlackListReport("blackListInfo"); + info.setReasonForBlackListing("reasonForBlackListing"); + info.setTrackerName("trackerName"); + ByteArrayOutputStream byteOut = new ByteArrayOutputStream(); + DataOutput out = new DataOutputStream(byteOut); + info.write(out); + BlackListInfo info2 = new BlackListInfo(); + info2.readFields(new DataInputStream(new ByteArrayInputStream(byteOut + .toByteArray()))); + assertEquals(info, info); + assertEquals(info.toString(), info.toString()); + assertEquals(info.getTrackerName(), "trackerName"); + assertEquals(info.getReasonForBlackListing(), "reasonForBlackListing"); + assertEquals(info.getBlackListReport(), "blackListInfo"); + + } +/** + * test run from command line JobQueueClient + * @throws Exception + */ + @Test (timeout=500000) + public void testJobQueueClient() throws Exception { + MiniMRClientCluster mr = null; + FileSystem fileSys = null; + PrintStream oldOut = System.out; + try { + Configuration conf = new Configuration(); + mr = MiniMRClientClusterFactory.create(this.getClass(), 2, conf); + + JobConf job = new JobConf(mr.getConfig()); + + fileSys = FileSystem.get(job); + fileSys.delete(testDir, true); + FSDataOutputStream out = fileSys.create(inFile, true); + out.writeBytes("This is a test file"); + out.close(); + + FileInputFormat.setInputPaths(job, inFile); + FileOutputFormat.setOutputPath(job, outDir); + + job.setInputFormat(TextInputFormat.class); + job.setOutputFormat(TextOutputFormat.class); + + job.setMapperClass(IdentityMapper.class); + job.setReducerClass(IdentityReducer.class); + job.setNumReduceTasks(0); + + JobClient client = new JobClient(mr.getConfig()); + + client.submitJob(job); + + JobQueueClient jobClient = new JobQueueClient(job); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bytes)); + String[] arg = { "-list" }; + jobClient.run(arg); + assertTrue(bytes.toString().contains("Queue Name : default")); + assertTrue(bytes.toString().contains("Queue State : running")); + bytes = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bytes)); + String[] arg1 = { "-showacls" }; + jobClient.run(arg1); + assertTrue(bytes.toString().contains("Queue acls for user :")); + assertTrue(bytes.toString().contains( + "root ADMINISTER_QUEUE,SUBMIT_APPLICATIONS")); + assertTrue(bytes.toString().contains( + "default ADMINISTER_QUEUE,SUBMIT_APPLICATIONS")); + + // test for info and default queue + + bytes = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bytes)); + String[] arg2 = { "-info", "default" }; + jobClient.run(arg2); + assertTrue(bytes.toString().contains("Queue Name : default")); + assertTrue(bytes.toString().contains("Queue State : running")); + assertTrue(bytes.toString().contains("Scheduling Info")); + + // test for info , default queue and jobs + bytes = new ByteArrayOutputStream(); + System.setOut(new PrintStream(bytes)); + String[] arg3 = { "-info", "default", "-showJobs" }; + jobClient.run(arg3); + assertTrue(bytes.toString().contains("Queue Name : default")); + assertTrue(bytes.toString().contains("Queue State : running")); + assertTrue(bytes.toString().contains("Scheduling Info")); + assertTrue(bytes.toString().contains("job_1")); + + String[] arg4 = {}; + jobClient.run(arg4); + + + } finally { + System.setOut(oldOut); + if (fileSys != null) { + fileSys.delete(testDir, true); + } + if (mr != null) { + mr.stop(); + } + } + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java new file mode 100644 index 00000000000..26d697a6165 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapred; + +import java.io.StringWriter; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import static org.junit.Assert.*; + +import org.junit.Test; + + +public class TestQueueConfigurationParser { +/** + * test xml generation + * @throws ParserConfigurationException + * @throws Exception + */ + @Test (timeout=5000) + public void testQueueConfigurationParser() + throws ParserConfigurationException, Exception { + JobQueueInfo info = new JobQueueInfo("root", "rootInfo"); + JobQueueInfo infoChild1 = new JobQueueInfo("child1", "child1Info"); + JobQueueInfo infoChild2 = new JobQueueInfo("child2", "child1Info"); + + info.addChild(infoChild1); + info.addChild(infoChild2); + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory + .newInstance(); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + + + Document document = builder.newDocument(); + + + // test QueueConfigurationParser.getQueueElement + Element e = QueueConfigurationParser.getQueueElement(document, info); + // transform result to string for check + DOMSource domSource = new DOMSource(e); + StringWriter writer = new StringWriter(); + StreamResult result = new StreamResult(writer); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.transform(domSource, result); + String str= writer.toString(); + assertTrue(str + .endsWith("rootrunningchild1runningchild2running")); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java index 87ab6b2683a..12568d09175 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.mapred; +import java.util.Map; + import junit.framework.TestCase; import org.apache.hadoop.mapred.StatisticsCollector.TimeWindow; @@ -24,6 +26,7 @@ import org.apache.hadoop.mapred.StatisticsCollector.Stat; public class TestStatisticsCollector extends TestCase{ + @SuppressWarnings("rawtypes") public void testMovingWindow() throws Exception { StatisticsCollector collector = new StatisticsCollector(1); TimeWindow window = new TimeWindow("test", 6, 2); @@ -78,6 +81,28 @@ public class TestStatisticsCollector extends TestCase{ collector.update(); assertEquals((10+10+10+12+13+14), stat.getValues().get(window).getValue()); assertEquals(95, stat.getValues().get(sincStart).getValue()); + + // test Stat class + Map updaters= collector.getUpdaters(); + assertEquals(updaters.size(),2); + Map ststistics=collector.getStatistics(); + assertNotNull(ststistics.get("m1")); + + Stat newStat= collector.createStat("m2"); + assertEquals(newStat.name, "m2"); + Stat st=collector.removeStat("m1"); + assertEquals(st.name, "m1"); + assertEquals((10+10+10+12+13+14), stat.getValues().get(window).getValue()); + assertEquals(95, stat.getValues().get(sincStart).getValue()); + st=collector.removeStat("m1"); + // try to remove stat again + assertNull(st); + collector.start(); + // waiting 2,5 sec + Thread.sleep(2500); + assertEquals(69, stat.getValues().get(window).getValue()); + assertEquals(95, stat.getValues().get(sincStart).getValue()); + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java index 4066a8e55f8..82c7b346b04 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java @@ -61,11 +61,12 @@ public class TestTextInputFormat { throw new RuntimeException("init failure", e); } } + @SuppressWarnings("deprecation") private static Path workDir = new Path(new Path(System.getProperty("test.build.data", "/tmp")), "TestTextInputFormat").makeQualified(localFs); - @Test + @Test (timeout=500000) public void testFormat() throws Exception { JobConf job = new JobConf(defaultConf); Path file = new Path(workDir, "test.txt"); @@ -145,7 +146,7 @@ public class TestTextInputFormat { } } - @Test + @Test (timeout=900000) public void testSplitableCodecs() throws IOException { JobConf conf = new JobConf(defaultConf); int seed = new Random().nextInt(); @@ -250,7 +251,7 @@ public class TestTextInputFormat { bufsz); } - @Test + @Test (timeout=5000) public void testUTF8() throws Exception { LineReader in = makeStream("abcd\u20acbdcd\u20ac"); Text line = new Text(); @@ -269,7 +270,7 @@ public class TestTextInputFormat { * * @throws Exception */ - @Test + @Test (timeout=5000) public void testNewLines() throws Exception { final String STR = "a\nbb\n\nccc\rdddd\r\r\r\n\r\neeeee"; final int STRLENBYTES = STR.getBytes().length; @@ -309,7 +310,7 @@ public class TestTextInputFormat { * * @throws Exception */ - @Test + @Test (timeout=5000) public void testMaxLineLength() throws Exception { final String STR = "a\nbb\n\nccc\rdddd\r\neeeee"; final int STRLENBYTES = STR.getBytes().length; @@ -334,7 +335,7 @@ public class TestTextInputFormat { } } - @Test + @Test (timeout=5000) public void testMRMaxLine() throws Exception { final int MAXPOS = 1024 * 1024; final int MAXLINE = 10 * 1024; @@ -354,6 +355,9 @@ public class TestTextInputFormat { position += b.length; return b.length; } + public void reset() { + position=0; + } }; final LongWritable key = new LongWritable(); final Text val = new Text(); @@ -362,8 +366,14 @@ public class TestTextInputFormat { conf.setInt(org.apache.hadoop.mapreduce.lib.input. LineRecordReader.MAX_LINE_LENGTH, MAXLINE); conf.setInt("io.file.buffer.size", BUF); // used by LRR - final LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf); + // test another constructor + LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf); assertFalse("Read a line from null", lrr.next(key, val)); + infNull.reset(); + lrr = new LineRecordReader(infNull, 0L, MAXLINE, MAXPOS); + assertFalse("Read a line from null", lrr.next(key, val)); + + } private static void writeFile(FileSystem fs, Path name, @@ -400,7 +410,7 @@ public class TestTextInputFormat { /** * Test using the gzip codec for reading */ - @Test + @Test (timeout=5000) public void testGzip() throws IOException { JobConf job = new JobConf(defaultConf); CompressionCodec gzip = new GzipCodec(); @@ -434,7 +444,7 @@ public class TestTextInputFormat { /** * Test using the gzip codec and an empty input file */ - @Test + @Test (timeout=5000) public void testGzipEmpty() throws IOException { JobConf job = new JobConf(defaultConf); CompressionCodec gzip = new GzipCodec(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java index 3d0f611d8a7..1d4a91558d8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java @@ -44,7 +44,6 @@ public class TestTextOutputFormat extends TestCase { "data"), FileOutputCommitter.TEMP_DIR_NAME), "_" + attempt); - @SuppressWarnings("unchecked") public void testFormat() throws Exception { JobConf job = new JobConf(); job.set(JobContext.TASK_ATTEMPT_ID, attempt); @@ -59,8 +58,8 @@ public class TestTextOutputFormat extends TestCase { // A reporter that does nothing Reporter reporter = Reporter.NULL; - TextOutputFormat theOutputFormat = new TextOutputFormat(); - RecordWriter theRecordWriter = + TextOutputFormat theOutputFormat = new TextOutputFormat(); + RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter); Text key1 = new Text("key1"); @@ -95,7 +94,6 @@ public class TestTextOutputFormat extends TestCase { } - @SuppressWarnings("unchecked") public void testFormatWithCustomSeparator() throws Exception { JobConf job = new JobConf(); String separator = "\u0001"; @@ -112,8 +110,8 @@ public class TestTextOutputFormat extends TestCase { // A reporter that does nothing Reporter reporter = Reporter.NULL; - TextOutputFormat theOutputFormat = new TextOutputFormat(); - RecordWriter theRecordWriter = + TextOutputFormat theOutputFormat = new TextOutputFormat(); + RecordWriter theRecordWriter = theOutputFormat.getRecordWriter(localFs, job, file, reporter); Text key1 = new Text("key1"); @@ -147,7 +145,61 @@ public class TestTextOutputFormat extends TestCase { assertEquals(output, expectedOutput.toString()); } + /** + * test compressed file + * @throws IOException + */ + public void testCompress() throws IOException{ + JobConf job = new JobConf(); + String separator = "\u0001"; + job.set("mapreduce.output.textoutputformat.separator", separator); + job.set(JobContext.TASK_ATTEMPT_ID, attempt); + job.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS,"true"); + + FileOutputFormat.setOutputPath(job, workDir.getParent().getParent()); + FileOutputFormat.setWorkOutputPath(job, workDir); + FileSystem fs = workDir.getFileSystem(job); + if (!fs.mkdirs(workDir)) { + fail("Failed to create output directory"); + } + String file = "test.txt"; + // A reporter that does nothing + Reporter reporter = Reporter.NULL; + + TextOutputFormat theOutputFormat = new TextOutputFormat(); + RecordWriter theRecordWriter = + theOutputFormat.getRecordWriter(localFs, job, file, reporter); + Text key1 = new Text("key1"); + Text key2 = new Text("key2"); + Text val1 = new Text("val1"); + Text val2 = new Text("val2"); + NullWritable nullWritable = NullWritable.get(); + + try { + theRecordWriter.write(key1, val1); + theRecordWriter.write(null, nullWritable); + theRecordWriter.write(null, val1); + theRecordWriter.write(nullWritable, val2); + theRecordWriter.write(key2, nullWritable); + theRecordWriter.write(key1, null); + theRecordWriter.write(null, null); + theRecordWriter.write(key2, val2); + + } finally { + theRecordWriter.close(reporter); + } + File expectedFile = new File(new Path(workDir, file).toString()); + StringBuffer expectedOutput = new StringBuffer(); + expectedOutput.append(key1).append(separator).append(val1).append("\n"); + expectedOutput.append(val1).append("\n"); + expectedOutput.append(val2).append("\n"); + expectedOutput.append(key2).append("\n"); + expectedOutput.append(key1).append("\n"); + expectedOutput.append(key2).append(separator).append(val2).append("\n"); + String output = UtilsForTests.slurp(expectedFile); + assertEquals(output, expectedOutput.toString()); + } public static void main(String[] args) throws Exception { new TestTextOutputFormat().testFormat(); } From d41e67b966b4ced602ae27e6ccc6a73cd4068a05 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 26 Mar 2013 18:15:40 +0000 Subject: [PATCH 15/37] MAPREDUCE-5077. Remove mapreduce.util.ResourceCalculatorPlugin and related code. Contributed by Karthik Kambatla. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461251 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../util/LinuxResourceCalculatorPlugin.java | 418 ---------- .../util/ProcfsBasedProcessTree.java | 743 ------------------ .../util/ResourceCalculatorPlugin.java | 165 ---- .../TestLinuxResourceCalculatorPlugin.java | 236 ------ .../util/LinuxMemoryCalculatorPlugin.java | 51 -- .../util/MemoryCalculatorPlugin.java | 82 -- .../util/TestProcfsBasedProcessTree.java | 677 ---------------- .../apache/hadoop/mapred/gridmix/LoadJob.java | 2 +- .../CumulativeCpuUsageEmulatorPlugin.java | 8 +- .../ResourceUsageEmulatorPlugin.java | 2 +- .../resourceusage/ResourceUsageMatcher.java | 2 +- .../TotalHeapUsageEmulatorPlugin.java | 2 +- .../DummyResourceCalculatorPlugin.java | 30 +- .../gridmix/TestGridmixMemoryEmulation.java | 3 +- .../gridmix/TestResourceUsageEmulators.java | 13 +- 16 files changed, 23 insertions(+), 2414 deletions(-) delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/LinuxMemoryCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MemoryCalculatorPlugin.java delete mode 100644 hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java rename {hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred => hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix}/DummyResourceCalculatorPlugin.java (76%) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e81da764d07..ed9cf3d5de7 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -259,6 +259,9 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5062. Fix MR AM to read max-retries from the RM. (Zhijie Shen via vinodkv) + MAPREDUCE-5077. Remove mapreduce.util.ResourceCalculatorPlugin and related + code. (Karthik Kambatla via sseth) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java deleted file mode 100644 index c7bdea102ea..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java +++ /dev/null @@ -1,418 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -import com.google.common.base.Charsets; - -/** - * Plugin to calculate resource information on Linux systems. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class LinuxResourceCalculatorPlugin extends ResourceCalculatorPlugin { - private static final Log LOG = - LogFactory.getLog(LinuxResourceCalculatorPlugin.class); - - public static final int UNAVAILABLE = -1; - - /** - * proc's meminfo virtual file has keys-values in the format - * "key:[ \t]*value[ \t]kB". - */ - private static final String PROCFS_MEMFILE = "/proc/meminfo"; - private static final Pattern PROCFS_MEMFILE_FORMAT = - Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB"); - - // We need the values for the following keys in meminfo - private static final String MEMTOTAL_STRING = "MemTotal"; - private static final String SWAPTOTAL_STRING = "SwapTotal"; - private static final String MEMFREE_STRING = "MemFree"; - private static final String SWAPFREE_STRING = "SwapFree"; - private static final String INACTIVE_STRING = "Inactive"; - - /** - * Patterns for parsing /proc/cpuinfo - */ - private static final String PROCFS_CPUINFO = "/proc/cpuinfo"; - private static final Pattern PROCESSOR_FORMAT = - Pattern.compile("^processor[ \t]:[ \t]*([0-9]*)"); - private static final Pattern FREQUENCY_FORMAT = - Pattern.compile("^cpu MHz[ \t]*:[ \t]*([0-9.]*)"); - - /** - * Pattern for parsing /proc/stat - */ - private static final String PROCFS_STAT = "/proc/stat"; - private static final Pattern CPU_TIME_FORMAT = - Pattern.compile("^cpu[ \t]*([0-9]*)" + - "[ \t]*([0-9]*)[ \t]*([0-9]*)[ \t].*"); - - private String procfsMemFile; - private String procfsCpuFile; - private String procfsStatFile; - long jiffyLengthInMillis; - - private long ramSize = 0; - private long swapSize = 0; - private long ramSizeFree = 0; // free ram space on the machine (kB) - private long swapSizeFree = 0; // free swap space on the machine (kB) - private long inactiveSize = 0; // inactive cache memory (kB) - private int numProcessors = 0; // number of processors on the system - private long cpuFrequency = 0L; // CPU frequency on the system (kHz) - private long cumulativeCpuTime = 0L; // CPU used time since system is on (ms) - private long lastCumulativeCpuTime = 0L; // CPU used time read last time (ms) - // Unix timestamp while reading the CPU time (ms) - private float cpuUsage = UNAVAILABLE; - private long sampleTime = UNAVAILABLE; - private long lastSampleTime = UNAVAILABLE; - private ProcfsBasedProcessTree pTree = null; - - boolean readMemInfoFile = false; - boolean readCpuInfoFile = false; - - /** - * Get current time - * @return Unix time stamp in millisecond - */ - long getCurrentTime() { - return System.currentTimeMillis(); - } - - public LinuxResourceCalculatorPlugin() { - procfsMemFile = PROCFS_MEMFILE; - procfsCpuFile = PROCFS_CPUINFO; - procfsStatFile = PROCFS_STAT; - jiffyLengthInMillis = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS; - String pid = System.getenv().get("JVM_PID"); - pTree = new ProcfsBasedProcessTree(pid); - } - - /** - * Constructor which allows assigning the /proc/ directories. This will be - * used only in unit tests - * @param procfsMemFile fake file for /proc/meminfo - * @param procfsCpuFile fake file for /proc/cpuinfo - * @param procfsStatFile fake file for /proc/stat - * @param jiffyLengthInMillis fake jiffy length value - */ - public LinuxResourceCalculatorPlugin(String procfsMemFile, - String procfsCpuFile, - String procfsStatFile, - long jiffyLengthInMillis) { - this.procfsMemFile = procfsMemFile; - this.procfsCpuFile = procfsCpuFile; - this.procfsStatFile = procfsStatFile; - this.jiffyLengthInMillis = jiffyLengthInMillis; - String pid = System.getenv().get("JVM_PID"); - pTree = new ProcfsBasedProcessTree(pid); - } - - /** - * Read /proc/meminfo, parse and compute memory information only once - */ - private void readProcMemInfoFile() { - readProcMemInfoFile(false); - } - - /** - * Read /proc/meminfo, parse and compute memory information - * @param readAgain if false, read only on the first time - */ - private void readProcMemInfoFile(boolean readAgain) { - - if (readMemInfoFile && !readAgain) { - return; - } - - // Read "/proc/memInfo" file - BufferedReader in = null; - InputStreamReader fReader = null; - try { - fReader = new InputStreamReader(new FileInputStream(procfsMemFile), - Charsets.UTF_8); - in = new BufferedReader(fReader); - } catch (FileNotFoundException f) { - // shouldn't happen.... - return; - } - - Matcher mat = null; - - try { - String str = in.readLine(); - while (str != null) { - mat = PROCFS_MEMFILE_FORMAT.matcher(str); - if (mat.find()) { - if (mat.group(1).equals(MEMTOTAL_STRING)) { - ramSize = Long.parseLong(mat.group(2)); - } else if (mat.group(1).equals(SWAPTOTAL_STRING)) { - swapSize = Long.parseLong(mat.group(2)); - } else if (mat.group(1).equals(MEMFREE_STRING)) { - ramSizeFree = Long.parseLong(mat.group(2)); - } else if (mat.group(1).equals(SWAPFREE_STRING)) { - swapSizeFree = Long.parseLong(mat.group(2)); - } else if (mat.group(1).equals(INACTIVE_STRING)) { - inactiveSize = Long.parseLong(mat.group(2)); - } - } - str = in.readLine(); - } - } catch (IOException io) { - LOG.warn("Error reading the stream " + io); - } finally { - // Close the streams - try { - fReader.close(); - try { - in.close(); - } catch (IOException i) { - LOG.warn("Error closing the stream " + in); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader); - } - } - - readMemInfoFile = true; - } - - /** - * Read /proc/cpuinfo, parse and calculate CPU information - */ - private void readProcCpuInfoFile() { - // This directory needs to be read only once - if (readCpuInfoFile) { - return; - } - // Read "/proc/cpuinfo" file - BufferedReader in = null; - InputStreamReader fReader = null; - try { - fReader = new InputStreamReader(new FileInputStream(procfsCpuFile), - Charsets.UTF_8); - in = new BufferedReader(fReader); - } catch (FileNotFoundException f) { - // shouldn't happen.... - return; - } - Matcher mat = null; - try { - numProcessors = 0; - String str = in.readLine(); - while (str != null) { - mat = PROCESSOR_FORMAT.matcher(str); - if (mat.find()) { - numProcessors++; - } - mat = FREQUENCY_FORMAT.matcher(str); - if (mat.find()) { - cpuFrequency = (long)(Double.parseDouble(mat.group(1)) * 1000); // kHz - } - str = in.readLine(); - } - } catch (IOException io) { - LOG.warn("Error reading the stream " + io); - } finally { - // Close the streams - try { - fReader.close(); - try { - in.close(); - } catch (IOException i) { - LOG.warn("Error closing the stream " + in); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader); - } - } - readCpuInfoFile = true; - } - - /** - * Read /proc/stat file, parse and calculate cumulative CPU - */ - private void readProcStatFile() { - // Read "/proc/stat" file - BufferedReader in = null; - InputStreamReader fReader = null; - try { - fReader = new InputStreamReader(new FileInputStream(procfsStatFile), - Charsets.UTF_8); - in = new BufferedReader(fReader); - } catch (FileNotFoundException f) { - // shouldn't happen.... - return; - } - - Matcher mat = null; - try { - String str = in.readLine(); - while (str != null) { - mat = CPU_TIME_FORMAT.matcher(str); - if (mat.find()) { - long uTime = Long.parseLong(mat.group(1)); - long nTime = Long.parseLong(mat.group(2)); - long sTime = Long.parseLong(mat.group(3)); - cumulativeCpuTime = uTime + nTime + sTime; // milliseconds - break; - } - str = in.readLine(); - } - cumulativeCpuTime *= jiffyLengthInMillis; - } catch (IOException io) { - LOG.warn("Error reading the stream " + io); - } finally { - // Close the streams - try { - fReader.close(); - try { - in.close(); - } catch (IOException i) { - LOG.warn("Error closing the stream " + in); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader); - } - } - } - - /** {@inheritDoc} */ - @Override - public long getPhysicalMemorySize() { - readProcMemInfoFile(); - return ramSize * 1024; - } - - /** {@inheritDoc} */ - @Override - public long getVirtualMemorySize() { - readProcMemInfoFile(); - return (ramSize + swapSize) * 1024; - } - - /** {@inheritDoc} */ - @Override - public long getAvailablePhysicalMemorySize() { - readProcMemInfoFile(true); - return (ramSizeFree + inactiveSize) * 1024; - } - - /** {@inheritDoc} */ - @Override - public long getAvailableVirtualMemorySize() { - readProcMemInfoFile(true); - return (ramSizeFree + swapSizeFree + inactiveSize) * 1024; - } - - /** {@inheritDoc} */ - @Override - public int getNumProcessors() { - readProcCpuInfoFile(); - return numProcessors; - } - - /** {@inheritDoc} */ - @Override - public long getCpuFrequency() { - readProcCpuInfoFile(); - return cpuFrequency; - } - - /** {@inheritDoc} */ - @Override - public long getCumulativeCpuTime() { - readProcStatFile(); - return cumulativeCpuTime; - } - - /** {@inheritDoc} */ - @Override - public float getCpuUsage() { - readProcStatFile(); - sampleTime = getCurrentTime(); - if (lastSampleTime == UNAVAILABLE || - lastSampleTime > sampleTime) { - // lastSampleTime > sampleTime may happen when the system time is changed - lastSampleTime = sampleTime; - lastCumulativeCpuTime = cumulativeCpuTime; - return cpuUsage; - } - // When lastSampleTime is sufficiently old, update cpuUsage. - // Also take a sample of the current time and cumulative CPU time for the - // use of the next calculation. - final long MINIMUM_UPDATE_INTERVAL = 10 * jiffyLengthInMillis; - if (sampleTime > lastSampleTime + MINIMUM_UPDATE_INTERVAL) { - cpuUsage = (float)(cumulativeCpuTime - lastCumulativeCpuTime) * 100F / - ((float)(sampleTime - lastSampleTime) * getNumProcessors()); - lastSampleTime = sampleTime; - lastCumulativeCpuTime = cumulativeCpuTime; - } - return cpuUsage; - } - - /** - * Test the {@link LinuxResourceCalculatorPlugin} - * - * @param args - */ - public static void main(String[] args) { - LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin(); - System.out.println("Physical memory Size (bytes) : " - + plugin.getPhysicalMemorySize()); - System.out.println("Total Virtual memory Size (bytes) : " - + plugin.getVirtualMemorySize()); - System.out.println("Available Physical memory Size (bytes) : " - + plugin.getAvailablePhysicalMemorySize()); - System.out.println("Total Available Virtual memory Size (bytes) : " - + plugin.getAvailableVirtualMemorySize()); - System.out.println("Number of Processors : " + plugin.getNumProcessors()); - System.out.println("CPU frequency (kHz) : " + plugin.getCpuFrequency()); - System.out.println("Cumulative CPU time (ms) : " + - plugin.getCumulativeCpuTime()); - try { - // Sleep so we can compute the CPU usage - Thread.sleep(500L); - } catch (InterruptedException e) { - // do nothing - } - System.out.println("CPU usage % : " + plugin.getCpuUsage()); - } - - @Override - public ProcResourceValues getProcResourceValues() { - pTree.updateProcessTree(); - long cpuTime = pTree.getCumulativeCpuTime(); - long pMem = pTree.getCumulativeRssmem(); - long vMem = pTree.getCumulativeVmem(); - return new ProcResourceValues(cpuTime, pMem, vMem); - } -} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java deleted file mode 100644 index a510eb569a4..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java +++ /dev/null @@ -1,743 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStreamReader; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.HashMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.LinkedList; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; -import org.apache.hadoop.util.StringUtils; - -import com.google.common.base.Charsets; - -/** - * A Proc file-system based ProcessTree. Works only on Linux. - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class ProcfsBasedProcessTree extends ProcessTree { - - static final Log LOG = LogFactory - .getLog(ProcfsBasedProcessTree.class); - - private static final String PROCFS = "/proc/"; - - private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern .compile( - "^([0-9-]+)\\s([^\\s]+)\\s[^\\s]\\s([0-9-]+)\\s([0-9-]+)\\s([0-9-]+)\\s" + - "([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)\\s([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)" + - "(\\s[0-9-]+){15}"); - - static final String PROCFS_STAT_FILE = "stat"; - static final String PROCFS_CMDLINE_FILE = "cmdline"; - public static final long PAGE_SIZE; - static { - ShellCommandExecutor shellExecutor = - new ShellCommandExecutor(new String[]{"getconf", "PAGESIZE"}); - long pageSize = -1; - try { - shellExecutor.execute(); - pageSize = Long.parseLong(shellExecutor.getOutput().replace("\n", "")); - } catch (IOException e) { - LOG.error(StringUtils.stringifyException(e)); - } finally { - PAGE_SIZE = pageSize; - } - } - public static final long JIFFY_LENGTH_IN_MILLIS; // in millisecond - static { - ShellCommandExecutor shellExecutor = - new ShellCommandExecutor(new String[]{"getconf", "CLK_TCK"}); - long jiffiesPerSecond = -1; - try { - shellExecutor.execute(); - jiffiesPerSecond = Long.parseLong(shellExecutor.getOutput().replace("\n", "")); - } catch (IOException e) { - LOG.error(StringUtils.stringifyException(e)); - } finally { - JIFFY_LENGTH_IN_MILLIS = jiffiesPerSecond != -1 ? - Math.round(1000D / jiffiesPerSecond) : -1; - } - } - - // to enable testing, using this variable which can be configured - // to a test directory. - private String procfsDir; - - static private String deadPid = "-1"; - private String pid = deadPid; - static private Pattern numberPattern = Pattern.compile("[1-9][0-9]*"); - private Long cpuTime = 0L; - private boolean setsidUsed = false; - private long sleeptimeBeforeSigkill = DEFAULT_SLEEPTIME_BEFORE_SIGKILL; - - private Map processTree = new HashMap(); - - public ProcfsBasedProcessTree(String pid) { - this(pid, false, DEFAULT_SLEEPTIME_BEFORE_SIGKILL); - } - - public ProcfsBasedProcessTree(String pid, boolean setsidUsed, - long sigkillInterval) { - this(pid, setsidUsed, sigkillInterval, PROCFS); - } - - /** - * Build a new process tree rooted at the pid. - * - * This method is provided mainly for testing purposes, where - * the root of the proc file system can be adjusted. - * - * @param pid root of the process tree - * @param setsidUsed true, if setsid was used for the root pid - * @param sigkillInterval how long to wait between a SIGTERM and SIGKILL - * when killing a process tree - * @param procfsDir the root of a proc file system - only used for testing. - */ - public ProcfsBasedProcessTree(String pid, boolean setsidUsed, - long sigkillInterval, String procfsDir) { - this.pid = getValidPID(pid); - this.setsidUsed = setsidUsed; - sleeptimeBeforeSigkill = sigkillInterval; - this.procfsDir = procfsDir; - } - - /** - * Sets SIGKILL interval - * @deprecated Use {@link ProcfsBasedProcessTree#ProcfsBasedProcessTree( - * String, boolean, long)} instead - * @param interval The time to wait before sending SIGKILL - * after sending SIGTERM - */ - @Deprecated - public void setSigKillInterval(long interval) { - sleeptimeBeforeSigkill = interval; - } - - /** - * Checks if the ProcfsBasedProcessTree is available on this system. - * - * @return true if ProcfsBasedProcessTree is available. False otherwise. - */ - public static boolean isAvailable() { - try { - String osName = System.getProperty("os.name"); - if (!osName.startsWith("Linux")) { - LOG.info("ProcfsBasedProcessTree currently is supported only on " - + "Linux."); - return false; - } - } catch (SecurityException se) { - LOG.warn("Failed to get Operating System name. " + se); - return false; - } - return true; - } - - /** - * Update the process-tree with latest state. If the root-process is not alive, - * tree will become empty. - */ - public void updateProcessTree() { - if (!pid.equals(deadPid)) { - // Get the list of processes - List processList = getProcessList(); - - Map allProcessInfo = new HashMap(); - - // cache the processTree to get the age for processes - Map oldProcs = - new HashMap(processTree); - processTree.clear(); - - ProcessInfo me = null; - for (String proc : processList) { - // Get information for each process - ProcessInfo pInfo = new ProcessInfo(proc); - if (constructProcessInfo(pInfo, procfsDir) != null) { - allProcessInfo.put(proc, pInfo); - if (proc.equals(this.pid)) { - me = pInfo; // cache 'me' - processTree.put(proc, pInfo); - } - } - } - - if (me == null) { - return; - } - - // Add each process to its parent. - for (Map.Entry entry : allProcessInfo.entrySet()) { - String pID = entry.getKey(); - if (!pID.equals("1")) { - ProcessInfo pInfo = entry.getValue(); - ProcessInfo parentPInfo = allProcessInfo.get(pInfo.getPpid()); - if (parentPInfo != null) { - parentPInfo.addChild(pInfo); - } - } - } - - // now start constructing the process-tree - LinkedList pInfoQueue = new LinkedList(); - pInfoQueue.addAll(me.getChildren()); - while (!pInfoQueue.isEmpty()) { - ProcessInfo pInfo = pInfoQueue.remove(); - if (!processTree.containsKey(pInfo.getPid())) { - processTree.put(pInfo.getPid(), pInfo); - } - pInfoQueue.addAll(pInfo.getChildren()); - } - - // update age values and compute the number of jiffies since last update - for (Map.Entry procs : processTree.entrySet()) { - ProcessInfo oldInfo = oldProcs.get(procs.getKey()); - if (procs.getValue() != null) { - procs.getValue().updateJiffy(oldInfo); - if (oldInfo != null) { - procs.getValue().updateAge(oldInfo); - } - } - } - - if (LOG.isDebugEnabled()) { - // Log.debug the ProcfsBasedProcessTree - LOG.debug(this.toString()); - } - } - } - - /** - * Is the root-process alive? - * - * @return true if the root-process is alive, false otherwise. - */ - public boolean isAlive() { - if (pid.equals(deadPid)) { - return false; - } else { - return isAlive(pid); - } - } - - /** - * Is any of the subprocesses in the process-tree alive? - * - * @return true if any of the processes in the process-tree is - * alive, false otherwise. - */ - public boolean isAnyProcessInTreeAlive() { - for (String pId : processTree.keySet()) { - if (isAlive(pId)) { - return true; - } - } - return false; - } - - /** Verify that the given process id is same as its process group id. - * @param pidStr Process id of the to-be-verified-process - * @param procfsDir Procfs root dir - */ - static boolean checkPidPgrpidForMatch(String pidStr, String procfsDir) { - // Get information for this process - ProcessInfo pInfo = new ProcessInfo(pidStr); - pInfo = constructProcessInfo(pInfo, procfsDir); - if (pInfo == null) { - // process group leader may have finished execution, but we still need to - // kill the subProcesses in the process group. - return true; - } - - String pgrpId = pInfo.getPgrpId().toString(); - //make sure that pId and its pgrpId match - if (!pgrpId.equals(pidStr)) { - LOG.warn("Unexpected: Process with PID " + pidStr + - " is not a process group leader. pgrpId is: " + pInfo.getPgrpId()); - return false; - } - if (LOG.isDebugEnabled()) { - LOG.debug(pidStr + " is a process group leader, as expected."); - } - return true; - } - - /** Make sure that the given pid is a process group leader and then - * destroy the process group. - * @param pgrpId Process group id of to-be-killed-processes - * @param interval The time to wait before sending SIGKILL - * after sending SIGTERM - * @param inBackground Process is to be killed in the back ground with - * a separate thread - */ - public static void assertAndDestroyProcessGroup(String pgrpId, long interval, - boolean inBackground) - throws IOException { - // Make sure that the pid given is a process group leader - if (!checkPidPgrpidForMatch(pgrpId, PROCFS)) { - throw new IOException("Process with PID " + pgrpId + - " is not a process group leader."); - } - destroyProcessGroup(pgrpId, interval, inBackground); - } - - /** - * Destroy the process-tree. - */ - public void destroy() { - destroy(true); - } - - /** - * Destroy the process-tree. - * @param inBackground Process is to be killed in the back ground with - * a separate thread - */ - public void destroy(boolean inBackground) { - LOG.debug("Killing ProcfsBasedProcessTree of " + pid); - if (pid.equals(deadPid)) { - return; - } - if (isAlive(pid.toString())) { - if (isSetsidAvailable && setsidUsed) { - // In this case, we know that pid got created using setsid. So kill the - // whole processGroup. - try { - assertAndDestroyProcessGroup(pid.toString(), sleeptimeBeforeSigkill, - inBackground); - } catch (IOException e) { - LOG.warn(StringUtils.stringifyException(e)); - } - } - else { - //TODO: Destroy all the processes in the subtree in this case also. - // For the time being, killing only the root process. - destroyProcess(pid.toString(), sleeptimeBeforeSigkill, inBackground); - } - } - } - - private static final String PROCESSTREE_DUMP_FORMAT = - "\t|- %s %s %d %d %s %d %d %d %d %s%n"; - - /** - * Get a dump of the process-tree. - * - * @return a string concatenating the dump of information of all the processes - * in the process-tree - */ - public String getProcessTreeDump() { - StringBuilder ret = new StringBuilder(); - // The header. - ret.append(String.format("\t|- PID PPID PGRPID SESSID CMD_NAME " - + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " - + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE%n")); - for (ProcessInfo p : processTree.values()) { - if (p != null) { - ret.append(String.format(PROCESSTREE_DUMP_FORMAT, p.getPid(), p - .getPpid(), p.getPgrpId(), p.getSessionId(), p.getName(), p - .getUtime(), p.getStime(), p.getVmem(), p.getRssmemPage(), p - .getCmdLine(procfsDir))); - } - } - return ret.toString(); - } - - /** - * Get the cumulative virtual memory used by all the processes in the - * process-tree. - * - * @return cumulative virtual memory used by the process-tree in bytes. - */ - public long getCumulativeVmem() { - // include all processes.. all processes will be older than 0. - return getCumulativeVmem(0); - } - - /** - * Get the cumulative resident set size (rss) memory used by all the processes - * in the process-tree. - * - * @return cumulative rss memory used by the process-tree in bytes. return 0 - * if it cannot be calculated - */ - public long getCumulativeRssmem() { - // include all processes.. all processes will be older than 0. - return getCumulativeRssmem(0); - } - - /** - * Get the cumulative virtual memory used by all the processes in the - * process-tree that are older than the passed in age. - * - * @param olderThanAge processes above this age are included in the - * memory addition - * @return cumulative virtual memory used by the process-tree in bytes, - * for processes older than this age. - */ - public long getCumulativeVmem(int olderThanAge) { - long total = 0; - for (ProcessInfo p : processTree.values()) { - if ((p != null) && (p.getAge() > olderThanAge)) { - total += p.getVmem(); - } - } - return total; - } - - /** - * Get the cumulative resident set size (rss) memory used by all the processes - * in the process-tree that are older than the passed in age. - * - * @param olderThanAge processes above this age are included in the - * memory addition - * @return cumulative rss memory used by the process-tree in bytes, - * for processes older than this age. return 0 if it cannot be - * calculated - */ - public long getCumulativeRssmem(int olderThanAge) { - if (PAGE_SIZE < 0) { - return 0; - } - long totalPages = 0; - for (ProcessInfo p : processTree.values()) { - if ((p != null) && (p.getAge() > olderThanAge)) { - totalPages += p.getRssmemPage(); - } - } - return totalPages * PAGE_SIZE; // convert # pages to byte - } - - /** - * Get the CPU time in millisecond used by all the processes in the - * process-tree since the process-tree created - * - * @return cumulative CPU time in millisecond since the process-tree created - * return 0 if it cannot be calculated - */ - public long getCumulativeCpuTime() { - if (JIFFY_LENGTH_IN_MILLIS < 0) { - return 0; - } - long incJiffies = 0; - for (ProcessInfo p : processTree.values()) { - if (p != null) { - incJiffies += p.dtime; - } - } - cpuTime += incJiffies * JIFFY_LENGTH_IN_MILLIS; - return cpuTime; - } - - private static String getValidPID(String pid) { - if (pid == null) return deadPid; - Matcher m = numberPattern.matcher(pid); - if (m.matches()) return pid; - return deadPid; - } - - /** - * Get the list of all processes in the system. - */ - private List getProcessList() { - String[] processDirs = (new File(procfsDir)).list(); - List processList = new ArrayList(); - - for (String dir : processDirs) { - Matcher m = numberPattern.matcher(dir); - if (!m.matches()) continue; - try { - if ((new File(procfsDir, dir)).isDirectory()) { - processList.add(dir); - } - } catch (SecurityException s) { - // skip this process - } - } - return processList; - } - - /** - * Construct the ProcessInfo using the process' PID and procfs rooted at the - * specified directory and return the same. It is provided mainly to assist - * testing purposes. - * - * Returns null on failing to read from procfs, - * - * @param pinfo ProcessInfo that needs to be updated - * @param procfsDir root of the proc file system - * @return updated ProcessInfo, null on errors. - */ - private static ProcessInfo constructProcessInfo(ProcessInfo pinfo, - String procfsDir) { - ProcessInfo ret = null; - // Read "procfsDir//stat" file - typically /proc//stat - BufferedReader in = null; - InputStreamReader fReader = null; - try { - File pidDir = new File(procfsDir, pinfo.getPid()); - fReader = new InputStreamReader(new FileInputStream( - new File(pidDir, PROCFS_STAT_FILE)), Charsets.UTF_8); - in = new BufferedReader(fReader); - } catch (FileNotFoundException f) { - // The process vanished in the interim! - LOG.info("The process " + pinfo.getPid() - + " may have finished in the interim."); - return ret; - } - - ret = pinfo; - try { - String str = in.readLine(); // only one line - Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str); - boolean mat = m.find(); - if (mat) { - // Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss) - pinfo.updateProcessInfo(m.group(2), m.group(3), - Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)), - Long.parseLong(m.group(7)), new BigInteger(m.group(8)), - Long.parseLong(m.group(10)), Long.parseLong(m.group(11))); - } else { - LOG.warn("Unexpected: procfs stat file is not in the expected format" - + " for process with pid " + pinfo.getPid()); - ret = null; - } - } catch (IOException io) { - LOG.warn("Error reading the stream " + io); - ret = null; - } finally { - // Close the streams - try { - fReader.close(); - try { - in.close(); - } catch (IOException i) { - LOG.warn("Error closing the stream " + in); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader); - } - } - - return ret; - } - /** - * Returns a string printing PIDs of process present in the - * ProcfsBasedProcessTree. Output format : [pid pid ..] - */ - public String toString() { - StringBuffer pTree = new StringBuffer("[ "); - for (String p : processTree.keySet()) { - pTree.append(p); - pTree.append(" "); - } - return pTree.substring(0, pTree.length()) + "]"; - } - - /** - * - * Class containing information of a process. - * - */ - private static class ProcessInfo { - private String pid; // process-id - private String name; // command name - private Integer pgrpId; // process group-id - private String ppid; // parent process-id - private Integer sessionId; // session-id - private Long vmem; // virtual memory usage - private Long rssmemPage; // rss memory usage in # of pages - private Long utime = 0L; // # of jiffies in user mode - private final BigInteger MAX_LONG = BigInteger.valueOf(Long.MAX_VALUE); - private BigInteger stime = new BigInteger("0"); // # of jiffies in kernel mode - // how many times has this process been seen alive - private int age; - - // # of jiffies used since last update: - private Long dtime = 0L; - // dtime = (utime + stime) - (utimeOld + stimeOld) - // We need this to compute the cumulative CPU time - // because the subprocess may finish earlier than root process - - private List children = new ArrayList(); // list of children - - public ProcessInfo(String pid) { - this.pid = pid; - // seeing this the first time. - this.age = 1; - } - - public String getPid() { - return pid; - } - - public String getName() { - return name; - } - - public Integer getPgrpId() { - return pgrpId; - } - - public String getPpid() { - return ppid; - } - - public Integer getSessionId() { - return sessionId; - } - - public Long getVmem() { - return vmem; - } - - public Long getUtime() { - return utime; - } - - public BigInteger getStime() { - return stime; - } - - public Long getDtime() { - return dtime; - } - - public Long getRssmemPage() { // get rss # of pages - return rssmemPage; - } - - public int getAge() { - return age; - } - - public boolean isParent(ProcessInfo p) { - if (pid.equals(p.getPpid())) { - return true; - } - return false; - } - - public void updateProcessInfo(String name, String ppid, Integer pgrpId, - Integer sessionId, Long utime, BigInteger stime, Long vmem, Long rssmem) { - this.name = name; - this.ppid = ppid; - this.pgrpId = pgrpId; - this.sessionId = sessionId; - this.utime = utime; - this.stime = stime; - this.vmem = vmem; - this.rssmemPage = rssmem; - } - - public void updateJiffy(ProcessInfo oldInfo) { - if (oldInfo == null) { - BigInteger sum = this.stime.add(BigInteger.valueOf(this.utime)); - if (sum.compareTo(MAX_LONG) > 0) { - this.dtime = 0L; - LOG.warn("Sum of stime (" + this.stime + ") and utime (" + this.utime - + ") is greater than " + Long.MAX_VALUE); - } else { - this.dtime = sum.longValue(); - } - return; - } - this.dtime = (this.utime - oldInfo.utime + - this.stime.subtract(oldInfo.stime).longValue()); - } - - public void updateAge(ProcessInfo oldInfo) { - this.age = oldInfo.age + 1; - } - - public boolean addChild(ProcessInfo p) { - return children.add(p); - } - - public List getChildren() { - return children; - } - - public String getCmdLine(String procfsDir) { - String ret = "N/A"; - if (pid == null) { - return ret; - } - BufferedReader in = null; - InputStreamReader fReader = null; - try { - fReader = new InputStreamReader(new FileInputStream( - new File(new File(procfsDir, pid), PROCFS_CMDLINE_FILE)), - Charsets.UTF_8); - } catch (FileNotFoundException f) { - // The process vanished in the interim! - return ret; - } - - in = new BufferedReader(fReader); - - try { - ret = in.readLine(); // only one line - if (ret == null) { - ret = "N/A"; - } else { - ret = ret.replace('\0', ' '); // Replace each null char with a space - if (ret.equals("")) { - // The cmdline might be empty because the process is swapped out or - // is a zombie. - ret = "N/A"; - } - } - } catch (IOException io) { - LOG.warn("Error reading the stream " + io); - ret = "N/A"; - } finally { - // Close the streams - try { - fReader.close(); - try { - in.close(); - } catch (IOException i) { - LOG.warn("Error closing the stream " + in); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader); - } - } - - return ret; - } - } -} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceCalculatorPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceCalculatorPlugin.java deleted file mode 100644 index 23846786c9f..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceCalculatorPlugin.java +++ /dev/null @@ -1,165 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * Plugin to calculate resource information on the system. - * - */ -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class ResourceCalculatorPlugin extends Configured { - - /** - * Obtain the total size of the virtual memory present in the system. - * - * @return virtual memory size in bytes. - */ - public abstract long getVirtualMemorySize(); - - /** - * Obtain the total size of the physical memory present in the system. - * - * @return physical memory size bytes. - */ - public abstract long getPhysicalMemorySize(); - - /** - * Obtain the total size of the available virtual memory present - * in the system. - * - * @return available virtual memory size in bytes. - */ - public abstract long getAvailableVirtualMemorySize(); - - /** - * Obtain the total size of the available physical memory present - * in the system. - * - * @return available physical memory size bytes. - */ - public abstract long getAvailablePhysicalMemorySize(); - - /** - * Obtain the total number of processors present on the system. - * - * @return number of processors - */ - public abstract int getNumProcessors(); - - /** - * Obtain the CPU frequency of on the system. - * - * @return CPU frequency in kHz - */ - public abstract long getCpuFrequency(); - - /** - * Obtain the cumulative CPU time since the system is on. - * - * @return cumulative CPU time in milliseconds - */ - public abstract long getCumulativeCpuTime(); - - /** - * Obtain the CPU usage % of the machine. Return -1 if it is unavailable - * - * @return CPU usage in % - */ - public abstract float getCpuUsage(); - - /** - * Obtain resource status used by current process tree. - */ - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract ProcResourceValues getProcResourceValues(); - - public static class ProcResourceValues { - private final long cumulativeCpuTime; - private final long physicalMemorySize; - private final long virtualMemorySize; - public ProcResourceValues(long cumulativeCpuTime, long physicalMemorySize, - long virtualMemorySize) { - this.cumulativeCpuTime = cumulativeCpuTime; - this.physicalMemorySize = physicalMemorySize; - this.virtualMemorySize = virtualMemorySize; - } - /** - * Obtain the physical memory size used by current process tree. - * @return physical memory size in bytes. - */ - public long getPhysicalMemorySize() { - return physicalMemorySize; - } - - /** - * Obtain the virtual memory size used by a current process tree. - * @return virtual memory size in bytes. - */ - public long getVirtualMemorySize() { - return virtualMemorySize; - } - - /** - * Obtain the cumulative CPU time used by a current process tree. - * @return cumulative CPU time in milliseconds - */ - public long getCumulativeCpuTime() { - return cumulativeCpuTime; - } - } - - /** - * Get the ResourceCalculatorPlugin from the class name and configure it. If - * class name is null, this method will try and return a memory calculator - * plugin available for this system. - * - * @param clazz class-name - * @param conf configure the plugin with this. - * @return ResourceCalculatorPlugin - */ - public static ResourceCalculatorPlugin getResourceCalculatorPlugin( - Class clazz, Configuration conf) { - - if (clazz != null) { - return ReflectionUtils.newInstance(clazz, conf); - } - - // No class given, try a os specific class - try { - String osName = System.getProperty("os.name"); - if (osName.startsWith("Linux")) { - return new LinuxResourceCalculatorPlugin(); - } - } catch (SecurityException se) { - // Failed to get Operating System name. - return null; - } - - // Not supported on this system. - return null; - } -} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java deleted file mode 100644 index 413e6152123..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java +++ /dev/null @@ -1,236 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Random; - -import junit.framework.TestCase; - -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin; -import org.junit.Test; - -/** - * A JUnit test to test {@link LinuxResourceCalculatorPlugin} - * Create the fake /proc/ information and verify the parsing and calculation - */ -public class TestLinuxResourceCalculatorPlugin extends TestCase { - /** - * LinuxResourceCalculatorPlugin with a fake timer - */ - static class FakeLinuxResourceCalculatorPlugin extends - LinuxResourceCalculatorPlugin { - - long currentTime = 0; - public FakeLinuxResourceCalculatorPlugin(String procfsMemFile, - String procfsCpuFile, - String procfsStatFile, - long jiffyLengthInMillis) { - super(procfsMemFile, procfsCpuFile, procfsStatFile, jiffyLengthInMillis); - } - @Override - long getCurrentTime() { - return currentTime; - } - public void advanceTime(long adv) { - currentTime += adv * jiffyLengthInMillis; - } - } - private static final FakeLinuxResourceCalculatorPlugin plugin; - private static String TEST_ROOT_DIR = new Path(System.getProperty( - "test.build.data", "/tmp")).toString().replace(' ', '+'); - private static final String FAKE_MEMFILE; - private static final String FAKE_CPUFILE; - private static final String FAKE_STATFILE; - private static final long FAKE_JIFFY_LENGTH = 10L; - static { - int randomNum = (new Random()).nextInt(1000000000); - FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum; - FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum; - FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum; - plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE, - FAKE_STATFILE, - FAKE_JIFFY_LENGTH); - } - static final String MEMINFO_FORMAT = - "MemTotal: %d kB\n" + - "MemFree: %d kB\n" + - "Buffers: 138244 kB\n" + - "Cached: 947780 kB\n" + - "SwapCached: 142880 kB\n" + - "Active: 3229888 kB\n" + - "Inactive: %d kB\n" + - "SwapTotal: %d kB\n" + - "SwapFree: %d kB\n" + - "Dirty: 122012 kB\n" + - "Writeback: 0 kB\n" + - "AnonPages: 2710792 kB\n" + - "Mapped: 24740 kB\n" + - "Slab: 132528 kB\n" + - "SReclaimable: 105096 kB\n" + - "SUnreclaim: 27432 kB\n" + - "PageTables: 11448 kB\n" + - "NFS_Unstable: 0 kB\n" + - "Bounce: 0 kB\n" + - "CommitLimit: 4125904 kB\n" + - "Committed_AS: 4143556 kB\n" + - "VmallocTotal: 34359738367 kB\n" + - "VmallocUsed: 1632 kB\n" + - "VmallocChunk: 34359736375 kB\n" + - "HugePages_Total: 0\n" + - "HugePages_Free: 0\n" + - "HugePages_Rsvd: 0\n" + - "Hugepagesize: 2048 kB"; - - static final String CPUINFO_FORMAT = - "processor : %s\n" + - "vendor_id : AuthenticAMD\n" + - "cpu family : 15\n" + - "model : 33\n" + - "model name : Dual Core AMD Opteron(tm) Processor 280\n" + - "stepping : 2\n" + - "cpu MHz : %f\n" + - "cache size : 1024 KB\n" + - "physical id : 0\n" + - "siblings : 2\n" + - "core id : 0\n" + - "cpu cores : 2\n" + - "fpu : yes\n" + - "fpu_exception : yes\n" + - "cpuid level : 1\n" + - "wp : yes\n" + - "flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov " + - "pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt lm " + - "3dnowext 3dnow pni lahf_lm cmp_legacy\n" + - "bogomips : 4792.41\n" + - "TLB size : 1024 4K pages\n" + - "clflush size : 64\n" + - "cache_alignment : 64\n" + - "address sizes : 40 bits physical, 48 bits virtual\n" + - "power management: ts fid vid ttp"; - - static final String STAT_FILE_FORMAT = - "cpu %d %d %d 1646495089 831319 48713 164346 0\n" + - "cpu0 15096055 30805 3823005 411456015 206027 13 14269 0\n" + - "cpu1 14760561 89890 6432036 408707910 456857 48074 130857 0\n" + - "cpu2 12761169 20842 3758639 413976772 98028 411 10288 0\n" + - "cpu3 12355207 47322 5789691 412354390 70406 213 8931 0\n" + - "intr 114648668 20010764 2 0 945665 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0\n" + - "ctxt 242017731764\n" + - "btime 1257808753\n" + - "processes 26414943\n" + - "procs_running 1\n" + - "procs_blocked 0\n"; - - /** - * Test parsing /proc/stat and /proc/cpuinfo - * @throws IOException - */ - @Test - public void testParsingProcStatAndCpuFile() throws IOException { - // Write fake /proc/cpuinfo file. - long numProcessors = 8; - long cpuFrequencyKHz = 2392781; - String fileContent = ""; - for (int i = 0; i < numProcessors; i++) { - fileContent += String.format(CPUINFO_FORMAT, i, cpuFrequencyKHz / 1000D) + - "\n"; - } - File tempFile = new File(FAKE_CPUFILE); - tempFile.deleteOnExit(); - FileWriter fWriter = new FileWriter(FAKE_CPUFILE); - fWriter.write(fileContent); - fWriter.close(); - assertEquals(plugin.getNumProcessors(), numProcessors); - assertEquals(plugin.getCpuFrequency(), cpuFrequencyKHz); - - // Write fake /proc/stat file. - long uTime = 54972994; - long nTime = 188860; - long sTime = 19803373; - tempFile = new File(FAKE_STATFILE); - tempFile.deleteOnExit(); - updateStatFile(uTime, nTime, sTime); - assertEquals(plugin.getCumulativeCpuTime(), - FAKE_JIFFY_LENGTH * (uTime + nTime + sTime)); - assertEquals(plugin.getCpuUsage(), (float)(LinuxResourceCalculatorPlugin.UNAVAILABLE)); - - // Advance the time and sample again to test the CPU usage calculation - uTime += 100L; - plugin.advanceTime(200L); - updateStatFile(uTime, nTime, sTime); - assertEquals(plugin.getCumulativeCpuTime(), - FAKE_JIFFY_LENGTH * (uTime + nTime + sTime)); - assertEquals(plugin.getCpuUsage(), 6.25F); - - // Advance the time and sample again. This time, we call getCpuUsage() only. - uTime += 600L; - plugin.advanceTime(300L); - updateStatFile(uTime, nTime, sTime); - assertEquals(plugin.getCpuUsage(), 25F); - - // Advance very short period of time (one jiffy length). - // In this case, CPU usage should not be updated. - uTime += 1L; - plugin.advanceTime(1L); - updateStatFile(uTime, nTime, sTime); - assertEquals(plugin.getCumulativeCpuTime(), - FAKE_JIFFY_LENGTH * (uTime + nTime + sTime)); - assertEquals(plugin.getCpuUsage(), 25F); // CPU usage is not updated. - } - - /** - * Write information to fake /proc/stat file - */ - private void updateStatFile(long uTime, long nTime, long sTime) - throws IOException { - FileWriter fWriter = new FileWriter(FAKE_STATFILE); - fWriter.write(String.format(STAT_FILE_FORMAT, uTime, nTime, sTime)); - fWriter.close(); - } - - /** - * Test parsing /proc/meminfo - * @throws IOException - */ - @Test - public void testParsingProcMemFile() throws IOException { - long memTotal = 4058864L; - long memFree = 99632L; - long inactive = 567732L; - long swapTotal = 2096472L; - long swapFree = 1818480L; - File tempFile = new File(FAKE_MEMFILE); - tempFile.deleteOnExit(); - FileWriter fWriter = new FileWriter(FAKE_MEMFILE); - fWriter.write(String.format(MEMINFO_FORMAT, - memTotal, memFree, inactive, swapTotal, swapFree)); - - fWriter.close(); - assertEquals(plugin.getAvailablePhysicalMemorySize(), - 1024L * (memFree + inactive)); - assertEquals(plugin.getAvailableVirtualMemorySize(), - 1024L * (memFree + inactive + swapFree)); - assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal); - assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal)); - } -} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/LinuxMemoryCalculatorPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/LinuxMemoryCalculatorPlugin.java deleted file mode 100644 index fe39b0cc7a5..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/LinuxMemoryCalculatorPlugin.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * Plugin to calculate virtual and physical memories on Linux systems. - * @deprecated - * Use {@link org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin} - * instead - */ -@Deprecated -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class LinuxMemoryCalculatorPlugin extends MemoryCalculatorPlugin { - private LinuxResourceCalculatorPlugin resourceCalculatorPlugin; - // Use everything from LinuxResourceCalculatorPlugin - public LinuxMemoryCalculatorPlugin() { - resourceCalculatorPlugin = new LinuxResourceCalculatorPlugin(); - } - - /** {@inheritDoc} */ - @Override - public long getPhysicalMemorySize() { - return resourceCalculatorPlugin.getPhysicalMemorySize(); - } - - /** {@inheritDoc} */ - @Override - public long getVirtualMemorySize() { - return resourceCalculatorPlugin.getVirtualMemorySize(); - } -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MemoryCalculatorPlugin.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MemoryCalculatorPlugin.java deleted file mode 100644 index cd0b1f5927b..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MemoryCalculatorPlugin.java +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.util.ReflectionUtils; - -/** - * Plugin to calculate virtual and physical memories on the system. - * @deprecated Use - * {@link org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin} - * instead - */ -@Deprecated -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class MemoryCalculatorPlugin extends Configured { - - /** - * Obtain the total size of the virtual memory present in the system. - * - * @return virtual memory size in bytes. - */ - public abstract long getVirtualMemorySize(); - - /** - * Obtain the total size of the physical memory present in the system. - * - * @return physical memory size bytes. - */ - public abstract long getPhysicalMemorySize(); - - /** - * Get the MemoryCalculatorPlugin from the class name and configure it. If - * class name is null, this method will try and return a memory calculator - * plugin available for this system. - * - * @param clazz class-name - * @param conf configure the plugin with this. - * @return MemoryCalculatorPlugin - */ - public static MemoryCalculatorPlugin getMemoryCalculatorPlugin( - Class clazz, Configuration conf) { - - if (clazz != null) { - return ReflectionUtils.newInstance(clazz, conf); - } - - // No class given, try a os specific class - try { - String osName = System.getProperty("os.name"); - if (osName.startsWith("Linux")) { - return new LinuxMemoryCalculatorPlugin(); - } - } catch (SecurityException se) { - // Failed to get Operating System name. - return null; - } - - // Not supported on this system. - return null; - } -} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java deleted file mode 100644 index a80ee67be52..00000000000 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java +++ /dev/null @@ -1,677 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.mapreduce.util; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Arrays; -import java.util.Random; -import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.StringUtils; -import org.apache.hadoop.util.Shell.ExitCodeException; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; - -import junit.framework.TestCase; - -/** - * A JUnit test to test ProcfsBasedProcessTree. - */ -public class TestProcfsBasedProcessTree extends TestCase { - - private static final Log LOG = LogFactory - .getLog(TestProcfsBasedProcessTree.class); - private static String TEST_ROOT_DIR = new Path(System.getProperty( - "test.build.data", "/tmp")).toString().replace(' ', '+'); - - private ShellCommandExecutor shexec = null; - private String pidFile, lowestDescendant; - private String shellScript; - private static final int N = 6; // Controls the RogueTask - - private class RogueTaskThread extends Thread { - public void run() { - try { - Vector args = new Vector(); - if(ProcessTree.isSetsidAvailable) { - args.add("setsid"); - } - args.add("bash"); - args.add("-c"); - args.add(" echo $$ > " + pidFile + "; sh " + - shellScript + " " + N + ";") ; - shexec = new ShellCommandExecutor(args.toArray(new String[0])); - shexec.execute(); - } catch (ExitCodeException ee) { - LOG.info("Shell Command exit with a non-zero exit code. This is" + - " expected as we are killing the subprocesses of the" + - " task intentionally. " + ee); - } catch (IOException ioe) { - LOG.info("Error executing shell command " + ioe); - } finally { - LOG.info("Exit code: " + shexec.getExitCode()); - } - } - } - - private String getRogueTaskPID() { - File f = new File(pidFile); - while (!f.exists()) { - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - break; - } - } - - // read from pidFile - return getPidFromPidFile(pidFile); - } - - public void testProcessTree() { - - try { - if (!ProcfsBasedProcessTree.isAvailable()) { - System.out - .println("ProcfsBasedProcessTree is not available on this system. Not testing"); - return; - } - } catch (Exception e) { - LOG.info(StringUtils.stringifyException(e)); - return; - } - // create shell script - Random rm = new Random(); - File tempFile = new File(TEST_ROOT_DIR, this.getName() + "_shellScript_" + - rm.nextInt() + ".sh"); - tempFile.deleteOnExit(); - shellScript = TEST_ROOT_DIR + File.separator + tempFile.getName(); - - // create pid file - tempFile = new File(TEST_ROOT_DIR, this.getName() + "_pidFile_" + - rm.nextInt() + ".pid"); - tempFile.deleteOnExit(); - pidFile = TEST_ROOT_DIR + File.separator + tempFile.getName(); - - lowestDescendant = TEST_ROOT_DIR + File.separator + "lowestDescendantPidFile"; - - // write to shell-script - try { - FileWriter fWriter = new FileWriter(shellScript); - fWriter.write( - "# rogue task\n" + - "sleep 1\n" + - "echo hello\n" + - "if [ $1 -ne 0 ]\n" + - "then\n" + - " sh " + shellScript + " $(($1-1))\n" + - "else\n" + - " echo $$ > " + lowestDescendant + "\n" + - " while true\n do\n" + - " sleep 5\n" + - " done\n" + - "fi"); - fWriter.close(); - } catch (IOException ioe) { - LOG.info("Error: " + ioe); - return; - } - - Thread t = new RogueTaskThread(); - t.start(); - String pid = getRogueTaskPID(); - LOG.info("Root process pid: " + pid); - ProcfsBasedProcessTree p = new ProcfsBasedProcessTree(pid, - ProcessTree.isSetsidAvailable, - ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL); - p.updateProcessTree(); // initialize - LOG.info("ProcessTree: " + p.toString()); - - File leaf = new File(lowestDescendant); - //wait till lowest descendant process of Rougue Task starts execution - while (!leaf.exists()) { - try { - Thread.sleep(500); - } catch (InterruptedException ie) { - break; - } - } - - p.updateProcessTree(); // reconstruct - LOG.info("ProcessTree: " + p.toString()); - - // Get the process-tree dump - String processTreeDump = p.getProcessTreeDump(); - - // destroy the process and all its subprocesses - p.destroy(true/*in the background*/); - - if(ProcessTree.isSetsidAvailable) {// whole processtree should be gone - assertEquals(false, p.isAnyProcessInTreeAlive()); - } - else {// process should be gone - assertFalse("ProcessTree must have been gone", p.isAlive()); - } - - LOG.info("Process-tree dump follows: \n" + processTreeDump); - assertTrue("Process-tree dump doesn't start with a proper header", - processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " + - "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " + - "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n")); - for (int i = N; i >= 0; i--) { - String cmdLineDump = "\\|- [0-9]+ [0-9]+ [0-9]+ [0-9]+ \\(sh\\)" + - " [0-9]+ [0-9]+ [0-9]+ [0-9]+ sh " + shellScript + " " + i; - Pattern pat = Pattern.compile(cmdLineDump); - Matcher mat = pat.matcher(processTreeDump); - assertTrue("Process-tree dump doesn't contain the cmdLineDump of " + i - + "th process!", mat.find()); - } - - // Not able to join thread sometimes when forking with large N. - try { - t.join(2000); - LOG.info("RogueTaskThread successfully joined."); - } catch (InterruptedException ie) { - LOG.info("Interrupted while joining RogueTaskThread."); - } - - // ProcessTree is gone now. Any further calls should be sane. - p.updateProcessTree(); - assertFalse("ProcessTree must have been gone", p.isAlive()); - assertTrue("Cumulative vmem for the gone-process is " - + p.getCumulativeVmem() + " . It should be zero.", p - .getCumulativeVmem() == 0); - assertTrue(p.toString().equals("[ ]")); - } - - /** - * Get PID from a pid-file. - * - * @param pidFileName - * Name of the pid-file. - * @return the PID string read from the pid-file. Returns null if the - * pidFileName points to a non-existing file or if read fails from the - * file. - */ - public static String getPidFromPidFile(String pidFileName) { - BufferedReader pidFile = null; - FileReader fReader = null; - String pid = null; - - try { - fReader = new FileReader(pidFileName); - pidFile = new BufferedReader(fReader); - } catch (FileNotFoundException f) { - LOG.debug("PidFile doesn't exist : " + pidFileName); - return pid; - } - - try { - pid = pidFile.readLine(); - } catch (IOException i) { - LOG.error("Failed to read from " + pidFileName); - } finally { - try { - if (fReader != null) { - fReader.close(); - } - try { - if (pidFile != null) { - pidFile.close(); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + pidFile); - } - } catch (IOException i) { - LOG.warn("Error closing the stream " + fReader); - } - } - return pid; - } - - public static class ProcessStatInfo { - // sample stat in a single line : 3910 (gpm) S 1 3910 3910 0 -1 4194624 - // 83 0 0 0 0 0 0 0 16 0 1 0 7852 2408448 88 4294967295 134512640 - // 134590050 3220521392 3220520036 10975138 0 0 4096 134234626 - // 4294967295 0 0 17 1 0 0 - String pid; - String name; - String ppid; - String pgrpId; - String session; - String vmem = "0"; - String rssmemPage = "0"; - String utime = "0"; - String stime = "0"; - - public ProcessStatInfo(String[] statEntries) { - pid = statEntries[0]; - name = statEntries[1]; - ppid = statEntries[2]; - pgrpId = statEntries[3]; - session = statEntries[4]; - vmem = statEntries[5]; - if (statEntries.length > 6) { - rssmemPage = statEntries[6]; - } - if (statEntries.length > 7) { - utime = statEntries[7]; - stime = statEntries[8]; - } - } - - // construct a line that mimics the procfs stat file. - // all unused numerical entries are set to 0. - public String getStatLine() { - return String.format("%s (%s) S %s %s %s 0 0 0" + - " 0 0 0 0 %s %s 0 0 0 0 0 0 0 %s %s 0 0" + - " 0 0 0 0 0 0 0 0" + - " 0 0 0 0 0", - pid, name, ppid, pgrpId, session, - utime, stime, vmem, rssmemPage); - } - } - - /** - * A basic test that creates a few process directories and writes - * stat files. Verifies that the cpu time and memory is correctly - * computed. - * @throws IOException if there was a problem setting up the - * fake procfs directories or files. - */ - public void testCpuAndMemoryForProcessTree() throws IOException { - - // test processes - String[] pids = { "100", "200", "300", "400" }; - // create the fake procfs root directory. - File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); - - try { - setupProcfsRootDir(procfsRootDir); - setupPidDirs(procfsRootDir, pids); - - // create stat objects. - // assuming processes 100, 200, 300 are in tree and 400 is not. - ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; - procInfos[0] = new ProcessStatInfo(new String[] - {"100", "proc1", "1", "100", "100", "100000", "100", "1000", "200"}); - procInfos[1] = new ProcessStatInfo(new String[] - {"200", "proc2", "100", "100", "100", "200000", "200", "2000", "400"}); - procInfos[2] = new ProcessStatInfo(new String[] - {"300", "proc3", "200", "100", "100", "300000", "300", "3000", "600"}); - procInfos[3] = new ProcessStatInfo(new String[] - {"400", "proc4", "1", "400", "400", "400000", "400", "4000", "800"}); - - writeStatFiles(procfsRootDir, pids, procInfos); - - // crank up the process tree class. - ProcfsBasedProcessTree processTree = - new ProcfsBasedProcessTree("100", true, 100L, - procfsRootDir.getAbsolutePath()); - // build the process tree. - processTree.updateProcessTree(); - - // verify cumulative memory - assertEquals("Cumulative virtual memory does not match", 600000L, - processTree.getCumulativeVmem()); - - // verify rss memory - long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? - 600L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - assertEquals("Cumulative rss memory does not match", - cumuRssMem, processTree.getCumulativeRssmem()); - - // verify cumulative cpu time - long cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ? - 7200L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L; - assertEquals("Cumulative cpu time does not match", - cumuCpuTime, processTree.getCumulativeCpuTime()); - - // test the cpu time again to see if it cumulates - procInfos[0] = new ProcessStatInfo(new String[] - {"100", "proc1", "1", "100", "100", "100000", "100", "2000", "300"}); - procInfos[1] = new ProcessStatInfo(new String[] - {"200", "proc2", "100", "100", "100", "200000", "200", "3000", "500"}); - writeStatFiles(procfsRootDir, pids, procInfos); - - // build the process tree. - processTree.updateProcessTree(); - - // verify cumulative cpu time again - cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ? - 9400L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L; - assertEquals("Cumulative cpu time does not match", - cumuCpuTime, processTree.getCumulativeCpuTime()); - } finally { - FileUtil.fullyDelete(procfsRootDir); - } - } - - /** - * Tests that cumulative memory is computed only for - * processes older than a given age. - * @throws IOException if there was a problem setting up the - * fake procfs directories or files. - */ - public void testMemForOlderProcesses() throws IOException { - // initial list of processes - String[] pids = { "100", "200", "300", "400" }; - // create the fake procfs root directory. - File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); - - try { - setupProcfsRootDir(procfsRootDir); - setupPidDirs(procfsRootDir, pids); - - // create stat objects. - // assuming 100, 200 and 400 are in tree, 300 is not. - ProcessStatInfo[] procInfos = new ProcessStatInfo[4]; - procInfos[0] = new ProcessStatInfo(new String[] - {"100", "proc1", "1", "100", "100", "100000", "100"}); - procInfos[1] = new ProcessStatInfo(new String[] - {"200", "proc2", "100", "100", "100", "200000", "200"}); - procInfos[2] = new ProcessStatInfo(new String[] - {"300", "proc3", "1", "300", "300", "300000", "300"}); - procInfos[3] = new ProcessStatInfo(new String[] - {"400", "proc4", "100", "100", "100", "400000", "400"}); - - writeStatFiles(procfsRootDir, pids, procInfos); - - // crank up the process tree class. - ProcfsBasedProcessTree processTree = - new ProcfsBasedProcessTree("100", true, 100L, - procfsRootDir.getAbsolutePath()); - // build the process tree. - processTree.updateProcessTree(); - - // verify cumulative memory - assertEquals("Cumulative memory does not match", - 700000L, processTree.getCumulativeVmem()); - - // write one more process as child of 100. - String[] newPids = { "500" }; - setupPidDirs(procfsRootDir, newPids); - - ProcessStatInfo[] newProcInfos = new ProcessStatInfo[1]; - newProcInfos[0] = new ProcessStatInfo(new String[] - {"500", "proc5", "100", "100", "100", "500000", "500"}); - writeStatFiles(procfsRootDir, newPids, newProcInfos); - - // check memory includes the new process. - processTree.updateProcessTree(); - assertEquals("Cumulative vmem does not include new process", - 1200000L, processTree.getCumulativeVmem()); - long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? - 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - assertEquals("Cumulative rssmem does not include new process", - cumuRssMem, processTree.getCumulativeRssmem()); - - // however processes older than 1 iteration will retain the older value - assertEquals("Cumulative vmem shouldn't have included new process", - 700000L, processTree.getCumulativeVmem(1)); - cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? - 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - assertEquals("Cumulative rssmem shouldn't have included new process", - cumuRssMem, processTree.getCumulativeRssmem(1)); - - // one more process - newPids = new String[]{ "600" }; - setupPidDirs(procfsRootDir, newPids); - - newProcInfos = new ProcessStatInfo[1]; - newProcInfos[0] = new ProcessStatInfo(new String[] - {"600", "proc6", "100", "100", "100", "600000", "600"}); - writeStatFiles(procfsRootDir, newPids, newProcInfos); - - // refresh process tree - processTree.updateProcessTree(); - - // processes older than 2 iterations should be same as before. - assertEquals("Cumulative vmem shouldn't have included new processes", - 700000L, processTree.getCumulativeVmem(2)); - cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? - 700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - assertEquals("Cumulative rssmem shouldn't have included new processes", - cumuRssMem, processTree.getCumulativeRssmem(2)); - - // processes older than 1 iteration should not include new process, - // but include process 500 - assertEquals("Cumulative vmem shouldn't have included new processes", - 1200000L, processTree.getCumulativeVmem(1)); - cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ? - 1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L; - assertEquals("Cumulative rssmem shouldn't have included new processes", - cumuRssMem, processTree.getCumulativeRssmem(1)); - - // no processes older than 3 iterations, this should be 0 - assertEquals("Getting non-zero vmem for processes older than 3 iterations", - 0L, processTree.getCumulativeVmem(3)); - assertEquals("Getting non-zero rssmem for processes older than 3 iterations", - 0L, processTree.getCumulativeRssmem(3)); - } finally { - FileUtil.fullyDelete(procfsRootDir); - } - } - - /** - * Verifies ProcfsBasedProcessTree.checkPidPgrpidForMatch() in case of - * 'constructProcessInfo() returning null' by not writing stat file for the - * mock process - * @throws IOException if there was a problem setting up the - * fake procfs directories or files. - */ - public void testDestroyProcessTree() throws IOException { - // test process - String pid = "100"; - // create the fake procfs root directory. - File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); - - try { - setupProcfsRootDir(procfsRootDir); - - // crank up the process tree class. - ProcfsBasedProcessTree processTree = new ProcfsBasedProcessTree( - pid, true, 100L, procfsRootDir.getAbsolutePath()); - - // Let us not create stat file for pid 100. - assertTrue(ProcfsBasedProcessTree.checkPidPgrpidForMatch( - pid, procfsRootDir.getAbsolutePath())); - } finally { - FileUtil.fullyDelete(procfsRootDir); - } - } - - /** - * Test the correctness of process-tree dump. - * - * @throws IOException - */ - public void testProcessTreeDump() - throws IOException { - - String[] pids = { "100", "200", "300", "400", "500", "600" }; - - File procfsRootDir = new File(TEST_ROOT_DIR, "proc"); - - try { - setupProcfsRootDir(procfsRootDir); - setupPidDirs(procfsRootDir, pids); - - int numProcesses = pids.length; - // Processes 200, 300, 400 and 500 are descendants of 100. 600 is not. - ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses]; - procInfos[0] = new ProcessStatInfo(new String[] { - "100", "proc1", "1", "100", "100", "100000", "100", "1000", "200"}); - procInfos[1] = new ProcessStatInfo(new String[] { - "200", "proc2", "100", "100", "100", "200000", "200", "2000", "400"}); - procInfos[2] = new ProcessStatInfo(new String[] { - "300", "proc3", "200", "100", "100", "300000", "300", "3000", "600"}); - procInfos[3] = new ProcessStatInfo(new String[] { - "400", "proc4", "200", "100", "100", "400000", "400", "4000", "800"}); - procInfos[4] = new ProcessStatInfo(new String[] { - "500", "proc5", "400", "100", "100", "400000", "400", "4000", "800"}); - procInfos[5] = new ProcessStatInfo(new String[] { - "600", "proc6", "1", "1", "1", "400000", "400", "4000", "800"}); - - String[] cmdLines = new String[numProcesses]; - cmdLines[0] = "proc1 arg1 arg2"; - cmdLines[1] = "proc2 arg3 arg4"; - cmdLines[2] = "proc3 arg5 arg6"; - cmdLines[3] = "proc4 arg7 arg8"; - cmdLines[4] = "proc5 arg9 arg10"; - cmdLines[5] = "proc6 arg11 arg12"; - - writeStatFiles(procfsRootDir, pids, procInfos); - writeCmdLineFiles(procfsRootDir, pids, cmdLines); - - ProcfsBasedProcessTree processTree = - new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir - .getAbsolutePath()); - // build the process tree. - processTree.updateProcessTree(); - - // Get the process-tree dump - String processTreeDump = processTree.getProcessTreeDump(); - - LOG.info("Process-tree dump follows: \n" + processTreeDump); - assertTrue("Process-tree dump doesn't start with a proper header", - processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " + - "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " + - "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n")); - for (int i = 0; i < 5; i++) { - ProcessStatInfo p = procInfos[i]; - assertTrue( - "Process-tree dump doesn't contain the cmdLineDump of process " - + p.pid, processTreeDump.contains("\t|- " + p.pid + " " - + p.ppid + " " + p.pgrpId + " " + p.session + " (" + p.name - + ") " + p.utime + " " + p.stime + " " + p.vmem + " " - + p.rssmemPage + " " + cmdLines[i])); - } - - // 600 should not be in the dump - ProcessStatInfo p = procInfos[5]; - assertFalse( - "Process-tree dump shouldn't contain the cmdLineDump of process " - + p.pid, processTreeDump.contains("\t|- " + p.pid + " " + p.ppid - + " " + p.pgrpId + " " + p.session + " (" + p.name + ") " - + p.utime + " " + p.stime + " " + p.vmem + " " + cmdLines[5])); - } finally { - FileUtil.fullyDelete(procfsRootDir); - } - } - - /** - * Create a directory to mimic the procfs file system's root. - * @param procfsRootDir root directory to create. - * @throws IOException if could not delete the procfs root directory - */ - public static void setupProcfsRootDir(File procfsRootDir) { - // cleanup any existing process root dir. - if (procfsRootDir.exists()) { - assertTrue(FileUtil.fullyDelete(procfsRootDir)); - } - - // create afresh - assertTrue(procfsRootDir.mkdirs()); - } - - /** - * Create PID directories under the specified procfs root directory - * @param procfsRootDir root directory of procfs file system - * @param pids the PID directories to create. - * @throws IOException If PID dirs could not be created - */ - public static void setupPidDirs(File procfsRootDir, String[] pids) - throws IOException { - for (String pid : pids) { - File pidDir = new File(procfsRootDir, pid); - pidDir.mkdir(); - if (!pidDir.exists()) { - throw new IOException ("couldn't make process directory under " + - "fake procfs"); - } else { - LOG.info("created pid dir"); - } - } - } - - /** - * Write stat files under the specified pid directories with data - * setup in the corresponding ProcessStatInfo objects - * @param procfsRootDir root directory of procfs file system - * @param pids the PID directories under which to create the stat file - * @param procs corresponding ProcessStatInfo objects whose data should be - * written to the stat files. - * @throws IOException if stat files could not be written - */ - public static void writeStatFiles(File procfsRootDir, String[] pids, - ProcessStatInfo[] procs) throws IOException { - for (int i=0; iA {@link ResourceUsageEmulatorPlugin} that emulates the cumulative CPU @@ -166,7 +166,7 @@ implements ResourceUsageEmulatorPlugin { */ public void calibrate(ResourceCalculatorPlugin monitor, long totalCpuUsage) { - long initTime = monitor.getProcResourceValues().getCumulativeCpuTime(); + long initTime = monitor.getCumulativeCpuTime(); long defaultLoopSize = 0; long finalTime = initTime; @@ -175,7 +175,7 @@ implements ResourceUsageEmulatorPlugin { while (finalTime - initTime < 100) { // 100 ms ++defaultLoopSize; performUnitComputation(); //perform unit computation - finalTime = monitor.getProcResourceValues().getCumulativeCpuTime(); + finalTime = monitor.getCumulativeCpuTime(); } long referenceRuntime = finalTime - initTime; @@ -230,7 +230,7 @@ implements ResourceUsageEmulatorPlugin { } private synchronized long getCurrentCPUUsage() { - return monitor.getProcResourceValues().getCumulativeCpuTime(); + return monitor.getCumulativeCpuTime(); } @Override diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java index bff45fc5454..593c1a462bf 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java @@ -20,7 +20,7 @@ package org.apache.hadoop.mapred.gridmix.emulators.resourceusage; import java.io.IOException; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; import org.apache.hadoop.conf.Configuration; diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java index b10ad434609..38095800111 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java @@ -23,9 +23,9 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** *

This is the driver class for managing all the resource usage emulators. diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java index 47941ccfffb..b9971dc2af5 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java @@ -21,8 +21,8 @@ import java.io.IOException; import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.gridmix.Progressive; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** *

A {@link ResourceUsageEmulatorPlugin} that emulates the total heap diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/DummyResourceCalculatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java similarity index 76% rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/DummyResourceCalculatorPlugin.java rename to hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java index 5fd3f9f92e4..1e17f2a22f0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/DummyResourceCalculatorPlugin.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java @@ -16,18 +16,17 @@ * limitations under the License. */ -package org.apache.hadoop.mapred; +package org.apache.hadoop.mapred.gridmix; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** - * Plugin class to test resource information reported by TT. Use - * configuration items {@link #MAXVMEM_TESTING_PROPERTY} and - * {@link #MAXPMEM_TESTING_PROPERTY} to tell TT the total vmem and the total - * pmem. Use configuration items {@link #NUM_PROCESSORS}, - * {@link #CPU_FREQUENCY}, {@link #CUMULATIVE_CPU_TIME} and {@link #CPU_USAGE} - * to tell TT the CPU information. + * Plugin class to test resource information reported by NM. Use configuration + * items {@link #MAXVMEM_TESTING_PROPERTY} and {@link #MAXPMEM_TESTING_PROPERTY} + * to tell NM the total vmem and the total pmem. Use configuration items + * {@link #NUM_PROCESSORS}, {@link #CPU_FREQUENCY}, {@link #CUMULATIVE_CPU_TIME} + * and {@link #CPU_USAGE} to tell TT the CPU information. */ @InterfaceAudience.Private public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin { @@ -48,15 +47,14 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin { public static final String CUMULATIVE_CPU_TIME = "mapred.tasktracker.cumulativecputime.testing"; /** CPU usage percentage for testing */ - public static final String CPU_USAGE = - "mapred.tasktracker.cpuusage.testing"; + public static final String CPU_USAGE = "mapred.tasktracker.cpuusage.testing"; /** process cumulative CPU usage time for testing */ public static final String PROC_CUMULATIVE_CPU_TIME = "mapred.tasktracker.proccumulativecputime.testing"; - /** process pmem for testing*/ + /** process pmem for testing */ public static final String PROC_PMEM_TESTING_PROPERTY = "mapred.tasktracker.procpmem.testing"; - /** process vmem for testing*/ + /** process vmem for testing */ public static final String PROC_VMEM_TESTING_PROPERTY = "mapred.tasktracker.procvmem.testing"; @@ -107,12 +105,4 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin { public float getCpuUsage() { return getConf().getFloat(CPU_USAGE, -1); } - - @Override - public ProcResourceValues getProcResourceValues() { - long cpuTime = getConf().getLong(PROC_CUMULATIVE_CPU_TIME, -1); - long pMem = getConf().getLong(PROC_PMEM_TESTING_PROPERTY, -1); - long vMem = getConf().getLong(PROC_VMEM_TESTING_PROPERTY, -1); - return new ProcResourceValues(cpuTime, pMem, vMem); - } } diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java index 486165d9efa..7ec3c93d367 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java @@ -23,7 +23,6 @@ import static org.junit.Assert.*; import java.io.IOException; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob; import org.apache.hadoop.mapred.gridmix.TestHighRamJob.DummyGridmixJob; @@ -32,8 +31,8 @@ import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEm import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEmulatorPlugin.DefaultHeapUsageEmulator; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** * Test Gridmix memory emulation. diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java index 9874be3229e..f14087bf3af 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java @@ -31,14 +31,13 @@ import org.apache.hadoop.mapreduce.TaskInputOutputContext; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig; import org.apache.hadoop.mapreduce.task.MapContextImpl; -import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; -import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin; import org.apache.hadoop.mapred.gridmix.LoadJob.ResourceUsageMatcherRunner; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageEmulatorPlugin; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageMatcher; import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin.DefaultCpuUsageEmulator; +import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; /** * Test Gridmix's resource emulator framework and supported plugins. @@ -242,16 +241,6 @@ public class TestResourceUsageEmulators { public long getCumulativeCpuTime() { return core.getCpuUsage(); } - - /** - * Returns a {@link ProcResourceValues} with cumulative cpu usage - * computed using {@link #getCumulativeCpuTime()}. - */ - @Override - public ProcResourceValues getProcResourceValues() { - long usageValue = getCumulativeCpuTime(); - return new ProcResourceValues(usageValue, -1, -1); - } } /** From 66e90b205a079c7056bd85e6c5e3dbb2a96e7461 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 26 Mar 2013 18:20:36 +0000 Subject: [PATCH 16/37] YARN-440. Flatten RegisterNodeManagerResponse. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461256 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../RegisterNodeManagerResponse.java | 13 +- .../pb/RegisterNodeManagerResponsePBImpl.java | 72 ++++++---- .../api/records/RegistrationResponse.java | 29 ---- .../impl/pb/RegistrationResponsePBImpl.java | 133 ------------------ .../proto/yarn_server_common_protos.proto | 6 - .../yarn_server_common_service_protos.proto | 4 +- .../nodemanager/NodeStatusUpdaterImpl.java | 13 +- .../server/nodemanager/LocalRMInterface.java | 3 - .../nodemanager/MockNodeStatusUpdater.java | 5 - .../nodemanager/TestNodeStatusUpdater.java | 14 +- .../ResourceTrackerService.java | 11 +- .../yarn/server/resourcemanager/MockNM.java | 8 +- .../server/resourcemanager/NodeManager.java | 3 +- .../TestResourceTrackerService.java | 2 +- .../resourcetracker/TestNMExpiry.java | 2 +- .../hadoop/yarn/server/MiniYARNCluster.java | 4 +- .../yarn/server/TestRMNMSecretKeys.java | 4 +- 18 files changed, 82 insertions(+), 246 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c6d4b36d894..4c7bbc7c3ef 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -64,6 +64,8 @@ Release 2.0.5-beta - UNRELEASED YARN-439. Flatten NodeHeartbeatResponse. (Xuan Gong via sseth) + YARN-440. Flatten RegisterNodeManagerResponse. (Xuan Gong via sseth) + NEW FEATURES IMPROVEMENTS diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java index 45b877fd08a..1c1a9dd4a6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java @@ -18,11 +18,16 @@ package org.apache.hadoop.yarn.server.api.protocolrecords; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.api.records.NodeAction; public interface RegisterNodeManagerResponse { - public abstract RegistrationResponse getRegistrationResponse(); - - public abstract void setRegistrationResponse(RegistrationResponse registrationResponse); + MasterKey getMasterKey(); + + void setMasterKey(MasterKey secretKey); + + NodeAction getNodeAction(); + + void setNodeAction(NodeAction nodeAction); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java index 2b540a94d95..e8194af4053 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java @@ -20,12 +20,14 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; import org.apache.hadoop.yarn.api.records.ProtoBase; -import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.RegistrationResponseProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerResponseProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; -import org.apache.hadoop.yarn.server.api.records.impl.pb.RegistrationResponsePBImpl; +import org.apache.hadoop.yarn.server.api.records.MasterKey; +import org.apache.hadoop.yarn.server.api.records.NodeAction; +import org.apache.hadoop.yarn.server.api.records.impl.pb.MasterKeyPBImpl; @@ -34,7 +36,7 @@ public class RegisterNodeManagerResponsePBImpl extends ProtoBase implements RegistrationResponse { - RegistrationResponseProto proto = RegistrationResponseProto.getDefaultInstance(); - RegistrationResponseProto.Builder builder = null; - boolean viaProto = false; - - private MasterKey masterKey = null; - - public RegistrationResponsePBImpl() { - builder = RegistrationResponseProto.newBuilder(); - } - - public RegistrationResponsePBImpl(RegistrationResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public RegistrationResponseProto getProto() { - - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToBuilder() { - if (this.masterKey != null) { - builder.setMasterKey(convertToProtoFormat(this.masterKey)); - } - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = RegistrationResponseProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public MasterKey getMasterKey() { - RegistrationResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.masterKey != null) { - return this.masterKey; - } - if (!p.hasMasterKey()) { - return null; - } - this.masterKey = convertFromProtoFormat(p.getMasterKey()); - return this.masterKey; - } - - @Override - public void setMasterKey(MasterKey masterKey) { - maybeInitBuilder(); - if (masterKey == null) - builder.clearMasterKey(); - this.masterKey = masterKey; - } - - @Override - public NodeAction getNodeAction() { - RegistrationResponseProtoOrBuilder p = viaProto ? proto : builder; - if(!p.hasNodeAction()) { - return null; - } - return convertFromProtoFormat(p.getNodeAction()); - } - - @Override - public void setNodeAction(NodeAction nodeAction) { - maybeInitBuilder(); - if (nodeAction == null) { - builder.clearNodeAction(); - return; - } - builder.setNodeAction(convertToProtoFormat(nodeAction)); - } - - private NodeAction convertFromProtoFormat(NodeActionProto p) { - return NodeAction.valueOf(p.name()); - } - - private NodeActionProto convertToProtoFormat(NodeAction t) { - return NodeActionProto.valueOf(t.name()); - } - - private MasterKeyPBImpl convertFromProtoFormat(MasterKeyProto p) { - return new MasterKeyPBImpl(p); - } - - private MasterKeyProto convertToProtoFormat(MasterKey t) { - return ((MasterKeyPBImpl)t).getProto(); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto index 4d1ce15fca8..89ec81c3ab9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto @@ -42,9 +42,3 @@ message MasterKeyProto { optional bytes bytes = 2; } -message RegistrationResponseProto { - optional MasterKeyProto master_key = 1; - optional NodeActionProto nodeAction = 2; -} - - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto index 30d0cfeff33..d713b94332c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto @@ -29,8 +29,10 @@ message RegisterNodeManagerRequestProto { optional int32 http_port = 3; optional ResourceProto resource = 4; } + message RegisterNodeManagerResponseProto { - optional RegistrationResponseProto registration_response = 1; + optional MasterKeyProto master_key = 1; + optional NodeActionProto nodeAction = 2; } message NodeHeartbeatRequestProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 3df4311b600..b59145a9bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -52,10 +52,10 @@ import org.apache.hadoop.yarn.server.api.ResourceTracker; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.service.AbstractService; @@ -234,7 +234,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements request.setHttpPort(this.httpPort); request.setResource(this.totalResource); request.setNodeId(this.nodeId); - RegistrationResponse regResponse; + RegisterNodeManagerResponse regNMResponse; while(true) { try { @@ -242,9 +242,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements LOG.info("Connecting to ResourceManager at " + this.rmAddress + ". current no. of attempts is " + rmRetryCount); this.resourceTracker = getRMClient(); - regResponse = - this.resourceTracker.registerNodeManager(request) - .getRegistrationResponse(); + regNMResponse = + this.resourceTracker.registerNodeManager(request); break; } catch(Throwable e) { LOG.warn("Trying to connect to ResourceManager, " + @@ -267,13 +266,13 @@ public class NodeStatusUpdaterImpl extends AbstractService implements } } // if the Resourcemanager instructs NM to shutdown. - if (NodeAction.SHUTDOWN.equals(regResponse.getNodeAction())) { + if (NodeAction.SHUTDOWN.equals(regNMResponse.getNodeAction())) { throw new YarnException( "Recieved SHUTDOWN signal from Resourcemanager ,Registration of NodeManager failed"); } if (UserGroupInformation.isSecurityEnabled()) { - MasterKey masterKey = regResponse.getMasterKey(); + MasterKey masterKey = regNMResponse.getMasterKey(); // do this now so that its set before we start heartbeating to RM LOG.info("Security enabled - updating secret keys now"); // It is expected that status updater is started by this point and diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java index a16f72339e5..e5fbf3fe8c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/LocalRMInterface.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; public class LocalRMInterface implements ResourceTracker { @@ -34,9 +33,7 @@ public class LocalRMInterface implements ResourceTracker { @Override public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnRemoteException { - RegistrationResponse registrationResponse = recordFactory.newRecordInstance(RegistrationResponse.class); RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class); - response.setRegistrationResponse(registrationResponse); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java index bf46cde02f2..0e92d893491 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.NodeStatus; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; /** @@ -62,12 +61,8 @@ public class MockNodeStatusUpdater extends NodeStatusUpdaterImpl { @Override public RegisterNodeManagerResponse registerNodeManager( RegisterNodeManagerRequest request) throws YarnRemoteException { - RegistrationResponse regResponse = recordFactory - .newRecordInstance(RegistrationResponse.class); - RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); - response.setRegistrationResponse(regResponse); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index afeb0302422..a12de1da65f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -123,12 +122,9 @@ public class TestNodeStatusUpdater { Assert.assertEquals(NetUtils.getHostPortString(expected), nodeId.toString()); Assert.assertEquals(5 * 1024, resource.getMemory()); registeredNodes.add(nodeId); - RegistrationResponse regResponse = recordFactory - .newRecordInstance(RegistrationResponse.class); RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); - response.setRegistrationResponse(regResponse); return response; } @@ -320,10 +316,7 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); - RegistrationResponse regResponse = recordFactory - .newRecordInstance(RegistrationResponse.class); - regResponse.setNodeAction(registerNodeAction ); - response.setRegistrationResponse(regResponse); + response.setNodeAction(registerNodeAction ); return response; } @Override @@ -358,10 +351,7 @@ public class TestNodeStatusUpdater { RegisterNodeManagerResponse response = recordFactory.newRecordInstance(RegisterNodeManagerResponse.class); - RegistrationResponse regResponse = - recordFactory.newRecordInstance(RegistrationResponse.class); - regResponse.setNodeAction(registerNodeAction); - response.setRegistrationResponse(regResponse); + response.setNodeAction(registerNodeAction); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 2ca1360eb9d..815df2fe403 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeStatus; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; @@ -150,22 +149,19 @@ public class ResourceTrackerService extends AbstractService implements RegisterNodeManagerResponse response = recordFactory .newRecordInstance(RegisterNodeManagerResponse.class); - RegistrationResponse regResponse = recordFactory - .newRecordInstance(RegistrationResponse.class); // Check if this node is a 'valid' node if (!this.nodesListManager.isValidNode(host)) { LOG.info("Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager."); - regResponse.setNodeAction(NodeAction.SHUTDOWN); - response.setRegistrationResponse(regResponse); + response.setNodeAction(NodeAction.SHUTDOWN); return response; } if (isSecurityEnabled()) { MasterKey nextMasterKeyForNode = this.containerTokenSecretManager.getCurrentKey(); - regResponse.setMasterKey(nextMasterKeyForNode); + response.setMasterKey(nextMasterKeyForNode); } RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, @@ -188,8 +184,7 @@ public class ResourceTrackerService extends AbstractService implements + " httpPort: " + httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId " + nodeId); - regResponse.setNodeAction(NodeAction.NORMAL); - response.setRegistrationResponse(regResponse); + response.setNodeAction(NodeAction.NORMAL); return response; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index e22aa6ca7e0..c85f233ee18 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -35,9 +35,9 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeStatus; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.Records; @@ -79,7 +79,7 @@ public class MockNM { nodeHeartbeat(conts, true); } - public RegistrationResponse registerNode() throws Exception { + public RegisterNodeManagerResponse registerNode() throws Exception { RegisterNodeManagerRequest req = Records.newRecord( RegisterNodeManagerRequest.class); req.setNodeId(nodeId); @@ -87,8 +87,8 @@ public class MockNM { Resource resource = Records.newRecord(Resource.class); resource.setMemory(memory); req.setResource(resource); - RegistrationResponse registrationResponse = - resourceTracker.registerNodeManager(req).getRegistrationResponse(); + RegisterNodeManagerResponse registrationResponse = + resourceTracker.registerNodeManager(req); this.currentMasterKey = registrationResponse.getMasterKey(); return registrationResponse; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java index 610ec0b3f7a..170938d66d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java @@ -96,8 +96,7 @@ public class NodeManager implements ContainerManager { request.setNodeId(this.nodeId); request.setResource(capability); request.setNodeId(this.nodeId); - resourceTrackerService.registerNodeManager(request) - .getRegistrationResponse(); + resourceTrackerService.registerNodeManager(request); this.schedulerNode = new FiCaSchedulerNode(rmContext.getRMNodes().get( this.nodeId)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index cf679137f35..f969c6c888d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -241,7 +241,7 @@ public class TestResourceTrackerService { req.setHttpPort(1234); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req); - Assert.assertEquals(NodeAction.SHUTDOWN,response.getRegistrationResponse().getNodeAction()); + Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction()); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index 8ca0b43c35f..bf985d3e053 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -166,7 +166,7 @@ public class TestNMExpiry { request3.setHttpPort(0); request3.setResource(capability); resourceTrackerService - .registerNodeManager(request3).getRegistrationResponse(); + .registerNodeManager(request3); /* test to see if hostanme 3 does not expire */ stopT = false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 68a00362d3f..8ab0099b669 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -377,9 +377,7 @@ public class MiniYARNCluster extends CompositeService { RegisterNodeManagerResponse response = recordFactory. newRecordInstance(RegisterNodeManagerResponse.class); try { - response.setRegistrationResponse(rt - .registerNodeManager(request) - .getRegistrationResponse()); + response = rt.registerNodeManager(request); } catch (IOException ioe) { LOG.info("Exception in node registration from " + request.getNodeId().toString(), ioe); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java index 5bc2f7ad0c9..9d0fb0c6cb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java @@ -28,8 +28,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; +import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.MasterKey; -import org.apache.hadoop.yarn.server.api.records.RegistrationResponse; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -62,7 +62,7 @@ public class TestRMNMSecretKeys { rm.start(); MockNM nm = new MockNM("host:1234", 3072, rm.getResourceTrackerService()); - RegistrationResponse registrationResponse = nm.registerNode(); + RegisterNodeManagerResponse registrationResponse = nm.registerNode(); MasterKey masterKey = registrationResponse.getMasterKey(); Assert.assertNotNull("Registration should cause a key-update!", masterKey); dispatcher.await(); From af7b7667f52869d88fd99738ea7e1affeccb60ce Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 26 Mar 2013 18:57:47 +0000 Subject: [PATCH 17/37] HADOOP-9430. TestSSLFactory fails on IBM JVM. Contributed by Amir Sanjar. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461268 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java | 4 +++- .../apache/hadoop/security/ssl/ReloadingX509TrustManager.java | 4 ++-- .../main/java/org/apache/hadoop/security/ssl/SSLFactory.java | 3 +++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f25f91cf2e7..ad2b336fb38 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -592,6 +592,8 @@ Release 2.0.5-beta - UNRELEASED HADOOP-9299. kerberos name resolution is kicking in even when kerberos is not configured (daryn) + HADOOP-9430. TestSSLFactory fails on IBM JVM. (Amir Sanjar via suresh) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java index 9560caf7ed2..c6a7c6d5eec 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java @@ -164,7 +164,9 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory { } else { keystore.load(null, null); } - KeyManagerFactory keyMgrFactory = KeyManagerFactory.getInstance("SunX509"); + KeyManagerFactory keyMgrFactory = KeyManagerFactory + .getInstance(SSLFactory.SSLCERTIFICATE); + keyMgrFactory.init(keystore, (keystorePassword != null) ? keystorePassword.toCharArray() : null); keyManagers = keyMgrFactory.getKeyManagers(); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java index 58cdf00175f..1b2494093de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java @@ -169,8 +169,8 @@ public final class ReloadingX509TrustManager in.close(); } - TrustManagerFactory trustManagerFactory = - TrustManagerFactory.getInstance("SunX509"); + TrustManagerFactory trustManagerFactory = + TrustManagerFactory.getInstance(SSLFactory.SSLCERTIFICATE); trustManagerFactory.init(ks); TrustManager[] trustManagers = trustManagerFactory.getTrustManagers(); for (TrustManager trustManager1 : trustManagers) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java index 4234e3fc766..7f82f6aab3c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java @@ -58,6 +58,9 @@ public class SSLFactory implements ConnectionConfigurator { "hadoop.ssl.client.conf"; public static final String SSL_SERVER_CONF_KEY = "hadoop.ssl.server.conf"; + private static final boolean IBMJAVA = + System.getProperty("java.vendor").contains("IBM"); + public static final String SSLCERTIFICATE = IBMJAVA?"ibmX509":"SunX509"; public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false; From fdf1e6e07e26e0659442b787da4d0f26676c2293 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 26 Mar 2013 23:01:03 +0000 Subject: [PATCH 18/37] HDFS-4635. Move BlockManager#computeCapacity to LightWeightGSet. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461364 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../server/blockmanagement/BlockManager.java | 1 + .../server/blockmanagement/BlocksMap.java | 31 +------- .../hadoop/hdfs/util/LightWeightGSet.java | 53 +++++++++++++ .../org/apache/hadoop/hdfs/util/TestGSet.java | 78 +++++++++++++++++++ 5 files changed, 136 insertions(+), 29 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index aff2f516f22..57994c95d8f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -369,6 +369,8 @@ Release 2.0.5-beta - UNRELEASED HDFS-4246. The exclude node list should be more forgiving, for each output stream. (harsh via atm) + HDFS-4635. Move BlockManager#computeCapacity to LightWeightGSet. (suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index e411d5e24a4..8a22c55f389 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -235,6 +235,7 @@ public class BlockManager { heartbeatManager = datanodeManager.getHeartbeatManager(); invalidateBlocks = new InvalidateBlocks(datanodeManager); + // Compute the map capacity by allocating 2% of total memory blocksMap = new BlocksMap(DEFAULT_MAP_LOAD_FACTOR); blockplacement = BlockPlacementPolicy.getInstance( conf, stats, datanodeManager.getNetworkTopology()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index ef21dc3a332..dbfcaa70138 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@ -60,38 +60,11 @@ class BlocksMap { private GSet blocks; BlocksMap(final float loadFactor) { - this.capacity = computeCapacity(); + // Use 2% of total memory to size the GSet capacity + this.capacity = LightWeightGSet.computeCapacity(2.0, "BlocksMap"); this.blocks = new LightWeightGSet(capacity); } - /** - * Let t = 2% of max memory. - * Let e = round(log_2 t). - * Then, we choose capacity = 2^e/(size of reference), - * unless it is outside the close interval [1, 2^30]. - */ - private static int computeCapacity() { - //VM detection - //See http://java.sun.com/docs/hotspot/HotSpotFAQ.html#64bit_detection - final String vmBit = System.getProperty("sun.arch.data.model"); - - //2% of max memory - final double twoPC = Runtime.getRuntime().maxMemory()/50.0; - - //compute capacity - final int e1 = (int)(Math.log(twoPC)/Math.log(2.0) + 0.5); - final int e2 = e1 - ("32".equals(vmBit)? 2: 3); - final int exponent = e2 < 0? 0: e2 > 30? 30: e2; - final int c = 1 << exponent; - - if (LightWeightGSet.LOG.isDebugEnabled()) { - LightWeightGSet.LOG.debug("VM type = " + vmBit + "-bit"); - LightWeightGSet.LOG.debug("2% max memory = " + twoPC/(1 << 20) + " MB"); - LightWeightGSet.LOG.debug("capacity = 2^" + exponent - + " = " + c + " entries"); - } - return c; - } void close() { // Empty blocks once GSet#clear is implemented (HDFS-3940) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightGSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightGSet.java index cb0412d0344..5ab9a8cb355 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightGSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightGSet.java @@ -24,8 +24,11 @@ import java.util.Iterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.HadoopIllegalArgumentException; +import com.google.common.annotations.VisibleForTesting; + /** * A low memory footprint {@link GSet} implementation, * which uses an array for storing the elements @@ -285,4 +288,54 @@ public class LightWeightGSet implements GSet { throw new UnsupportedOperationException("Remove is not supported."); } } + + /** + * Let t = percentage of max memory. + * Let e = round(log_2 t). + * Then, we choose capacity = 2^e/(size of reference), + * unless it is outside the close interval [1, 2^30]. + */ + public static int computeCapacity(double percentage, String mapName) { + return computeCapacity(Runtime.getRuntime().maxMemory(), percentage, + mapName); + } + + @VisibleForTesting + static int computeCapacity(long maxMemory, double percentage, + String mapName) { + if (percentage > 100.0 || percentage < 0.0) { + throw new HadoopIllegalArgumentException("Percentage " + percentage + + " must be greater than or equal to 0 " + + " and less than or equal to 100"); + } + if (maxMemory < 0) { + throw new HadoopIllegalArgumentException("Memory " + maxMemory + + " must be greater than or equal to 0"); + } + if (percentage == 0.0 || maxMemory == 0) { + return 0; + } + //VM detection + //See http://java.sun.com/docs/hotspot/HotSpotFAQ.html#64bit_detection + final String vmBit = System.getProperty("sun.arch.data.model"); + + //Percentage of max memory + final double percentDivisor = 100.0/percentage; + final double percentMemory = maxMemory/percentDivisor; + + //compute capacity + final int e1 = (int)(Math.log(percentMemory)/Math.log(2.0) + 0.5); + final int e2 = e1 - ("32".equals(vmBit)? 2: 3); + final int exponent = e2 < 0? 0: e2 > 30? 30: e2; + final int c = 1 << exponent; + + if (LightWeightGSet.LOG.isDebugEnabled()) { + LOG.debug("Computing capacity for map " + mapName); + LOG.debug("VM type = " + vmBit + "-bit"); + LOG.debug(percentage + "% max memory = " + + StringUtils.TraditionalBinaryPrefix.long2String(maxMemory, "B", 1)); + LOG.debug("capacity = 2^" + exponent + " = " + c + " entries"); + } + return c; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestGSet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestGSet.java index ed2007775fe..971d538b272 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestGSet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestGSet.java @@ -21,6 +21,7 @@ import java.util.ConcurrentModificationException; import java.util.Iterator; import java.util.Random; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.util.Time; import org.junit.Assert; import org.junit.Test; @@ -452,4 +453,81 @@ public class TestGSet { next = e; } } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + * with invalid percent less than 0. + */ + @Test(expected=HadoopIllegalArgumentException.class) + public void testComputeCapacityNegativePercent() { + LightWeightGSet.computeCapacity(1024, -1.0, "testMap"); + } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + * with invalid percent greater than 100. + */ + @Test(expected=HadoopIllegalArgumentException.class) + public void testComputeCapacityInvalidPercent() { + LightWeightGSet.computeCapacity(1024, 101.0, "testMap"); + } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + * with invalid negative max memory + */ + @Test(expected=HadoopIllegalArgumentException.class) + public void testComputeCapacityInvalidMemory() { + LightWeightGSet.computeCapacity(-1, 50.0, "testMap"); + } + + private static boolean isPowerOfTwo(int num) { + return num == 0 || (num > 0 && Integer.bitCount(num) == 1); + } + + /** Return capacity as percentage of total memory */ + private static int getPercent(long total, int capacity) { + // Reference size in bytes + double referenceSize = + System.getProperty("sun.arch.data.model").equals("32") ? 4.0 : 8.0; + return (int)(((capacity * referenceSize)/total) * 100.0); + } + + /** Return capacity as percentage of total memory */ + private static void testCapacity(long maxMemory, double percent) { + int capacity = LightWeightGSet.computeCapacity(maxMemory, percent, "map"); + LightWeightGSet.LOG.info("Validating - total memory " + maxMemory + " percent " + + percent + " returned capacity " + capacity); + // Returned capacity is zero or power of two + Assert.assertTrue(isPowerOfTwo(capacity)); + + // Ensure the capacity returned is the nearest to the asked perecentage + int capacityPercent = getPercent(maxMemory, capacity); + if (capacityPercent == percent) { + return; + } else if (capacityPercent > percent) { + Assert.assertTrue(getPercent(maxMemory, capacity * 2) > percent); + } else { + Assert.assertTrue(getPercent(maxMemory, capacity / 2) < percent); + } + } + + /** + * Test for {@link LightWeightGSet#computeCapacity(double, String)} + */ + @Test + public void testComputeCapacity() { + // Tests for boundary conditions where percent or memory are zero + testCapacity(0, 0.0); + testCapacity(100, 0.0); + testCapacity(0, 100.0); + + // Compute capacity for some 100 random max memory and percentage + Random r = new Random(); + for (int i = 0; i < 100; i++) { + long maxMemory = r.nextInt(Integer.MAX_VALUE); + double percent = r.nextInt(101); + testCapacity(maxMemory, percent); + } + } } From 5319818487d5c139de06155834deecb18c10b7a1 Mon Sep 17 00:00:00 2001 From: Luke Lu Date: Tue, 26 Mar 2013 23:29:09 +0000 Subject: [PATCH 19/37] HADOOP-9194. RPC Support for QoS. (Junping Du via llu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461370 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../java/org/apache/hadoop/ipc/Client.java | 59 +++++++++++++++---- .../main/java/org/apache/hadoop/ipc/RPC.java | 2 +- .../java/org/apache/hadoop/ipc/Server.java | 31 +++++++++- .../java/org/apache/hadoop/ipc/TestIPC.java | 45 +++++++++++++- 5 files changed, 123 insertions(+), 16 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ad2b336fb38..a1ac0440be1 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -17,6 +17,8 @@ Trunk (Unreleased) HADOOP-9380 Add totalLength to rpc response (sanjay Radia) + HADOOP-9194. RPC Support for QoS. (Junping Du via llu) + NEW FEATURES HADOOP-8561. Introduce HADOOP_PROXY_USER for secure impersonation in child diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java index 5294aa3b94f..986b1a2c154 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java @@ -257,6 +257,7 @@ public class Client { private final ConnectionId remoteId; // connection id private AuthMethod authMethod; // authentication method private Token token; + private int serviceClass; private SaslRpcClient saslRpcClient; private Socket socket = null; // connected socket @@ -279,7 +280,7 @@ public class Client { private final Object sendRpcRequestLock = new Object(); - public Connection(ConnectionId remoteId) throws IOException { + public Connection(ConnectionId remoteId, int serviceClass) throws IOException { this.remoteId = remoteId; this.server = remoteId.getAddress(); if (server.isUnresolved()) { @@ -296,6 +297,7 @@ public class Client { this.tcpNoDelay = remoteId.getTcpNoDelay(); this.doPing = remoteId.getDoPing(); this.pingInterval = remoteId.getPingInterval(); + this.serviceClass = serviceClass; if (LOG.isDebugEnabled()) { LOG.debug("The ping interval is " + this.pingInterval + " ms."); } @@ -747,7 +749,9 @@ public class Client { * +----------------------------------+ * | "hrpc" 4 bytes | * +----------------------------------+ - * | Version (1 bytes) | + * | Version (1 byte) | + * +----------------------------------+ + * | Service Class (1 byte) | * +----------------------------------+ * | Authmethod (1 byte) | * +----------------------------------+ @@ -760,6 +764,7 @@ public class Client { // Write out the header, version and authentication method out.write(Server.HEADER.array()); out.write(Server.CURRENT_VERSION); + out.write(serviceClass); authMethod.write(out); Server.IpcSerializationType.PROTOBUF.write(out); out.flush(); @@ -1179,19 +1184,33 @@ public class Client { /** - * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress, + * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress, * Class, UserGroupInformation, int, Configuration)} * except that rpcKind is writable. */ - public Writable call(Writable param, InetSocketAddress addr, + public Writable call(Writable param, InetSocketAddress addr, Class protocol, UserGroupInformation ticket, - int rpcTimeout, Configuration conf) + int rpcTimeout, Configuration conf) throws InterruptedException, IOException { - ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol, + ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol, ticket, rpcTimeout, conf); return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId); } + /** + * Same as {@link #call(Writable, InetSocketAddress, + * Class, UserGroupInformation, int, Configuration)} + * except that specifying serviceClass. + */ + public Writable call(Writable param, InetSocketAddress addr, + Class protocol, UserGroupInformation ticket, + int rpcTimeout, int serviceClass, Configuration conf) + throws InterruptedException, IOException { + ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol, + ticket, rpcTimeout, conf); + return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId, serviceClass); + } + /** * Make a call, passing param, to the IPC server running at * address which is servicing the protocol protocol, @@ -1218,10 +1237,10 @@ public class Client { return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId); } - /** + /** * Make a call, passing rpcRequest, to the IPC server defined by * remoteId, returning the rpc respond. - * + * * @param rpcKind * @param rpcRequest - contains serialized method and method parameters * @param remoteId - the target rpc server @@ -1231,8 +1250,26 @@ public class Client { */ public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest, ConnectionId remoteId) throws InterruptedException, IOException { + return call(rpcKind, rpcRequest, remoteId, RPC.RPC_SERVICE_CLASS_DEFAULT); + } + + /** + * Make a call, passing rpcRequest, to the IPC server defined by + * remoteId, returning the rpc respond. + * + * @param rpcKind + * @param rpcRequest - contains serialized method and method parameters + * @param remoteId - the target rpc server + * @param serviceClass - service class for RPC + * @returns the rpc response + * Throws exceptions if there are network problems or if the remote code + * threw an exception. + */ + public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest, + ConnectionId remoteId, int serviceClass) + throws InterruptedException, IOException { Call call = new Call(rpcKind, rpcRequest); - Connection connection = getConnection(remoteId, call); + Connection connection = getConnection(remoteId, call, serviceClass); try { connection.sendRpcRequest(call); // send the rpc request } catch (RejectedExecutionException e) { @@ -1289,7 +1326,7 @@ public class Client { /** Get a connection from the pool, or create a new one and add it to the * pool. Connections to a given ConnectionId are reused. */ private Connection getConnection(ConnectionId remoteId, - Call call) + Call call, int serviceClass) throws IOException, InterruptedException { if (!running.get()) { // the client is stopped @@ -1304,7 +1341,7 @@ public class Client { synchronized (connections) { connection = connections.get(remoteId); if (connection == null) { - connection = new Connection(remoteId); + connection = new Connection(remoteId, serviceClass); connections.put(remoteId, connection); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java index 8ee22aa415a..3563e07dbad 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java @@ -77,12 +77,12 @@ import com.google.protobuf.BlockingService; @InterfaceAudience.LimitedPrivate(value = { "Common", "HDFS", "MapReduce", "Yarn" }) @InterfaceStability.Evolving public class RPC { + final static int RPC_SERVICE_CLASS_DEFAULT = 0; public enum RpcKind { RPC_BUILTIN ((short) 1), // Used for built in calls by tests RPC_WRITABLE ((short) 2), // Use WritableRpcEngine RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size - private static final short FIRST_INDEX = RPC_BUILTIN.value; public final short value; //TODO make it private RpcKind(short val) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index c44eb9426d3..f76690c27f5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -438,6 +438,11 @@ public abstract class Server { return Arrays.asList(handlers); } + @VisibleForTesting + List getConnections() { + return connectionList; + } + /** * Refresh the service authorization ACL for the service handled by this server. */ @@ -1104,6 +1109,7 @@ public abstract class Server { private ByteBuffer connectionHeaderBuf = null; private ByteBuffer unwrappedData; private ByteBuffer unwrappedDataLengthBuffer; + private int serviceClass; UserGroupInformation user = null; public UserGroupInformation attemptingUser = null; // user name before auth @@ -1314,14 +1320,17 @@ public abstract class Server { if (!connectionHeaderRead) { //Every connection is expected to send the header. if (connectionHeaderBuf == null) { - connectionHeaderBuf = ByteBuffer.allocate(3); + connectionHeaderBuf = ByteBuffer.allocate(4); } count = channelRead(channel, connectionHeaderBuf); if (count < 0 || connectionHeaderBuf.remaining() > 0) { return count; } int version = connectionHeaderBuf.get(0); - byte[] method = new byte[] {connectionHeaderBuf.get(1)}; + // TODO we should add handler for service class later + this.setServiceClass(connectionHeaderBuf.get(1)); + + byte[] method = new byte[] {connectionHeaderBuf.get(2)}; authMethod = AuthMethod.read(new DataInputStream( new ByteArrayInputStream(method))); dataLengthBuffer.flip(); @@ -1345,7 +1354,7 @@ public abstract class Server { } IpcSerializationType serializationType = IpcSerializationType - .fromByte(connectionHeaderBuf.get(2)); + .fromByte(connectionHeaderBuf.get(3)); if (serializationType != IpcSerializationType.PROTOBUF) { respondUnsupportedSerialization(serializationType); return -1; @@ -1735,6 +1744,22 @@ public abstract class Server { return true; } + /** + * Get service class for connection + * @return the serviceClass + */ + public int getServiceClass() { + return serviceClass; + } + + /** + * Set service class for connection + * @param serviceClass the serviceClass to set + */ + public void setServiceClass(int serviceClass) { + this.serviceClass = serviceClass; + } + private synchronized void close() throws IOException { disposeSasl(); data = null; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java index 3847bfd0814..24b20f89efc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java @@ -25,6 +25,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.LongWritable; +import org.apache.hadoop.ipc.Server.Connection; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.net.NetUtils; @@ -520,11 +521,53 @@ public class TestIPC { } } + /** + * Check service class byte in IPC header is correct on wire. + */ + @Test(timeout=60000) + public void testIpcWithServiceClass() throws Exception { + // start server + Server server = new TestServer(5, false); + InetSocketAddress addr = NetUtils.getConnectAddress(server); + server.start(); + + // start client + Client.setConnectTimeout(conf, 10000); + + callAndVerify(server, addr, 0, true); + // Service Class is low to -128 as byte on wire. + // -128 shouldn't be casted on wire but -129 should. + callAndVerify(server, addr, -128, true); + callAndVerify(server, addr, -129, false); + + // Service Class is up to 127. + // 127 shouldn't be casted on wire but 128 should. + callAndVerify(server, addr, 127, true); + callAndVerify(server, addr, 128, false); + + server.stop(); + } + + /** + * Make a call from a client and verify if header info is changed in server side + */ + private void callAndVerify(Server server, InetSocketAddress addr, + int serviceClass, boolean noChanged) throws Exception{ + Client client = new Client(LongWritable.class, conf); + + client.call(new LongWritable(RANDOM.nextLong()), + addr, null, null, MIN_SLEEP_TIME, serviceClass, conf); + Connection connection = server.getConnections().get(0); + int serviceClass2 = connection.getServiceClass(); + assertFalse(noChanged ^ serviceClass == serviceClass2); + client.stop(); + } + /** * Check that file descriptors aren't leaked by starting * and stopping IPC servers. */ - @Test + @Test(timeout=60000) public void testSocketLeak() throws Exception { Assume.assumeTrue(FD_DIR.exists()); // only run on Linux From d60c2fa17f1b6ccb412ce25fc9e28b7af4c8c0a5 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 27 Mar 2013 03:44:34 +0000 Subject: [PATCH 20/37] YARN-474. Fix CapacityScheduler to trigger application-activation when am-resource-percent configuration is refreshed. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461402 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../scheduler/capacity/LeafQueue.java | 4 ++ .../scheduler/capacity/TestLeafQueue.java | 62 ++++++++++++++++++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4c7bbc7c3ef..e70fe9381fd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -143,6 +143,9 @@ Release 2.0.5-beta - UNRELEASED YARN-498. Unmanaged AM launcher does not set various constants in env for an AM, also does not handle failed AMs properly. (Hitesh Shah via bikas) + YARN-474. Fix CapacityScheduler to trigger application-activation when + am-resource-percent configuration is refreshed. (Zhijie Shen via vinodkv) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 719cf1e578b..1785ec565a6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -607,6 +607,10 @@ public class LeafQueue implements CSQueue { newlyParsedLeafQueue.getMaximumActiveApplications(), newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(), newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls); + + // queue metrics are updated, more resource may be available + // activate the pending applications if possible + activateApplications(); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index ccf2a47c128..0460b3f6ad3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -138,6 +138,7 @@ public class TestLeafQueue { private static final String C = "c"; private static final String C1 = "c1"; private static final String D = "d"; + private static final String E = "e"; private void setupQueueConfiguration( CapacitySchedulerConfiguration conf, final String newRoot) { @@ -148,7 +149,7 @@ public class TestLeafQueue { conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot; - conf.setQueues(Q_newRoot, new String[] {A, B, C, D}); + conf.setQueues(Q_newRoot, new String[] {A, B, C, D, E}); conf.setCapacity(Q_newRoot, 100); conf.setMaximumCapacity(Q_newRoot, 100); conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); @@ -174,10 +175,14 @@ public class TestLeafQueue { conf.setCapacity(Q_C1, 100); final String Q_D = Q_newRoot + "." + D; - conf.setCapacity(Q_D, 10); + conf.setCapacity(Q_D, 9); conf.setMaximumCapacity(Q_D, 11); conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d"); + final String Q_E = Q_newRoot + "." + E; + conf.setCapacity(Q_E, 1); + conf.setMaximumCapacity(Q_E, 1); + conf.setAcl(Q_E, QueueACL.SUBMIT_APPLICATIONS, "user_e"); } static LeafQueue stubLeafQueue(LeafQueue queue) { @@ -1567,6 +1572,59 @@ public class TestLeafQueue { } + @Test (timeout = 30000) + public void testActivateApplicationAfterQueueRefresh() throws Exception { + + // Manipulate queue 'e' + LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); + + // Users + final String user_e = "user_e"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = + new FiCaSchedulerApp(appAttemptId_0, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_0, user_e, E); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = + new FiCaSchedulerApp(appAttemptId_1, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_1, user_e, E); // same user + + final ApplicationAttemptId appAttemptId_2 = + TestUtils.getMockApplicationAttemptId(2, 0); + FiCaSchedulerApp app_2 = + new FiCaSchedulerApp(appAttemptId_2, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_2, user_e, E); // same user + + // before reinitialization + assertEquals(2, e.activeApplications.size()); + assertEquals(1, e.pendingApplications.size()); + + csConf.setDouble(CapacitySchedulerConfiguration + .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + CapacitySchedulerConfiguration + .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); + Map newQueues = new HashMap(); + CSQueue newRoot = + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, + newQueues, queues, + TestUtils.spyHook); + queues = newQueues; + root.reinitialize(newRoot, cs.getClusterResources()); + + // after reinitialization + assertEquals(3, e.activeApplications.size()); + assertEquals(0, e.pendingApplications.size()); + } + public boolean hasQueueACL(List aclInfos, QueueACL acl) { for (QueueUserACLInfo aclInfo : aclInfos) { if (aclInfo.getUserAcls().contains(acl)) { From 375584ae0fa49b04e159f41783760208394620df Mon Sep 17 00:00:00 2001 From: Thomas White Date: Wed, 27 Mar 2013 11:47:06 +0000 Subject: [PATCH 21/37] MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461541 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 112 +----------------- .../mapred/LocalClientProtocolProvider.java | 4 +- .../streaming/TestStreamReduceNone.java | 1 + .../streaming/TestStreamXmlRecordReader.java | 2 + 4 files changed, 7 insertions(+), 112 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ed9cf3d5de7..4c10ed40d8a 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -14,10 +14,6 @@ Trunk (Unreleased) MAPREDUCE-4887. Add RehashPartitioner, to smooth distributions with poor implementations of Object#hashCode(). (Radim Kolar via cutting) - HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows - Azure environments. (See breakdown of tasks below for subtasks and - contributors) - IMPROVEMENTS MAPREDUCE-3787. [Gridmix] Optimize job monitoring and STRESS mode for @@ -75,9 +71,6 @@ Trunk (Unreleased) MAPREDUCE-4735. Make arguments in TestDFSIO case insensitive. (Brandon Li via suresh) - MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing. - (Srikanth Sundarrajan via amareshwari) - BUG FIXES MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant. @@ -161,32 +154,11 @@ Trunk (Unreleased) MAPREDUCE-5012. Typo in javadoc for IdentityMapper class. (Adam Monsen via suresh) - - MAPREDUCE-5078. TestMRAppMaster fails on Windows due to mismatched path - separators. (Chris Nauroth via sseth) - BREAKDOWN OF HADOOP-8562 SUBTASKS + MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994. + (Sandy Ryza via tomwhite) - MAPREDUCE-4739. Some MapReduce tests fail to find winutils. - (Chris Nauroth via suresh) - - MAPREDUCE-4780. MapReduce distribution build fails on Windows. - (Chris Nauroth via suresh) - - MAPREDUCE-4790. MapReduce build script would be more readable using abspath. - (Chris Nauroth via suresh) - - MAPREDUCE-4869. Fix TestMapReduceChildJVM. (Chris Nauroth via acmurthy) - - MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy) - - MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, - so that they can pass on Windows too. (Chris Nauroth via vinodkv) - - HADOOP-9372. Fix bad timeout annotations on tests. - (Arpit Agarwal via suresh) - -Release 2.0.5-beta - UNRELEASED +Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES @@ -197,15 +169,6 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5033. mapred shell script should respect usage flags (--help -help -h). (Andrew Wang via atm) - MAPREDUCE-4892. Modify CombineFileInputFormat to not skew input slits' - allocation on small clusters. (Bikas Saha via vinodkv) - - MAPREDUCE-4990. Construct debug strings conditionally in - ShuffleHandler.Shuffle#sendMapOutput(). (kkambatl via tucu) - - MAPREDUCE-4875. coverage fixing for org.apache.hadoop.mapred - (Aleksey Gorshkov via bobby) - OPTIMIZATIONS BUG FIXES @@ -232,48 +195,6 @@ Release 2.0.5-beta - UNRELEASED MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. (Sandy Ryza via tomwhite) - MAPREDUCE-4693. History server should include counters for failed tasks. - (Xuan Gong via sseth) - - MAPREDUCE-4896. mapred queue -info spits out ugly exception when queue does - not exist. (sandyr via tucu) - - MAPREDUCE-3685. Fix bugs in MergeManager to ensure compression codec is - appropriately used and that on-disk segments are correctly sorted on - file-size. (Anty Rao and Ravi Prakash via acmurthy) - - MAPREDUCE-4571. TestHsWebServicesJobs fails on jdk7. (tgraves via tucu) - - MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid - fails with jdk7. (tgraves via tucu) - - MAPREDUCE-5075. DistCp leaks input file handles since ThrottledInputStream - does not close the wrapped InputStream. (Chris Nauroth via szetszwo) - - MAPREDUCE-3872. Fix an event handling races in ContainerLauncherImpl. - (Robert Kanter via sseth) - - MAPREDUCE-5083. MiniMRCluster should use a random component when creating an - actual cluster (Siddharth Seth via hitesh) - - MAPREDUCE-5062. Fix MR AM to read max-retries from the RM. (Zhijie Shen via - vinodkv) - - MAPREDUCE-5077. Remove mapreduce.util.ResourceCalculatorPlugin and related - code. (Karthik Kambatla via sseth) - -Release 2.0.4-alpha - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES @@ -804,12 +725,6 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi Prakash via jlowe) - MAPREDUCE-5027. Shuffle does not limit number of outstanding connections - (Robert Parker via jeagles) - - MAPREDUCE-4972. Coverage fixing for org.apache.hadoop.mapreduce.jobhistory - (Aleksey Gorshkov via bobby) - OPTIMIZATIONS MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the @@ -829,27 +744,6 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-5009. Killing the Task Attempt slated for commit does not clear the value from the Task commitAttempt member (Robert Parker via jeagles) - MAPREDUCE-4871. AM uses mapreduce.jobtracker.split.metainfo.maxsize but - mapred-default has mapreduce.job.split.metainfo.maxsize (Jason Lowe via - jeagles) - - MAPREDUCE-4794. DefaultSpeculator generates error messages on normal - shutdown (Jason Lowe via jeagles) - - MAPREDUCE-5043. Fetch failure processing can cause AM event queue to - backup and eventually OOM (Jason Lowe via bobby) - - MAPREDUCE-5023. History Server Web Services missing Job Counters (Ravi - Prakash via tgraves) - - MAPREDUCE-5060. Fetch failures that time out only count against the first - map task (Robert Joseph Evans via jlowe) - - MAPREDUCE-5042. Reducer unable to fetch for a map task that was recovered - (Jason Lowe via bobby) - - MAPREDUCE-5053. java.lang.InternalError from decompression codec cause - reducer to fail (Robert Parker via jeagles) Release 0.23.6 - UNRELEASED diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java index 5b7e3a0b92a..3d6aa62adce 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java @@ -37,9 +37,7 @@ public class LocalClientProtocolProvider extends ClientProtocolProvider { if (!MRConfig.LOCAL_FRAMEWORK_NAME.equals(framework)) { return null; } - if (conf.get("mapreduce.job.maps") == null) { - conf.setInt("mapreduce.job.maps", 1); - } + conf.setInt(JobContext.NUM_MAPS, 1); return new LocalJobRunner(conf); } diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java index 5da9e03a48b..766402184ce 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java @@ -68,6 +68,7 @@ public class TestStreamReduceNone "-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer", "-numReduceTasks", "0", "-jobconf", "mapreduce.task.files.preserve.failedtasks=true", + "-jobconf", "mapreduce.job.maps=1", "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp") }; } diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java index 7a1c6f6106b..da0bdae484c 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java @@ -54,6 +54,8 @@ public class TestStreamXmlRecordReader extends TestStreaming { protected String[] genArgs() { args.add("-inputreader"); args.add("StreamXmlRecordReader,begin=,end="); + args.add("-jobconf"); + args.add("mapreduce.job.maps=1"); return super.genArgs(); } } From c9f5052803df7982389da9f41691baa00daf3654 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Wed, 27 Mar 2013 14:21:13 +0000 Subject: [PATCH 22/37] HDFS-4581. DataNode.checkDiskError should not be called on network errors. Contributed by Rohit Kochar. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461597 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/DataNode.java | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 57994c95d8f..b79493c676a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2467,6 +2467,9 @@ Release 0.23.7 - UNRELEASED HDFS-3367. WebHDFS doesn't use the logged in user when opening connections (daryn) + HDFS-4581. checkDiskError should not be called on network errors (Rohit + Kochar via kihwal) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 10a34f43be7..dae6a108bb3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -60,8 +60,11 @@ import java.io.PrintStream; import java.net.InetSocketAddress; import java.net.ServerSocket; import java.net.Socket; +import java.net.SocketException; +import java.net.SocketTimeoutException; import java.net.URI; import java.net.UnknownHostException; +import java.nio.channels.ClosedByInterruptException; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; import java.security.PrivilegedExceptionAction; @@ -1172,7 +1175,13 @@ public class DataNode extends Configured protected void checkDiskError(Exception e ) throws IOException { LOG.warn("checkDiskError: exception: ", e); - + if (e instanceof SocketException || e instanceof SocketTimeoutException + || e instanceof ClosedByInterruptException + || e.getMessage().startsWith("Broken pipe")) { + LOG.info("Not checking disk as checkDiskError was called on a network" + + " related exception"); + return; + } if (e.getMessage() != null && e.getMessage().startsWith("No space left on device")) { throw new DiskOutOfSpaceException("No space left on device"); @@ -1484,8 +1493,12 @@ public class DataNode extends Configured } catch (IOException ie) { LOG.warn(bpReg + ":Failed to transfer " + b + " to " + targets[0] + " got ", ie); - // check if there are any disk problem - checkDiskError(); + // check if there are any disk problem + try{ + checkDiskError(ie); + } catch(IOException e) { + LOG.warn("DataNode.checkDiskError failed in run() with: ", e); + } } finally { xmitsInProgress.getAndDecrement(); From 414458ff9dd1f8f2e3e0b9079eabbc8c391222a2 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Wed, 27 Mar 2013 14:42:41 +0000 Subject: [PATCH 23/37] YARN-496. Fair scheduler configs are refreshed inconsistently in reinitialize. Contributed by Sandy Ryza git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461614 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/fair/FairScheduler.java | 28 ++++++++----------- .../fair/FairSchedulerConfiguration.java | 5 ---- .../scheduler/fair/TestFairScheduler.java | 20 +++++++++++++ 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e70fe9381fd..a5dc6865868 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -146,6 +146,9 @@ Release 2.0.5-beta - UNRELEASED YARN-474. Fix CapacityScheduler to trigger application-activation when am-resource-percent configuration is refreshed. (Zhijie Shen via vinodkv) + YARN-496. Fair scheduler configs are refreshed inconsistently in + reinitialize. (Sandy Ryza via tomwhite) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 53cc45a382e..49226a05797 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -921,25 +921,25 @@ public class FairScheduler implements ResourceScheduler { @Override public synchronized void reinitialize(Configuration conf, RMContext rmContext) throws IOException { + this.conf = new FairSchedulerConfiguration(conf); + minimumAllocation = this.conf.getMinimumMemoryAllocation(); + maximumAllocation = this.conf.getMaximumMemoryAllocation(); + userAsDefaultQueue = this.conf.getUserAsDefaultQueue(); + nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); + rackLocalityThreshold = this.conf.getLocalityThresholdRack(); + preemptionEnabled = this.conf.getPreemptionEnabled(); + assignMultiple = this.conf.getAssignMultiple(); + maxAssign = this.conf.getMaxAssign(); + sizeBasedWeight = this.conf.getSizeBasedWeight(); + if (!initialized) { - this.conf = new FairSchedulerConfiguration(conf); rootMetrics = QueueMetrics.forQueue("root", null, true, conf); this.rmContext = rmContext; this.eventLog = new FairSchedulerEventLog(); eventLog.init(this.conf); - minimumAllocation = this.conf.getMinimumMemoryAllocation(); - maximumAllocation = this.conf.getMaximumMemoryAllocation(); - userAsDefaultQueue = this.conf.getUserAsDefaultQueue(); - nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); - rackLocalityThreshold = this.conf.getLocalityThresholdRack(); - preemptionEnabled = this.conf.getPreemptionEnabled(); - assignMultiple = this.conf.getAssignMultiple(); - maxAssign = this.conf.getMaxAssign(); initialized = true; - sizeBasedWeight = this.conf.getSizeBasedWeight(); - try { queueMgr.initialize(); } catch (Exception e) { @@ -951,14 +951,8 @@ public class FairScheduler implements ResourceScheduler { updateThread.setDaemon(true); updateThread.start(); } else { - this.conf = new FairSchedulerConfiguration(conf); - userAsDefaultQueue = this.conf.getUserAsDefaultQueue(); - nodeLocalityThreshold = this.conf.getLocalityThresholdNode(); - rackLocalityThreshold = this.conf.getLocalityThresholdRack(); - preemptionEnabled = this.conf.getPreemptionEnabled(); try { queueMgr.reloadAllocs(); - } catch (Exception e) { throw new IOException("Failed to initialize FairScheduler", e); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index cf460996d8d..b41c7029ffa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -37,7 +37,6 @@ public class FairSchedulerConfiguration extends Configuration { protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue"; protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true; - protected static final String LOCALITY_THRESHOLD = CONF_PREFIX + "locality.threshold"; protected static final float DEFAULT_LOCALITY_THRESHOLD = -1.0f; /** Cluster threshold for node locality. */ @@ -89,10 +88,6 @@ public class FairSchedulerConfiguration extends Configuration { return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE); } - public float getLocalityThreshold() { - return getFloat(LOCALITY_THRESHOLD, DEFAULT_LOCALITY_THRESHOLD); - } - public float getLocalityThresholdNode() { return getFloat(LOCALITY_THRESHOLD_NODE, DEFAULT_LOCALITY_THRESHOLD_NODE); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index d7b32ec532c..709005c417d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -184,6 +184,26 @@ public class TestFairScheduler { // TESTS + @Test(timeout=2000) + public void testLoadConfigurationOnInitialize() throws IOException { + Configuration conf = createConfiguration(); + conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true); + conf.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 3); + conf.setBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, true); + conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, .5); + conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); + conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); + scheduler.reinitialize(conf, resourceManager.getRMContext()); + Assert.assertEquals(true, scheduler.assignMultiple); + Assert.assertEquals(3, scheduler.maxAssign); + Assert.assertEquals(true, scheduler.sizeBasedWeight); + Assert.assertEquals(.5, scheduler.nodeLocalityThreshold, .01); + Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01); + Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory()); + Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory()); + } + @Test public void testAggregateCapacityTracking() throws Exception { // Add a node From 85859313e7679825668a5318abbf857f54244931 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 27 Mar 2013 17:21:00 +0000 Subject: [PATCH 24/37] HADOOP-8415. Add getDouble() and setDouble() in org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh) Merging into branch-2. Updating CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461727 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a1ac0440be1..9b2bb88dd8e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -92,9 +92,6 @@ Trunk (Unreleased) HADOOP-8367 Improve documentation of declaringClassProtocolName in rpc headers. (Sanjay Radia) - HADOOP-8415. Add getDouble() and setDouble() in - org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh) - HADOOP-7659. fs -getmerge isn't guaranteed to work well over non-HDFS filesystems (harsh) @@ -514,6 +511,9 @@ Release 2.0.5-beta - UNRELEASED HADOOP-9283. Add support for running the Hadoop client on AIX. (atm) + HADOOP-8415. Add getDouble() and setDouble() in + org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh) + IMPROVEMENTS HADOOP-9253. Capture ulimit info in the logs at service start time. From d0bbff6c32592cb5d49d7be8d8a7346788a9ba19 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 27 Mar 2013 18:38:28 +0000 Subject: [PATCH 25/37] YARN-209. Fix CapacityScheduler to trigger application-activation when the cluster capacity changes. Contributed by Zhijie Shen. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461773 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../scheduler/capacity/LeafQueue.java | 6 ++- .../yarn/server/resourcemanager/TestRM.java | 47 +++++++++++++++++++ .../scheduler/capacity/TestLeafQueue.java | 43 +++++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a5dc6865868..77ae483207d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -149,6 +149,9 @@ Release 2.0.5-beta - UNRELEASED YARN-496. Fair scheduler configs are refreshed inconsistently in reinitialize. (Sandy Ryza via tomwhite) + YARN-209. Fix CapacityScheduler to trigger application-activation when + the cluster capacity changes. (Zhijie Shen via vinodkv) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 1785ec565a6..efc6e3efb25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1481,7 +1481,11 @@ public class LeafQueue implements CSQueue { CSQueueUtils.updateQueueStatistics( resourceCalculator, this, getParent(), clusterResource, minimumAllocation); - + + // queue metrics are updated, more resource may be available + // activate the pending applications if possible + activateApplications(); + // Update application properties for (FiCaSchedulerApp application : activeApplications) { synchronized (application) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java index 1a161268474..939443bc5ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java @@ -26,10 +26,12 @@ import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; @@ -135,6 +137,51 @@ public class TestRM { rm.stop(); } + @Test (timeout = 30000) + public void testActivatingApplicationAfterAddingNM() throws Exception { + YarnConfiguration conf = new YarnConfiguration(); + + MockRM rm1 = new MockRM(conf); + + // start like normal because state is empty + rm1.start(); + + // app that gets launched + RMApp app1 = rm1.submitApp(200); + + // app that does not get launched + RMApp app2 = rm1.submitApp(200); + + // app1 and app2 should be scheduled, but because no resource is available, + // they are not activated. + RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); + ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId(); + rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED); + RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); + ApplicationAttemptId attemptId2 = attempt2.getAppAttemptId(); + rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED); + + MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService()); + MockNM nm2 = new MockNM("h2:5678", 15120, rm1.getResourceTrackerService()); + nm1.registerNode(); + nm2.registerNode(); + + //kick the scheduling + nm1.nodeHeartbeat(true); + + // app1 should be allocated now + rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED); + rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED); + + nm2.nodeHeartbeat(true); + + // app2 should be allocated now + rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED); + rm1.waitForState(attemptId2, RMAppAttemptState.ALLOCATED); + + rm1.stop(); + } + public static void main(String[] args) throws Exception { TestRM t = new TestRM(); t.testGetNewAppId(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 0460b3f6ad3..828c796d011 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -1625,6 +1625,49 @@ public class TestLeafQueue { assertEquals(0, e.pendingApplications.size()); } + @Test (timeout = 30000) + public void testActivateApplicationByUpdatingClusterResource() + throws Exception { + + // Manipulate queue 'e' + LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E)); + + // Users + final String user_e = "user_e"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = + TestUtils.getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = + new FiCaSchedulerApp(appAttemptId_0, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_0, user_e, E); + + final ApplicationAttemptId appAttemptId_1 = + TestUtils.getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = + new FiCaSchedulerApp(appAttemptId_1, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_1, user_e, E); // same user + + final ApplicationAttemptId appAttemptId_2 = + TestUtils.getMockApplicationAttemptId(2, 0); + FiCaSchedulerApp app_2 = + new FiCaSchedulerApp(appAttemptId_2, user_e, e, + mock(ActiveUsersManager.class), rmContext); + e.submitApplication(app_2, user_e, E); // same user + + // before updating cluster resource + assertEquals(2, e.activeApplications.size()); + assertEquals(1, e.pendingApplications.size()); + + e.updateClusterResource(Resources.createResource(200 * 16 * GB, 100 * 32)); + + // after updating cluster resource + assertEquals(3, e.activeApplications.size()); + assertEquals(0, e.pendingApplications.size()); + } + public boolean hasQueueACL(List aclInfos, QueueACL acl) { for (QueueUserACLInfo aclInfo : aclInfos) { if (aclInfo.getUserAcls().contains(acl)) { From 81192e4e415d359ca832eff50d6f64c3da7acb73 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 27 Mar 2013 18:42:43 +0000 Subject: [PATCH 26/37] HDFS-4621. Additional logging to help diagnose slow QJM syncs. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461777 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../qjournal/client/IPCLoggerChannel.java | 8 ++++++ .../hdfs/qjournal/client/QuorumCall.java | 28 +++++++++++++++++++ .../hadoop/hdfs/qjournal/server/Journal.java | 8 ++++++ 4 files changed, 46 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b79493c676a..856a1c74cf0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -371,6 +371,8 @@ Release 2.0.5-beta - UNRELEASED HDFS-4635. Move BlockManager#computeCapacity to LightWeightGSet. (suresh) + HDFS-4621. Additional logging to help diagnose slow QJM syncs. (todd) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java index 1c82af858c0..c64acd85d7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java @@ -133,6 +133,8 @@ public class IPCLoggerChannel implements AsyncLogger { private Stopwatch lastHeartbeatStopwatch = new Stopwatch(); private static final long HEARTBEAT_INTERVAL_MILLIS = 1000; + + private static final long WARN_JOURNAL_MILLIS_THRESHOLD = 1000; static final Factory FACTORY = new AsyncLogger.Factory() { @Override @@ -371,6 +373,12 @@ public class IPCLoggerChannel implements AsyncLogger { now - submitNanos, TimeUnit.NANOSECONDS); metrics.addWriteEndToEndLatency(endToEndTime); metrics.addWriteRpcLatency(rpcTime); + if (rpcTime / 1000 > WARN_JOURNAL_MILLIS_THRESHOLD) { + QuorumJournalManager.LOG.warn( + "Took " + (rpcTime / 1000) + "ms to send a batch of " + + numTxns + " edits (" + data.length + " bytes) to " + + "remote journal " + IPCLoggerChannel.this); + } } synchronized (IPCLoggerChannel.this) { highestAckedTxId = firstTxnId + numTxns - 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java index c42f6b917a6..f15e4626f41 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java @@ -24,6 +24,7 @@ import java.util.concurrent.TimeoutException; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.Time; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.util.concurrent.FutureCallback; @@ -120,6 +121,15 @@ class QuorumCall { String msg = String.format( "Waited %s ms (timeout=%s ms) for a response for %s", waited, millis, operationName); + if (!successes.isEmpty()) { + msg += ". Succeeded so far: [" + Joiner.on(",").join(successes.keySet()) + "]"; + } + if (!exceptions.isEmpty()) { + msg += ". Exceptions so far: [" + getExceptionMapString() + "]"; + } + if (successes.isEmpty() && exceptions.isEmpty()) { + msg += ". No responses yet."; + } if (waited > millis * WAIT_PROGRESS_WARN_THRESHOLD) { QuorumJournalManager.LOG.warn(msg); } else { @@ -227,4 +237,22 @@ class QuorumCall { } return sb.toString(); } + + /** + * Return a string suitable for displaying to the user, containing + * any exceptions that have been received so far. + */ + private String getExceptionMapString() { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Map.Entry e : exceptions.entrySet()) { + if (!first) { + sb.append(", "); + } + first = false; + sb.append(e.getKey()).append(": ") + .append(e.getValue().getLocalizedMessage()); + } + return sb.toString(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java index e6b179009c5..58965ce02a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java @@ -128,6 +128,10 @@ class Journal implements Closeable { private final JournalMetrics metrics; + /** + * Time threshold for sync calls, beyond which a warning should be logged to the console. + */ + private static final int WARN_SYNC_MILLIS_THRESHOLD = 1000; Journal(File logDir, String journalId, StorageErrorReporter errorReporter) throws IOException { @@ -370,6 +374,10 @@ class Journal implements Closeable { sw.stop(); metrics.addSync(sw.elapsedTime(TimeUnit.MICROSECONDS)); + if (sw.elapsedTime(TimeUnit.MILLISECONDS) > WARN_SYNC_MILLIS_THRESHOLD) { + LOG.warn("Sync of transaction range " + firstTxnId + "-" + lastTxnId + + " took " + sw.elapsedTime(TimeUnit.MILLISECONDS) + "ms"); + } if (isLagging) { // This batch of edits has already been committed on a quorum of other From 4809bfa6ed34de09ad4bf34c9293f45075589494 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 27 Mar 2013 18:46:32 +0000 Subject: [PATCH 27/37] HDFS-4618. Default transaction interval for checkpoints is too low. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461783 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +- .../hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 856a1c74cf0..6b2b2aed792 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -373,6 +373,8 @@ Release 2.0.5-beta - UNRELEASED HDFS-4621. Additional logging to help diagnose slow QJM syncs. (todd) + HDFS-4618. Default transaction interval for checkpoints is too low. (todd) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 4d3b6733eeb..1545ff63e87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -122,7 +122,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period"; public static final long DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT = 3600; public static final String DFS_NAMENODE_CHECKPOINT_TXNS_KEY = "dfs.namenode.checkpoint.txns"; - public static final long DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT = 40000; + public static final long DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT = 1000000; public static final String DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY = "dfs.namenode.checkpoint.max-retries"; public static final int DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT = 3; public static final String DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY = "dfs.namenode.heartbeat.recheck-interval"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index cbd27ad3f64..e3da167a6f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -635,7 +635,7 @@ dfs.namenode.checkpoint.txns - 40000 + 1000000 The Secondary NameNode or CheckpointNode will create a checkpoint of the namespace every 'dfs.namenode.checkpoint.txns' transactions, regardless of whether 'dfs.namenode.checkpoint.period' has expired. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm index 82f1046eb47..b9d1c637a32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm @@ -193,7 +193,7 @@ HDFS Users Guide * <<>>, set to 1 hour by default, specifies the maximum delay between two consecutive checkpoints, and - * <<>>, set to 40000 default, defines the + * <<>>, set to 1 million by default, defines the number of uncheckpointed transactions on the NameNode which will force an urgent checkpoint, even if the checkpoint period has not been reached. @@ -232,7 +232,7 @@ HDFS Users Guide * <<>>, set to 1 hour by default, specifies the maximum delay between two consecutive checkpoints - * <<>>, set to 40000 default, defines the + * <<>>, set to 1 million by default, defines the number of uncheckpointed transactions on the NameNode which will force an urgent checkpoint, even if the checkpoint period has not been reached. From fc0a4de670c3e6dcf5d30305d3969fe4946a0fed Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Wed, 27 Mar 2013 18:51:06 +0000 Subject: [PATCH 28/37] HADOOP-9358. "Auth failed" log should include exception string. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461788 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/ipc/Server.java | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9b2bb88dd8e..6cd887ab9eb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -534,6 +534,8 @@ Release 2.0.5-beta - UNRELEASED HADOOP-9318. When exiting on a signal, print the signal name first. (Colin Patrick McCabe via atm) + HADOOP-9358. "Auth failed" log should include exception string (todd) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index f76690c27f5..6472e77f9cf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -1237,7 +1237,8 @@ public abstract class Server { rpcMetrics.incrAuthenticationFailures(); String clientIP = this.toString(); // attempting user could be null - AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser); + AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser + + " (" + e.getLocalizedMessage() + ")"); throw e; } if (saslServer.isComplete() && replyToken == null) { From 0e6604aab30671618355982aa86c0a7661132a9c Mon Sep 17 00:00:00 2001 From: Sanjay Radia Date: Wed, 27 Mar 2013 20:49:48 +0000 Subject: [PATCH 29/37] HDFS-4633 TestDFSClientExcludedNodes fails sporadically if excluded nodes cache expires too quickly (Chris Nauroth via Sanjay) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461846 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hdfs/TestDFSClientExcludedNodes.java | 38 +++++++++++++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6b2b2aed792..7a9e504c434 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -181,6 +181,9 @@ Trunk (Unreleased) HDFS-4346. Add SequentialNumber as a base class for INodeId and GenerationStamp. (szetszwo) + HDFS-4633 TestDFSClientExcludedNodes fails sporadically if excluded nodes + cache expires too quickly (Chris Nauroth via Sanjay) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java index fa88e394577..3574323d545 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java @@ -32,6 +32,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; import org.apache.hadoop.util.ThreadUtil; +import org.junit.After; +import org.junit.Before; import org.junit.Test; @@ -41,10 +43,25 @@ import org.junit.Test; */ public class TestDFSClientExcludedNodes { - @Test(timeout=10000) + private MiniDFSCluster cluster; + private Configuration conf; + + @Before + public void setUp() { + cluster = null; + conf = new HdfsConfiguration(); + } + + @After + public void tearDown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test(timeout=60000) public void testExcludedNodes() throws IOException { - Configuration conf = new HdfsConfiguration(); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); FileSystem fs = cluster.getFileSystem(); Path filePath = new Path("/testExcludedNodes"); @@ -67,17 +84,16 @@ public class TestDFSClientExcludedNodes { } } - @Test(timeout=10000) + @Test(timeout=60000) public void testExcludedNodesForgiveness() throws IOException { - Configuration conf = new HdfsConfiguration(); - // Forgive nodes in under 1s for this test case. + // Forgive nodes in under 2.5s for this test case. conf.setLong( DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL, - 1000); + 2500); // We'll be using a 512 bytes block size just for tests // so making sure the checksum bytes too match it. conf.setInt("io.bytes.per.checksum", 512); - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); List props = cluster.dataNodes; FileSystem fs = cluster.getFileSystem(); Path filePath = new Path("/testForgivingExcludedNodes"); @@ -112,11 +128,11 @@ public class TestDFSClientExcludedNodes { Assert.assertEquals(true, cluster.restartDataNode(two, true)); cluster.waitActive(); - // Sleep for 2s, to let the excluded nodes be expired + // Sleep for 5s, to let the excluded nodes be expired // from the excludes list (i.e. forgiven after the configured wait period). - // [Sleeping just in case the restart of the DNs completed < 2s cause + // [Sleeping just in case the restart of the DNs completed < 5s cause // otherwise, we'll end up quickly excluding those again.] - ThreadUtil.sleepAtLeastIgnoreInterrupts(2000); + ThreadUtil.sleepAtLeastIgnoreInterrupts(5000); // Terminate the last good DN, to assert that there's no // single-DN-available scenario, caused by not forgiving the other From fc5fd80e9fa21b9c3981fb5afc8fce376aa6a2d9 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 27 Mar 2013 21:49:43 +0000 Subject: [PATCH 30/37] HADOOP-9125. LdapGroupsMapping threw CommunicationException after some idle time. Contributed by Kai Zheng. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461863 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../hadoop/security/LdapGroupsMapping.java | 84 +++++++++++++------ .../security/TestLdapGroupsMapping.java | 59 +++++++++---- 3 files changed, 105 insertions(+), 41 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6cd887ab9eb..c2a1b4c7478 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -598,6 +598,9 @@ Release 2.0.5-beta - UNRELEASED HADOOP-9430. TestSSLFactory fails on IBM JVM. (Amir Sanjar via suresh) + HADOOP-9125. LdapGroupsMapping threw CommunicationException after some + idle time. (Kai Zheng via atm) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java index bcccb198ebc..eda711547c3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Hashtable; import java.util.List; +import javax.naming.CommunicationException; import javax.naming.Context; import javax.naming.NamingEnumeration; import javax.naming.NamingException; @@ -166,6 +167,8 @@ public class LdapGroupsMapping private String groupMemberAttr; private String groupNameAttr; + public static int RECONNECT_RETRY_COUNT = 3; + /** * Returns list of groups for a user. * @@ -178,34 +181,63 @@ public class LdapGroupsMapping */ @Override public synchronized List getGroups(String user) throws IOException { - List groups = new ArrayList(); - + List emptyResults = new ArrayList(); + /* + * Normal garbage collection takes care of removing Context instances when they are no longer in use. + * Connections used by Context instances being garbage collected will be closed automatically. + * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection. + */ try { - DirContext ctx = getDirContext(); - - // Search for the user. We'll only ever need to look at the first result - NamingEnumeration results = ctx.search(baseDN, - userSearchFilter, - new Object[]{user}, - SEARCH_CONTROLS); - if (results.hasMoreElements()) { - SearchResult result = results.nextElement(); - String userDn = result.getNameInNamespace(); - - NamingEnumeration groupResults = - ctx.search(baseDN, - "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))", - new Object[]{userDn}, - SEARCH_CONTROLS); - while (groupResults.hasMoreElements()) { - SearchResult groupResult = groupResults.nextElement(); - Attribute groupName = groupResult.getAttributes().get(groupNameAttr); - groups.add(groupName.get().toString()); - } - } + return doGetGroups(user); + } catch (CommunicationException e) { + LOG.warn("Connection is closed, will try to reconnect"); } catch (NamingException e) { LOG.warn("Exception trying to get groups for user " + user, e); - return new ArrayList(); + return emptyResults; + } + + int retryCount = 0; + while (retryCount ++ < RECONNECT_RETRY_COUNT) { + //reset ctx so that new DirContext can be created with new connection + this.ctx = null; + + try { + return doGetGroups(user); + } catch (CommunicationException e) { + LOG.warn("Connection being closed, reconnecting failed, retryCount = " + retryCount); + } catch (NamingException e) { + LOG.warn("Exception trying to get groups for user " + user, e); + return emptyResults; + } + } + + return emptyResults; + } + + List doGetGroups(String user) throws NamingException { + List groups = new ArrayList(); + + DirContext ctx = getDirContext(); + + // Search for the user. We'll only ever need to look at the first result + NamingEnumeration results = ctx.search(baseDN, + userSearchFilter, + new Object[]{user}, + SEARCH_CONTROLS); + if (results.hasMoreElements()) { + SearchResult result = results.nextElement(); + String userDn = result.getNameInNamespace(); + + NamingEnumeration groupResults = + ctx.search(baseDN, + "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))", + new Object[]{userDn}, + SEARCH_CONTROLS); + while (groupResults.hasMoreElements()) { + SearchResult groupResult = groupResults.nextElement(); + Attribute groupName = groupResult.getAttributes().get(groupNameAttr); + groups.add(groupName.get().toString()); + } } return groups; @@ -236,7 +268,7 @@ public class LdapGroupsMapping return ctx; } - + /** * Caches groups, no need to do that for this provider */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java index 19ac7b68211..331e288e8ba 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java @@ -26,6 +26,7 @@ import java.io.Writer; import java.util.Arrays; import java.util.List; +import javax.naming.CommunicationException; import javax.naming.NamingEnumeration; import javax.naming.NamingException; import javax.naming.directory.Attribute; @@ -46,21 +47,15 @@ public class TestLdapGroupsMapping { private DirContext mockContext; private LdapGroupsMapping mappingSpy = spy(new LdapGroupsMapping()); + private NamingEnumeration mockUserNamingEnum = mock(NamingEnumeration.class); + private NamingEnumeration mockGroupNamingEnum = mock(NamingEnumeration.class); + private String[] testGroups = new String[] {"group1", "group2"}; @Before public void setupMocks() throws NamingException { mockContext = mock(DirContext.class); doReturn(mockContext).when(mappingSpy).getDirContext(); - - NamingEnumeration mockUserNamingEnum = mock(NamingEnumeration.class); - NamingEnumeration mockGroupNamingEnum = mock(NamingEnumeration.class); - - // The search functionality of the mock context is reused, so we will - // return the user NamingEnumeration first, and then the group - when(mockContext.search(anyString(), anyString(), any(Object[].class), - any(SearchControls.class))) - .thenReturn(mockUserNamingEnum, mockGroupNamingEnum); - + SearchResult mockUserResult = mock(SearchResult.class); // We only ever call hasMoreElements once for the user NamingEnum, so // we can just have one return value @@ -76,23 +71,57 @@ public class TestLdapGroupsMapping { // Define the attribute for the name of the first group Attribute group1Attr = new BasicAttribute("cn"); - group1Attr.add("group1"); + group1Attr.add(testGroups[0]); Attributes group1Attrs = new BasicAttributes(); group1Attrs.put(group1Attr); // Define the attribute for the name of the second group Attribute group2Attr = new BasicAttribute("cn"); - group2Attr.add("group2"); + group2Attr.add(testGroups[1]); Attributes group2Attrs = new BasicAttributes(); group2Attrs.put(group2Attr); // This search result gets reused, so return group1, then group2 when(mockGroupResult.getAttributes()).thenReturn(group1Attrs, group2Attrs); - } @Test public void testGetGroups() throws IOException, NamingException { + // The search functionality of the mock context is reused, so we will + // return the user NamingEnumeration first, and then the group + when(mockContext.search(anyString(), anyString(), any(Object[].class), + any(SearchControls.class))) + .thenReturn(mockUserNamingEnum, mockGroupNamingEnum); + + doTestGetGroups(Arrays.asList(testGroups), 2); + } + + @Test + public void testGetGroupsWithConnectionClosed() throws IOException, NamingException { + // The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException, + // then after reconnected return the user NamingEnumeration first, and then the group + when(mockContext.search(anyString(), anyString(), any(Object[].class), + any(SearchControls.class))) + .thenThrow(new CommunicationException("Connection is closed")) + .thenReturn(mockUserNamingEnum, mockGroupNamingEnum); + + // Although connection is down but after reconnected it still should retrieve the result groups + doTestGetGroups(Arrays.asList(testGroups), 1 + 2); // 1 is the first failure call + } + + @Test + public void testGetGroupsWithLdapDown() throws IOException, NamingException { + // This mocks the case where Ldap server is down, and always throws CommunicationException + when(mockContext.search(anyString(), anyString(), any(Object[].class), + any(SearchControls.class))) + .thenThrow(new CommunicationException("Connection is closed")); + + // Ldap server is down, no groups should be retrieved + doTestGetGroups(Arrays.asList(new String[] {}), + 1 + LdapGroupsMapping.RECONNECT_RETRY_COUNT); // 1 is the first normal call + } + + private void doTestGetGroups(List expectedGroups, int searchTimes) throws IOException, NamingException { Configuration conf = new Configuration(); // Set this, so we don't throw an exception conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test"); @@ -102,10 +131,10 @@ public class TestLdapGroupsMapping { // regardless of input List groups = mappingSpy.getGroups("some_user"); - Assert.assertEquals(Arrays.asList("group1", "group2"), groups); + Assert.assertEquals(expectedGroups, groups); // We should have searched for a user, and then two groups - verify(mockContext, times(2)).search(anyString(), + verify(mockContext, times(searchTimes)).search(anyString(), anyString(), any(Object[].class), any(SearchControls.class)); From 967b62f5d23e5e3b3573caab5cf03d089e5bcc11 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 27 Mar 2013 23:14:05 +0000 Subject: [PATCH 31/37] YARN-24. Nodemanager fails to start if log aggregation enabled and namenode unavailable. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461891 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../logaggregation/LogAggregationService.java | 14 ++-- .../TestLogAggregationService.java | 74 ++++++++++++++++++- 3 files changed, 81 insertions(+), 10 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 77ae483207d..56528af93a3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -152,6 +152,9 @@ Release 2.0.5-beta - UNRELEASED YARN-209. Fix CapacityScheduler to trigger application-activation when the cluster capacity changes. (Zhijie Shen via vinodkv) + YARN-24. Nodemanager fails to start if log aggregation enabled and + namenode unavailable. (sandyr via tucu) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 88a01eb3854..f183d9e8aba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; @@ -97,7 +96,7 @@ public class LogAggregationService extends AbstractService implements private final ConcurrentMap appLogAggregators; private final ExecutorService threadPool; - + public LogAggregationService(Dispatcher dispatcher, Context context, DeletionService deletionService, LocalDirsHandlerService dirsHandler) { super(LogAggregationService.class.getName()); @@ -129,7 +128,6 @@ public class LogAggregationService extends AbstractService implements // NodeId is only available during start, the following cannot be moved // anywhere else. this.nodeId = this.context.getNodeId(); - verifyAndCreateRemoteLogDir(getConfig()); super.start(); } @@ -164,7 +162,7 @@ public class LogAggregationService extends AbstractService implements } } - private void verifyAndCreateRemoteLogDir(Configuration conf) { + void verifyAndCreateRemoteLogDir(Configuration conf) { // Checking the existance of the TLD FileSystem remoteFS = null; try { @@ -177,7 +175,7 @@ public class LogAggregationService extends AbstractService implements remoteExists = remoteFS.exists(this.remoteRootLogDir); } catch (IOException e) { throw new YarnException("Failed to check for existence of remoteLogDir [" - + this.remoteRootLogDir + "]"); + + this.remoteRootLogDir + "]", e); } if (remoteExists) { try { @@ -191,8 +189,8 @@ public class LogAggregationService extends AbstractService implements } } catch (IOException e) { throw new YarnException( - "Failed while attempting to check permissions for dir [" - + this.remoteRootLogDir + "]"); + "Failed to check permissions for dir [" + + this.remoteRootLogDir + "]", e); } } else { LOG.warn("Remote Root Log Dir [" + this.remoteRootLogDir @@ -208,7 +206,6 @@ public class LogAggregationService extends AbstractService implements + this.remoteRootLogDir + "]", e); } } - } Path getRemoteNodeLogFileForApp(ApplicationId appId, String user) { @@ -296,6 +293,7 @@ public class LogAggregationService extends AbstractService implements Map appAcls) { ApplicationEvent eventResponse; try { + verifyAndCreateRemoteLogDir(getConfig()); initAppAggregator(appId, user, credentials, logRetentionPolicy, appAcls); eventResponse = new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 2408681ddbd..6a9a6767567 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -44,6 +44,7 @@ import junit.framework.Assert; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.io.DataInputBuffer; @@ -79,7 +80,6 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; @@ -87,6 +87,7 @@ import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.mortbay.util.MultiException; @@ -393,7 +394,76 @@ public class TestLogAggregationService extends BaseContainerManagerTest { @Test @SuppressWarnings("unchecked") - public void testLogAggregationInitFailsWithoutKillingNM() throws Exception { + public void testVerifyAndCreateRemoteDirsFailure() + throws Exception { + this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + this.remoteRootLogDir.getAbsolutePath()); + + DrainDispatcher dispatcher = createDispatcher(); + EventHandler appEventHandler = mock(EventHandler.class); + dispatcher.register(ApplicationEventType.class, appEventHandler); + + LogAggregationService logAggregationService = spy( + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler)); + logAggregationService.init(this.conf); + + YarnException e = new YarnException("KABOOM!"); + doThrow(e) + .when(logAggregationService).verifyAndCreateRemoteLogDir( + any(Configuration.class)); + + logAggregationService.start(); + + // Now try to start an application + ApplicationId appId = BuilderUtils.newApplicationId( + System.currentTimeMillis(), (int)Math.random()); + logAggregationService.handle(new LogHandlerAppStartedEvent(appId, + this.user, null, + ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, + this.acls)); + dispatcher.await(); + + // Verify that it failed + ApplicationEvent[] expectedEvents = new ApplicationEvent[] { + new ApplicationEvent(appId, + ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) + }; + checkEvents(appEventHandler, expectedEvents, false, + "getType", "getApplicationID", "getDiagnostic"); + + Mockito.reset(logAggregationService); + + // Now try to start another one + ApplicationId appId2 = BuilderUtils.newApplicationId( + System.currentTimeMillis(), (int)Math.random()); + File appLogDir = + new File(localLogDir, ConverterUtils.toString(appId2)); + appLogDir.mkdir(); + + logAggregationService.handle(new LogHandlerAppStartedEvent(appId2, + this.user, null, + ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, + this.acls)); + dispatcher.await(); + + // Verify that it worked + expectedEvents = new ApplicationEvent[] { + new ApplicationEvent(appId, // original failure + ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED), + new ApplicationEvent(appId2, // success + ApplicationEventType.APPLICATION_LOG_HANDLING_INITED) + }; + checkEvents(appEventHandler, expectedEvents, false, + "getType", "getApplicationID", "getDiagnostic"); + + logAggregationService.stop(); + } + + @Test + @SuppressWarnings("unchecked") + public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception { this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); From 799e3c344ebc6e1f64072ae211d62fe541625310 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 27 Mar 2013 23:43:45 +0000 Subject: [PATCH 32/37] HADOOP-9357. Fallback to default authority if not specified in FileContext. Contributed by Andrew Wang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1461898 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/fs/FileContext.java | 24 ++++++++++--- .../main/java/org/apache/hadoop/fs/Path.java | 4 +++ .../fs/FileContextMainOperationsBaseTest.java | 36 +++++++++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c2a1b4c7478..309ea50241a 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -601,6 +601,9 @@ Release 2.0.5-beta - UNRELEASED HADOOP-9125. LdapGroupsMapping threw CommunicationException after some idle time. (Kai Zheng via atm) + HADOOP-9357. Fallback to default authority if not specified in FileContext. + (Andrew Wang via eli) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 26f50503fef..3ba48d3b4cd 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -244,17 +244,33 @@ public final class FileContext { } /* - * Remove relative part - return "absolute": - * If input is relative path ("foo/bar") add wd: ie "//foo/bar" - * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path + * Resolve a relative path passed from the user. + * + * Relative paths are resolved against the current working directory + * (e.g. "foo/bar" becomes "//foo/bar"). + * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths * ("/foo/bar") are returned unchanged. * + * Additionally, we fix malformed URIs that specify a scheme but not an + * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme + * if it matches the default FS, and let the default FS add in the default + * scheme and authority later (see {@link #AbstractFileSystem#checkPath}). + * * Applications that use FileContext should use #makeQualified() since - * they really want a fully qualified URI. + * they really want a fully-qualified URI. * Hence this method is not called makeAbsolute() and * has been deliberately declared private. */ private Path fixRelativePart(Path p) { + // Per RFC 2396 5.2, drop schema if there is a scheme but no authority. + if (p.hasSchemeAndNoAuthority()) { + String scheme = p.toUri().getScheme(); + if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) { + p = new Path(p.toUri().getSchemeSpecificPart()); + } + } + // Absolute paths are unchanged. Relative paths are resolved against the + // current working directory. if (p.isUriPathAbsolute()) { return p; } else { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index feef1c7bab2..8299aa3be89 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -256,6 +256,10 @@ public class Path implements Comparable { return (isUriPathAbsolute() && uri.getScheme() == null && uri.getAuthority() == null); } + + public boolean hasSchemeAndNoAuthority() { + return uri.getScheme() != null && uri.getAuthority() == null; + } /** * True if the path component (i.e. directory) of this URI is absolute. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index d6efc52c42a..df2f7eb9597 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.EnumSet; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -1164,6 +1166,40 @@ public abstract class FileContextMainOperationsBaseTest { Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file)); } } + + /** + * Test that URIs with a scheme, no authority, and absolute path component + * resolve with the authority of the default FS. + */ + @Test(timeout=30000) + public void testAbsolutePathSchemeNoAuthority() throws IOException, + URISyntaxException { + Path file = getTestRootPath(fc, "test/file"); + createFile(file); + URI uri = file.toUri(); + URI noAuthorityUri = new URI(uri.getScheme(), null, uri.getPath(), + uri.getQuery(), uri.getFragment()); + Path noAuthority = new Path(noAuthorityUri); + Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority)); + } + + /** + * Test that URIs with a scheme, no authority, and relative path component + * resolve with the authority of the default FS. + */ + @Test(timeout=30000) + public void testRelativePathSchemeNoAuthority() throws IOException, + URISyntaxException { + Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test")); + fc.setWorkingDirectory(workDir); + Path file = new Path(workDir, "file"); + createFile(file); + URI uri = file.toUri(); + URI noAuthorityUri = new URI(uri.getScheme() + ":file"); + System.out.println(noAuthorityUri); + Path noAuthority = new Path(noAuthorityUri); + Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority)); + } protected void createFile(Path path) throws IOException { FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE), From 520033b1cd81c76b38fcdcfcfeed16158db4bbba Mon Sep 17 00:00:00 2001 From: Bikas Saha Date: Thu, 28 Mar 2013 19:44:28 +0000 Subject: [PATCH 33/37] YARN-450. Define value for * in the scheduling protocol (Zhijie Shen via bikas) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1462271 13f79535-47bb-0310-9956-ffa450edef68 --- .../v2/app/rm/RMContainerRequestor.java | 5 +- .../mapreduce/v2/app/MRAppBenchmark.java | 2 +- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/api/records/ResourceRequest.java | 20 +++++++ .../apache/hadoop/yarn/client/AMRMClient.java | 5 -- .../hadoop/yarn/client/AMRMClientImpl.java | 6 +- .../hadoop/yarn/client/TestAMRMClient.java | 18 +++--- .../rmapp/attempt/RMAppAttemptImpl.java | 6 +- .../server/resourcemanager/rmnode/RMNode.java | 2 - .../scheduler/AppSchedulingInfo.java | 14 ++--- .../scheduler/capacity/LeafQueue.java | 8 +-- .../common/fica/FiCaSchedulerApp.java | 3 +- .../common/fica/FiCaSchedulerNode.java | 2 - .../scheduler/fair/AppSchedulable.java | 3 +- .../scheduler/fair/FSSchedulerApp.java | 5 +- .../scheduler/fair/FSSchedulerNode.java | 2 - .../scheduler/fifo/FifoScheduler.java | 8 +-- .../server/resourcemanager/Application.java | 11 +--- .../yarn/server/resourcemanager/MockAM.java | 4 +- .../yarn/server/resourcemanager/Task.java | 4 +- .../resourcemanager/TestFifoScheduler.java | 8 +-- .../resourcemanager/TestResourceManager.java | 3 +- .../capacity/TestApplicationLimits.java | 9 ++- .../capacity/TestCapacityScheduler.java | 3 +- .../scheduler/capacity/TestLeafQueue.java | 55 +++++++++---------- .../scheduler/fair/TestFairScheduler.java | 30 ++++++---- .../scheduler/fifo/TestFifoScheduler.java | 10 ++-- .../server/TestContainerManagerSecurity.java | 9 +-- 28 files changed, 132 insertions(+), 126 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java index d9a2c1f4ecc..79c97689c1f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java @@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.util.BuilderUtils; public abstract class RMContainerRequestor extends RMCommunicator { private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class); - static final String ANY = "*"; private int lastResponseID; private Resource availableResources; @@ -278,7 +277,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { } // Off-switch - addResourceRequest(req.priority, ANY, req.capability); + addResourceRequest(req.priority, ResourceRequest.ANY, req.capability); } protected void decContainerReq(ContainerRequest req) { @@ -291,7 +290,7 @@ public abstract class RMContainerRequestor extends RMCommunicator { decResourceRequest(req.priority, rack, req.capability); } - decResourceRequest(req.priority, ANY, req.capability); + decResourceRequest(req.priority, ResourceRequest.ANY, req.capability); } private void addResourceRequest(Priority priority, String resourceName, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java index e50d91c0cf8..efb8b7a134b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java @@ -230,7 +230,7 @@ public class MRAppBenchmark { List askList = request.getAskList(); List containers = new ArrayList(); for (ResourceRequest req : askList) { - if (req.getHostName() != "*") { + if (!ResourceRequest.isAnyLocation(req.getHostName())) { continue; } int numContainers = req.getNumContainers(); diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 56528af93a3..7114465aea3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -100,6 +100,9 @@ Release 2.0.5-beta - UNRELEASED YARN-469. Make scheduling mode in FS pluggable. (kkambatl via tucu) + YARN-450. Define value for * in the scheduling protocol (Zhijie Shen via + bikas) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java index c1b7541265d..0406a57525d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java @@ -49,6 +49,26 @@ import org.apache.hadoop.yarn.api.AMRMProtocol; @Public @Stable public abstract class ResourceRequest implements Comparable { + + /** + * The constant string representing no locality. + * It should be used by all references that want to pass an arbitrary host + * name in. + */ + public static final String ANY = "*"; + + /** + * Check whether the given host/rack string represents an arbitrary + * host name. + * + * @param hostName host/rack on which the allocation is desired + * @return whether the given host/rack string represents an arbitrary + * host name + */ + public static boolean isAnyLocation(String hostName) { + return ANY.equals(hostName); + } + /** * Get the Priority of the request. * @return Priority of the request diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java index 1fa86d2cc24..996d4668b54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClient.java @@ -34,11 +34,6 @@ import org.apache.hadoop.yarn.service.Service; @InterfaceStability.Unstable public interface AMRMClient extends Service { - /** - * Value used to define no locality - */ - static final String ANY = "*"; - /** * Object to represent container request for resources. * Resources may be localized to nodes and racks. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java index 9f4358717bd..139a46c1988 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java @@ -258,7 +258,8 @@ public class AMRMClientImpl extends AbstractService implements AMRMClient { } // Off-switch - addResourceRequest(req.priority, ANY, req.capability, req.containerCount); + addResourceRequest(req.priority, ResourceRequest.ANY, req.capability, + req.containerCount); } @Override @@ -276,7 +277,8 @@ public class AMRMClientImpl extends AbstractService implements AMRMClient { } } - decResourceRequest(req.priority, ANY, req.capability, req.containerCount); + decResourceRequest(req.priority, ResourceRequest.ANY, req.capability, + req.containerCount); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java index fd298ce96a7..a4d65db032c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestAMRMClient.java @@ -18,18 +18,15 @@ package org.apache.hadoop.yarn.client; -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.util.List; import java.util.Set; import java.util.TreeSet; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; @@ -57,6 +54,11 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.apache.hadoop.yarn.service.Service.STATE; import org.apache.hadoop.yarn.util.Records; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestAMRMClient { Configuration conf = null; @@ -182,7 +184,7 @@ public class TestAMRMClient { int containersRequestedRack = amClient.remoteRequestsTable.get(priority) .get(rack).get(capability).getNumContainers(); int containersRequestedAny = amClient.remoteRequestsTable.get(priority) - .get(AMRMClient.ANY).get(capability).getNumContainers(); + .get(ResourceRequest.ANY).get(capability).getNumContainers(); assertTrue(containersRequestedNode == 2); assertTrue(containersRequestedRack == 2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index bb3e7338491..c58613e70c8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -38,9 +38,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; @@ -63,10 +63,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; @@ -737,7 +737,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { // Request a container for the AM. ResourceRequest request = BuilderUtils.newResourceRequest( - AM_CONTAINER_PRIORITY, "*", appAttempt.submissionContext + AM_CONTAINER_PRIORITY, ResourceRequest.ANY, appAttempt.submissionContext .getAMContainerSpec().getResource(), 1); Allocation amContainerAllocation = appAttempt.scheduler.allocate( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index 9d1103d4fc0..82986702601 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -36,8 +36,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; */ public interface RMNode { - public static final String ANY = "*"; - /** * the node id of of this node. * @return the node id of this node. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index 452cb9b6138..645d34bba3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -38,8 +38,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; /** * This class keeps track of all the consumption of an application. This also @@ -129,7 +127,7 @@ public class AppSchedulingInfo { boolean updatePendingResources = false; ResourceRequest lastRequest = null; - if (hostName.equals(RMNode.ANY)) { + if (hostName.equals(ResourceRequest.ANY)) { if (LOG.isDebugEnabled()) { LOG.debug("update:" + " application=" + applicationId + " request=" + request); @@ -195,7 +193,7 @@ public class AppSchedulingInfo { } public synchronized Resource getResource(Priority priority) { - ResourceRequest request = getResourceRequest(priority, RMNode.ANY); + ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); return request.getCapability(); } @@ -261,7 +259,7 @@ public class AppSchedulingInfo { this.requests.get(priority).remove(node.getRackName()); } - decrementOutstanding(requests.get(priority).get(RMNode.ANY)); + decrementOutstanding(requests.get(priority).get(ResourceRequest.ANY)); } /** @@ -284,7 +282,7 @@ public class AppSchedulingInfo { this.requests.get(priority).remove(node.getRackName()); } - decrementOutstanding(requests.get(priority).get(RMNode.ANY)); + decrementOutstanding(requests.get(priority).get(ResourceRequest.ANY)); } /** @@ -322,7 +320,7 @@ public class AppSchedulingInfo { synchronized private void checkForDeactivation() { boolean deactivate = true; for (Priority priority : getPriorities()) { - ResourceRequest request = getResourceRequest(priority, RMNodeImpl.ANY); + ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY); if (request.getNumContainers() > 0) { deactivate = false; break; @@ -351,7 +349,7 @@ public class AppSchedulingInfo { // clear pending resources metrics for the application QueueMetrics metrics = queue.getMetrics(); for (Map asks : requests.values()) { - ResourceRequest request = asks.get(RMNode.ANY); + ResourceRequest request = asks.get(ResourceRequest.ANY); if (request != null) { metrics.decrPendingResources(user, request.getNumContainers(), Resources.multiply(request.getCapability(), request diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index efc6e3efb25..d0e03a73ee4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -816,7 +815,8 @@ public class LeafQueue implements CSQueue { for (Priority priority : application.getPriorities()) { // Required resource Resource required = - application.getResourceRequest(priority, RMNode.ANY).getCapability(); + application.getResourceRequest( + priority, ResourceRequest.ANY).getCapability(); // Do we need containers at this 'priority'? if (!needContainers(application, priority, required)) { @@ -1161,7 +1161,7 @@ public class LeafQueue implements CSQueue { FiCaSchedulerApp application, Priority priority, RMContainer reservedContainer) { ResourceRequest request = - application.getResourceRequest(priority, RMNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { if (canAssign(application, priority, node, NodeType.OFF_SWITCH, reservedContainer)) { @@ -1183,7 +1183,7 @@ public class LeafQueue implements CSQueue { // 'Delay' off-switch ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, RMNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); long missedOpportunities = application.getSchedulingOpportunities(priority); long requiredContainers = offSwitchRequest.getNumContainers(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 317f2a211bd..1ff6e024f6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -51,7 +51,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; @@ -153,7 +152,7 @@ public class FiCaSchedulerApp extends SchedulerApplication { } public synchronized int getTotalRequiredResources(Priority priority) { - return getResourceRequest(priority, RMNode.ANY).getNumContainers(); + return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers(); } public Resource getResource(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java index b84607560ec..f6bac00218d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java @@ -60,8 +60,6 @@ public class FiCaSchedulerNode extends SchedulerNode { private final RMNode rmNode; - public static final String ANY = "*"; - public FiCaSchedulerNode(RMNode node) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java index 46a6fe9bb7a..32328be5a68 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -336,7 +335,7 @@ public class AppSchedulable extends Schedulable { } ResourceRequest offSwitchRequest = app.getResourceRequest(priority, - RMNode.ANY); + ResourceRequest.ANY); if (offSwitchRequest != null && offSwitchRequest.getNumContainers() != 0 && allowedLocality.equals(NodeType.OFF_SWITCH)) { return assignContainer(node, app, priority, offSwitchRequest, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java index 49a52bbb8dc..0734a4aead5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java @@ -40,8 +40,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; -import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; @@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo; @@ -153,7 +152,7 @@ public class FSSchedulerApp extends SchedulerApplication { } public synchronized int getTotalRequiredResources(Priority priority) { - return getResourceRequest(priority, RMNode.ANY).getNumContainers(); + return getResourceRequest(priority, ResourceRequest.ANY).getNumContainers(); } public Resource getResource(Priority priority) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java index 2833ca67b95..efc1c9f6842 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java @@ -59,8 +59,6 @@ public class FSSchedulerNode extends SchedulerNode { private final RMNode rmNode; - public static final String ANY = "*"; - public FSSchedulerNode(RMNode node) { this.rmNode = node; this.availableResource.setMemory(node.getTotalCapability().getMemory()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 3f255378231..df143566a7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -57,8 +57,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; @@ -393,7 +393,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { private int getMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority, FiCaSchedulerNode node, NodeType type) { ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); int maxContainers = offSwitchRequest.getNumContainers(); if (type == NodeType.OFF_SWITCH) { @@ -483,7 +483,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { if (request != null) { // Don't allocate on this rack if the application doens't need containers ResourceRequest offSwitchRequest = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (offSwitchRequest.getNumContainers() <= 0) { return 0; } @@ -504,7 +504,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { FiCaSchedulerApp application, Priority priority) { int assignedContainers = 0; ResourceRequest request = - application.getResourceRequest(priority, FiCaSchedulerNode.ANY); + application.getResourceRequest(priority, ResourceRequest.ANY); if (request != null) { assignedContainers = assignContainer(node, application, priority, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java index ee6cc99f0bf..20ffeabfeba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java @@ -43,11 +43,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.Task.State; @@ -200,9 +198,7 @@ public class Application { } // Off-switch - addResourceRequest(priority, requests, - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY, - capability); + addResourceRequest(priority, requests, ResourceRequest.ANY, capability); } public synchronized void finishTask(Task task) throws IOException { @@ -377,10 +373,7 @@ public class Application { } } - updateResourceRequest( - requests.get( - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY) - ); + updateResourceRequest(requests.get(ResourceRequest.ANY)); if(LOG.isDebugEnabled()) { LOG.debug("updateResourceRequests:" + " application=" + applicationId diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java index 9eb5bfb9b6a..1ed4652a96e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java @@ -120,8 +120,8 @@ public class MockAM { reqs.add(rackReq); } - ResourceRequest offRackReq = createResourceReq("*", memory, priority, - containers); + ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory, + priority, containers); reqs.add(offRackReq); return reqs; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java index 0fac0cd02ab..eebfa1da9e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Task.java @@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; public class Task { @@ -57,8 +58,7 @@ public class Task { // Special case: Don't care about locality if (!(hosts.length == 1 && - hosts[0].equals( - org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY))) { + hosts[0].equals(ResourceRequest.ANY))) { for (String host : hosts) { this.hosts.add(host); this.racks.add(Application.resolve(host)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 8965f40f1b1..4780ec38821 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -264,14 +264,14 @@ public class TestFifoScheduler { // Ask for a 1 GB container for app 1 List ask1 = new ArrayList(); - ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(GB, 1), 1)); + ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1)); fs.allocate(appAttemptId1, ask1, emptyId); // Ask for a 2 GB container for app 2 List ask2 = new ArrayList(); - ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(2 * GB, 1), 1)); + ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1)); fs.allocate(appAttemptId2, ask2, emptyId); // Trigger container assignment diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java index 1596bfddd6e..4b2323d241d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; @@ -124,7 +125,7 @@ public class TestResourceManager { Task t2 = new Task(application, priority1, new String[] {host1, host2}); application.addTask(t2); - Task t3 = new Task(application, priority0, new String[] {RMNode.ANY}); + Task t3 = new Task(application, priority0, new String[] {ResourceRequest.ANY}); application.addTask(t3); // Send resource requests to the scheduler diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 28894358fd2..5953f843df3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -47,10 +47,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.junit.After; @@ -506,7 +505,7 @@ public class TestApplicationLimits { List app_0_0_requests = new ArrayList(); app_0_0_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_0_0.updateResourceRequests(app_0_0_requests); @@ -525,7 +524,7 @@ public class TestApplicationLimits { List app_0_1_requests = new ArrayList(); app_0_1_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_0_1.updateResourceRequests(app_0_1_requests); @@ -544,7 +543,7 @@ public class TestApplicationLimits { List app_1_0_requests = new ArrayList(); app_1_0_requests.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); app_1_0.updateResourceRequests(app_1_0_requests); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index d3a7b0c35f7..48717ed8d02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.server.resourcemanager.Application; @@ -183,7 +184,7 @@ public class TestCapacityScheduler { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); application_1.schedule(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 828c796d011..65b9d0cc1f9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -24,8 +24,8 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -55,13 +55,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; -import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -287,7 +286,7 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); // Start testing... @@ -409,11 +408,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Start testing... @@ -542,11 +541,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -635,11 +634,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); /** @@ -674,7 +673,7 @@ public class TestLeafQueue { // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, priority, recordFactory))); assertEquals(2, a.getActiveUsersManager().getNumActiveUsers()); @@ -691,7 +690,7 @@ public class TestLeafQueue { // Check headroom for app_2 LOG.info("here"); app_1.updateResourceRequests(Collections.singletonList( // unset - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 0, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, priority, recordFactory))); assertEquals(1, a.getActiveUsersManager().getNumActiveUsers()); a.assignContainers(clusterResource, node_1); @@ -752,11 +751,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 10, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 10, priority, recordFactory))); /** @@ -786,11 +785,11 @@ public class TestLeafQueue { // Submit resource requests for other apps now to 'activate' them app_2.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 3*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 3*GB, 1, priority, recordFactory))); app_3.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); // Now allocations should goto app_2 since @@ -914,11 +913,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1016,7 +1015,7 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority, recordFactory))); // Setup app_1 to request a 4GB container on host_0 and @@ -1027,7 +1026,7 @@ public class TestLeafQueue { priority, recordFactory)); appRequests_1.add(TestUtils.createResourceRequest(DEFAULT_RACK, 4*GB, 1, priority, recordFactory)); - appRequests_1.add(TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 2, + appRequests_1.add(TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 2, priority, recordFactory)); app_1.updateResourceRequests(appRequests_1); @@ -1122,11 +1121,11 @@ public class TestLeafQueue { // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority, recordFactory))); app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 4*GB, 1, priority, + TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, priority, recordFactory))); // Start testing... @@ -1248,7 +1247,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 3, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1313,7 +1312,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, // one extra + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, // one extra priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); assertEquals(2, app_0.getTotalRequiredResources(priority)); @@ -1391,7 +1390,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_1, 1*GB, 1, priority_1, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 2, + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, priority_1, recordFactory)); // P2 @@ -1403,7 +1402,7 @@ public class TestLeafQueue { TestUtils.createResourceRequest(rack_2, 2*GB, 1, priority_2, recordFactory)); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 2*GB, 1, + TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, priority_2, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1528,7 +1527,7 @@ public class TestLeafQueue { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); @@ -1551,7 +1550,7 @@ public class TestLeafQueue { // Add one request app_0_requests_0.clear(); app_0_requests_0.add( - TestUtils.createResourceRequest(RMNodeImpl.ANY, 1*GB, 1, // only one + TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, // only one priority, recordFactory)); app_0.updateResourceRequests(app_0_requests_0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 709005c417d..caab5ed0725 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -142,7 +141,8 @@ public class TestFairScheduler { } - private ResourceRequest createResourceRequest(int memory, String host, int priority, int numContainers) { + private ResourceRequest createResourceRequest(int memory, String host, + int priority, int numContainers) { ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class); request.setCapability(Resources.createResource(memory)); request.setHostName(host); @@ -157,27 +157,33 @@ public class TestFairScheduler { * Creates a single container priority-1 request and submits to * scheduler. */ - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId) { return createSchedulingRequest(memory, queueId, userId, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers) { return createSchedulingRequest(memory, queueId, userId, numContainers, 1); } - private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, String userId, int numContainers, int priority) { + private ApplicationAttemptId createSchedulingRequest(int memory, String queueId, + String userId, int numContainers, int priority) { ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++); scheduler.addApplication(id, queueId, userId); List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, "*", priority, numContainers); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, numContainers); ask.add(request); scheduler.allocate(id, ask, new ArrayList()); return id; } - private void createSchedulingRequestExistingApplication(int memory, int priority, ApplicationAttemptId attId) { + private void createSchedulingRequestExistingApplication(int memory, int priority, + ApplicationAttemptId attId) { List ask = new ArrayList(); - ResourceRequest request = createResourceRequest(memory, "*", priority, 1); + ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY, + priority, 1); ask.add(request); scheduler.allocate(attId, ask, new ArrayList()); } @@ -459,7 +465,8 @@ public class TestFairScheduler { // First ask, queue1 requests 1 large (minReqSize * 2). List ask1 = new ArrayList(); - ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1); + ResourceRequest request1 = + createResourceRequest(minReqSize * 2, ResourceRequest.ANY, 1, 1); ask1.add(request1); scheduler.allocate(id11, ask1, new ArrayList()); @@ -473,7 +480,8 @@ public class TestFairScheduler { // Third ask, queue2 requests 1 large List ask3 = new ArrayList(); - ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1); + ResourceRequest request4 = + createResourceRequest(2 * minReqSize, ResourceRequest.ANY, 1, 1); ask3.add(request4); scheduler.allocate(id22, ask3, new ArrayList()); @@ -1311,7 +1319,7 @@ public class TestFairScheduler { asks.add(createResourceRequest(1024, node3.getHostName(), 1, 1)); asks.add(createResourceRequest(1024, node1.getRackName(), 1, 1)); asks.add(createResourceRequest(1024, node3.getRackName(), 1, 1)); - asks.add(createResourceRequest(1024, RMNode.ANY, 1, 2)); + asks.add(createResourceRequest(1024, ResourceRequest.ANY, 1, 2)); scheduler.allocate(appId, asks, new ArrayList()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index c4dbe876c78..8f45f4535cb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.InlineDispatcher; @@ -40,7 +41,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.Task; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; @@ -200,15 +200,15 @@ public class TestFifoScheduler { LOG.info("Adding new tasks..."); Task task_1_1 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_1); Task task_1_2 = new Task(application_1, priority_1, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_2); Task task_1_3 = new Task(application_1, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_1.addTask(task_1_3); application_1.schedule(); @@ -222,7 +222,7 @@ public class TestFifoScheduler { application_0.addTask(task_0_2); Task task_0_3 = new Task(application_0, priority_0, - new String[] {RMNode.ANY}); + new String[] {ResourceRequest.ANY}); application_0.addTask(task_0_3); application_0.schedule(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java index 2f810fdb483..27986cc1a96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java @@ -22,7 +22,6 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -70,11 +69,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerToken; import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.LocalResourceType; -import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; -import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -89,7 +85,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -486,8 +481,8 @@ public class TestContainerManagerSecurity { // Request a container allocation. List ask = new ArrayList(); - ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), "*", - BuilderUtils.newResource(1024, 1), 1)); + ask.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0), + ResourceRequest.ANY, BuilderUtils.newResource(1024, 1), 1)); AllocateRequest allocateRequest = BuilderUtils.newAllocateRequest( BuilderUtils.newApplicationAttemptId(appID, 1), 0, 0F, ask, From 1611b51a9780d18e76c0e9f9777bfae6ee70df12 Mon Sep 17 00:00:00 2001 From: Todd Lipcon Date: Thu, 28 Mar 2013 20:58:45 +0000 Subject: [PATCH 34/37] HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs. Contributed by Todd Lipcon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1462303 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../java/org/apache/hadoop/fs/FileSystem.java | 43 ++++++++++- .../apache/hadoop/fs/FilterFileSystem.java | 14 ++-- .../fs/TestFileSystemCanonicalization.java | 6 ++ .../apache/hadoop/test/GenericTestUtils.java | 19 ++++- .../hadoop/hdfs/DistributedFileSystem.java | 12 +++ .../apache/hadoop/hdfs/HftpFileSystem.java | 5 ++ .../hadoop/hdfs/web/WebHdfsFileSystem.java | 5 ++ .../hadoop/hdfs/TestDFSClientFailover.java | 77 +++++++++++++++++++ 9 files changed, 172 insertions(+), 12 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 309ea50241a..6d8e2f93617 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -538,6 +538,9 @@ Release 2.0.5-beta - UNRELEASED OPTIMIZATIONS + HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs + (todd) + BUG FIXES HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index a26d3570586..ca80e91abc3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -21,6 +21,7 @@ import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; +import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.Arrays; @@ -211,12 +212,46 @@ public abstract class FileSystem extends Configured implements Closeable { public abstract URI getUri(); /** - * Resolve the uri's hostname and add the default port if not in the uri + * Return a canonicalized form of this FileSystem's URI. + * + * The default implementation simply calls {@link #canonicalizeUri(URI)} + * on the filesystem's own URI, so subclasses typically only need to + * implement that method. + * + * @see #canonicalizeUri(URI) + */ + protected URI getCanonicalUri() { + return canonicalizeUri(getUri()); + } + + /** + * Canonicalize the given URI. + * + * This is filesystem-dependent, but may for example consist of + * canonicalizing the hostname using DNS and adding the default + * port if not specified. + * + * The default implementation simply fills in the default port if + * not specified and if the filesystem has a default port. + * * @return URI * @see NetUtils#getCanonicalUri(URI, int) */ - protected URI getCanonicalUri() { - return NetUtils.getCanonicalUri(getUri(), getDefaultPort()); + protected URI canonicalizeUri(URI uri) { + if (uri.getPort() == -1 && getDefaultPort() > 0) { + // reconstruct the uri with the default port set + try { + uri = new URI(uri.getScheme(), uri.getUserInfo(), + uri.getHost(), getDefaultPort(), + uri.getPath(), uri.getQuery(), uri.getFragment()); + } catch (URISyntaxException e) { + // Should never happen! + throw new AssertionError("Valid URI became unparseable: " + + uri); + } + } + + return uri; } /** @@ -581,7 +616,7 @@ public abstract class FileSystem extends Configured implements Closeable { } if (uri != null) { // canonicalize uri before comparing with this fs - uri = NetUtils.getCanonicalUri(uri, getDefaultPort()); + uri = canonicalizeUri(uri); thatAuthority = uri.getAuthority(); if (thisAuthority == thatAuthority || // authorities match (thisAuthority != null && diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index a2b34f056ba..9fc8bb2d974 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -95,16 +95,18 @@ public class FilterFileSystem extends FileSystem { public URI getUri() { return fs.getUri(); } - - /** - * Returns a qualified URI whose scheme and authority identify this - * FileSystem. - */ + + @Override protected URI getCanonicalUri() { return fs.getCanonicalUri(); } - + + @Override + protected URI canonicalizeUri(URI uri) { + return fs.canonicalizeUri(uri); + } + /** Make sure that a path specifies a FileSystem. */ @Override public Path makeQualified(Path path) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java index ac9c05e89c8..2b8be39193a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java @@ -26,6 +26,7 @@ import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.util.Progressable; import org.junit.BeforeClass; @@ -312,6 +313,11 @@ public class TestFileSystemCanonicalization { return defaultPort; } + @Override + protected URI canonicalizeUri(URI uri) { + return NetUtils.getCanonicalUri(uri, getDefaultPort()); + } + @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { throw new IOException("not supposed to be here"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java index bfb52a8f928..5b08058d081 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java @@ -20,6 +20,7 @@ package org.apache.hadoop.test; import java.io.File; import java.io.IOException; import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.Random; import java.util.Set; @@ -266,15 +267,29 @@ public abstract class GenericTestUtils { */ public static class DelegateAnswer implements Answer { private final Object delegate; + private final Log log; public DelegateAnswer(Object delegate) { + this(null, delegate); + } + + public DelegateAnswer(Log log, Object delegate) { + this.log = log; this.delegate = delegate; } @Override public Object answer(InvocationOnMock invocation) throws Throwable { - return invocation.getMethod().invoke( - delegate, invocation.getArguments()); + try { + if (log != null) { + log.info("Call to " + invocation + " on " + delegate, + new Exception("TRACE")); + } + return invocation.getMethod().invoke( + delegate, invocation.getArguments()); + } catch (InvocationTargetException ite) { + throw ite.getCause(); + } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 3476f678708..e772859b8ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; @@ -893,6 +894,17 @@ public class DistributedFileSystem extends FileSystem { public String getCanonicalServiceName() { return dfs.getCanonicalServiceName(); } + + @Override + protected URI canonicalizeUri(URI uri) { + if (HAUtil.isLogicalUri(getConf(), uri)) { + // Don't try to DNS-resolve logical URIs, since the 'authority' + // portion isn't a proper hostname + return uri; + } else { + return NetUtils.getCanonicalUri(uri, getDefaultPort()); + } + } /** * Utility function that returns if the NameNode is in safemode or not. In HA diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java index d97c62012e9..7d97c972532 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java @@ -161,6 +161,11 @@ public class HftpFileSystem extends FileSystem // actual port in the uri return SecurityUtil.buildTokenService(nnSecureUri).toString(); } + + @Override + protected URI canonicalizeUri(URI uri) { + return NetUtils.getCanonicalUri(uri, getDefaultPort()); + } /** * Return the protocol scheme for the FileSystem. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 3a73d5c0e79..6f33827fa74 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -238,6 +238,11 @@ public class WebHdfsFileSystem extends FileSystem public URI getUri() { return this.uri; } + + @Override + protected URI canonicalizeUri(URI uri) { + return NetUtils.getCanonicalUri(uri, getDefaultPort()); + } /** @return the home directory. */ public static String getHomeDirectoryString(final UserGroupInformation ugi) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java index 86bcae3a0a2..aacebce3595 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java @@ -26,8 +26,11 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketAddress; +import java.lang.reflect.Field; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; import javax.net.SocketFactory; @@ -35,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -48,10 +52,13 @@ import org.apache.hadoop.util.StringUtils; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; import org.junit.After; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import sun.net.spi.nameservice.NameService; + public class TestDFSClientFailover { private static final Log LOG = LogFactory.getLog(TestDFSClientFailover.class); @@ -201,4 +208,74 @@ public class TestDFSClientFailover { "Could not find any configured addresses for URI " + uri)); } } + + /** + * Spy on the Java DNS infrastructure. + * This likely only works on Sun-derived JDKs, but uses JUnit's + * Assume functionality so that any tests using it are skipped on + * incompatible JDKs. + */ + private NameService spyOnNameService() { + try { + Field f = InetAddress.class.getDeclaredField("nameServices"); + f.setAccessible(true); + Assume.assumeNotNull(f); + @SuppressWarnings("unchecked") + List nsList = (List) f.get(null); + + NameService ns = nsList.get(0); + Log log = LogFactory.getLog("NameServiceSpy"); + + ns = Mockito.mock(NameService.class, + new GenericTestUtils.DelegateAnswer(log, ns)); + nsList.set(0, ns); + return ns; + } catch (Throwable t) { + LOG.info("Unable to spy on DNS. Skipping test.", t); + // In case the JDK we're testing on doesn't work like Sun's, just + // skip the test. + Assume.assumeNoException(t); + throw new RuntimeException(t); + } + } + + /** + * Test that the client doesn't ever try to DNS-resolve the logical URI. + * Regression test for HADOOP-9150. + */ + @Test + public void testDoesntDnsResolveLogicalURI() throws Exception { + NameService spyNS = spyOnNameService(); + + FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); + String logicalHost = fs.getUri().getHost(); + Path qualifiedRoot = fs.makeQualified(new Path("/")); + + // Make a few calls against the filesystem. + fs.getCanonicalServiceName(); + fs.listStatus(qualifiedRoot); + + // Ensure that the logical hostname was never resolved. + Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost)); + } + + /** + * Same test as above, but for FileContext. + */ + @Test + public void testFileContextDoesntDnsResolveLogicalURI() throws Exception { + NameService spyNS = spyOnNameService(); + FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf); + String logicalHost = fs.getUri().getHost(); + Configuration haClientConf = fs.getConf(); + + FileContext fc = FileContext.getFileContext(haClientConf); + Path root = new Path("/"); + fc.listStatus(root); + fc.listStatus(fc.makeQualified(root)); + fc.getDefaultFileSystem().getCanonicalServiceName(); + + // Ensure that the logical hostname was never resolved. + Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost)); + } } From 1b87baa06f837d4c0b024388f12edb18e91bf0d1 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 29 Mar 2013 06:29:22 +0000 Subject: [PATCH 35/37] HADOOP-9444. Modify hadoop-policy.xml to replace unexpanded variables to a default value of '*'. Contributed by Roman Shaposhnik. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1462417 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/main/conf/hadoop-policy.xml | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6d8e2f93617..82d7ad8a021 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -627,6 +627,9 @@ Release 2.0.4-alpha - UNRELEASED HADOOP-9408. misleading description for net.topology.table.file.name property in core-default.xml. (rajeshbabu via suresh) + HADOOP-9444. Modify hadoop-policy.xml to replace unexpanded variables to a + default value of '*'. (Roman Shaposhnik via vinodkv) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml index 4dd5b9fbb02..c17966d7f08 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml @@ -77,7 +77,7 @@ security.admin.operations.protocol.acl - ${HADOOP_HDFS_USER} + * ACL for AdminOperationsProtocol. Used for admin commands. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. "alice,bob users,wheel". @@ -86,7 +86,7 @@ security.refresh.usertogroups.mappings.protocol.acl - ${HADOOP_HDFS_USER} + * ACL for RefreshUserMappingsProtocol. Used to refresh users mappings. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For @@ -96,7 +96,7 @@ security.refresh.policy.protocol.acl - ${HADOOP_HDFS_USER} + * ACL for RefreshAuthorizationPolicyProtocol, used by the dfsadmin and mradmin commands to refresh the security policy in-effect. The ACL is a comma-separated list of user and group names. The user and @@ -120,7 +120,7 @@ security.qjournal.service.protocol.acl - ${HADOOP_HDFS_USER} + * ACL for QJournalProtocol, used by the NN to communicate with JNs when using the QuorumJournalManager for edit logs. @@ -139,7 +139,7 @@ security.resourcetracker.protocol.acl - ${HADOOP_YARN_USER} + * ACL for ResourceTracker protocol, used by the ResourceManager and NodeManager to communicate with each other. The ACL is a comma-separated list of user and group names. The user and @@ -149,7 +149,7 @@ security.admin.protocol.acl - ${HADOOP_YARN_USER} + * ACL for RMAdminProtocol, for admin commands. The ACL is a comma-separated list of user and group names. The user and group list is separated by a blank. For e.g. "alice,bob users,wheel". From 01aabf73639785a29d1651f2e268cec04a8b9b94 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 29 Mar 2013 14:36:53 +0000 Subject: [PATCH 36/37] YARN-460. CS user left in list of active users for the queue even when application finished (tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1462486 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../scheduler/capacity/CapacityScheduler.java | 8 ++++++++ .../scheduler/common/fica/FiCaSchedulerApp.java | 16 +++++++++++++++- .../scheduler/fifo/FifoScheduler.java | 9 +++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 7114465aea3..42e5e0373dc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -499,6 +499,9 @@ Release 0.23.7 - UNRELEASED YARN-109. .tmp file is not deleted for localized archives (Mayank Bansal via bobby) + YARN-460. CS user left in list of active users for the queue even when + application finished (tgraves) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 2fc754069fd..1b3858cc864 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -504,6 +504,14 @@ implements ResourceScheduler, CapacitySchedulerContext, Configurable { synchronized (application) { + // make sure we aren't stopping/removing the application + // when the allocate comes in + if (application.isStopped()) { + LOG.info("Calling allocate on a stopped " + + "application " + applicationAttemptId); + return EMPTY_ALLOCATION; + } + if (!ask.isEmpty()) { if(LOG.isDebugEnabled()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 1ff6e024f6a..f7a8f7cb1e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -91,6 +91,9 @@ public class FiCaSchedulerApp extends SchedulerApplication { final Map> reservedContainers = new HashMap>(); + + private boolean isStopped = false; + /** * Count how many times the application has been given an opportunity @@ -132,7 +135,9 @@ public class FiCaSchedulerApp extends SchedulerApplication { public synchronized void updateResourceRequests( List requests) { - this.appSchedulingInfo.updateResourceRequests(requests); + if (!isStopped) { + this.appSchedulingInfo.updateResourceRequests(requests); + } } public Map getResourceRequests(Priority priority) { @@ -168,6 +173,10 @@ public class FiCaSchedulerApp extends SchedulerApplication { return this.appSchedulingInfo.isPending(); } + public synchronized boolean isStopped() { + return this.isStopped; + } + public String getQueueName() { return this.appSchedulingInfo.getQueueName(); } @@ -183,6 +192,7 @@ public class FiCaSchedulerApp extends SchedulerApplication { public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) { // Cleanup all scheduling information + this.isStopped = true; this.appSchedulingInfo.stop(rmAppAttemptFinalState); } @@ -234,6 +244,10 @@ public class FiCaSchedulerApp extends SchedulerApplication { synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node, Priority priority, ResourceRequest request, Container container) { + + if (isStopped) { + return null; + } // Required sanity check - AM can call 'allocate' to update resource // request without locking the scheduler, hence we need to check diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index df143566a7d..56d66c34e36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -252,6 +252,15 @@ public class FifoScheduler implements ResourceScheduler, Configurable { } synchronized (application) { + + // make sure we aren't stopping/removing the application + // when the allocate comes in + if (application.isStopped()) { + LOG.info("Calling allocate on a stopped " + + "application " + applicationAttemptId); + return EMPTY_ALLOCATION; + } + if (!ask.isEmpty()) { LOG.debug("allocate: pre-update" + " applicationId=" + applicationAttemptId + From 2fd56e33b4171c73491f58b2eeb21ef1d8eaa67a Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Fri, 29 Mar 2013 20:20:19 +0000 Subject: [PATCH 37/37] YARN-515. Node Manager not getting the master key. Contributed by Robert Joseph Evans git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1462632 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../pb/RegisterNodeManagerResponsePBImpl.java | 6 +- .../TestRegisterNodeManagerResponse.java | 75 +++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestRegisterNodeManagerResponse.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 42e5e0373dc..d998803695a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -158,6 +158,9 @@ Release 2.0.5-beta - UNRELEASED YARN-24. Nodemanager fails to start if log aggregation enabled and namenode unavailable. (sandyr via tucu) + YARN-515. Node Manager not getting the master key. (Robert Joseph Evans + via jlowe) + Release 2.0.4-alpha - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java index e8194af4053..c28d4c96103 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java @@ -98,6 +98,7 @@ public class RegisterNodeManagerResponsePBImpl extends ProtoBase