Merge r1460409 through r1462697 from trunk.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1462698 13f79535-47bb-0310-9956-ffa450edef68

BUILDING.txt

@@ -139,6 +139,15 @@ Create a local staging version of the website (in /tmp/hadoop-site)
 
 ----------------------------------------------------------------------------------
 
+Building on OS/X
+
+----------------------------------------------------------------------------------
+
+Hadoop does not build on OS/X with Java 7.
+see: https://issues.apache.org/jira/browse/HADOOP-9350
+
+----------------------------------------------------------------------------------
+
 Building on Windows
 
 ----------------------------------------------------------------------------------

dev-support/test-patch.sh

@@ -323,7 +323,7 @@ checkAuthor () {
 }
 
 ###############################################################################
-### Check for tests and their timeout in the patch
+### Check for tests in the patch
 checkTests () {
   echo ""
   echo ""
@@ -357,25 +357,7 @@ checkTests () {
   JIRA_COMMENT="$JIRA_COMMENT
 
     {color:green}+1 tests included{color}.  The patch appears to include $testReferences new or modified test files."
-  echo ""
+  return 0
-  echo "======================================================================"
-  echo "======================================================================"
-  echo "    Checking if the tests have timeout assigned in this patch."
-  echo "======================================================================"
-  echo "======================================================================"
-  
-  nontimeoutTests=`cat $PATCH_DIR/patch | $AWK '{ printf "%s ", $0 }'  | $GREP --extended-regex --count '[ ]*\+[ ]*((@Test[\+ ]*[A-Za-z]+)|([\+ ]*@Test[ \+]*\([ \+]*\)[\ ]*\+?[ ]*[A-Za-z]+)|([\+ ]*@Test[\+ ]*\(exception[ \+]*=[ \+]*[A-Z.a-z0-9A-Z ]*\)))'`
-
-  if [[ $nontimeoutTests == 0 ]] ; then
-    JIRA_COMMENT="$JIRA_COMMENT
-
-    {color:green}+1 tests included appear to have a timeout.{color}"
-	return 0
-  fi
-  JIRA_COMMENT="$JIRA_COMMENT
-
-  {color:red}-1 one of tests included doesn't have a timeout.{color}"
-  return 1
 }
 
 cleanUpXml () {

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -17,6 +17,8 @@ Trunk (Unreleased)
 
     HADOOP-9380 Add totalLength to rpc response  (sanjay Radia)
 
+    HADOOP-9194. RPC Support for QoS. (Junping Du via llu)
+
   NEW FEATURES
     
     HADOOP-8561. Introduce HADOOP_PROXY_USER for secure impersonation in child
@@ -90,9 +92,6 @@ Trunk (Unreleased)
     HADOOP-8367 Improve documentation of declaringClassProtocolName in 
     rpc headers. (Sanjay Radia)
 
-    HADOOP-8415. Add getDouble() and setDouble() in
-    org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
-
     HADOOP-7659. fs -getmerge isn't guaranteed to work well over non-HDFS
     filesystems (harsh)
 
@@ -162,9 +161,7 @@ Trunk (Unreleased)
 
     HADOOP-9218 Document the Rpc-wrappers used internally (sanjay Radia)
 
-    HADOOP-9112. test-patch should -1 for @Tests without a timeout 
+	HADOOP-9258 Add stricter tests to FileSystemContractTestBase (stevel)
-    (Surenkumar Nihalani via bobby)
-
 
   BUG FIXES
 
@@ -358,6 +355,15 @@ Trunk (Unreleased)
     HADOOP-9431 TestSecurityUtil#testLocalHostNameForNullOrWild on systems where hostname
     contains capital letters  (Chris Nauroth via sanjay)
 
+    HADOOP-9261 S3n filesystem can move a directory under itself -and so lose data
+    (fixed in HADOOP-9258) (stevel)
+
+    HADOOP-9265 S3 blockstore filesystem breaks part of the Filesystem contract
+    (fixed in HADOOP-9258) (stevel)
+
+    HADOOP-9433 TestLocalFileSystem#testHasFileDescriptor leaks file handle
+    (Chris Nauroth via sanjay)
+
   OPTIMIZATIONS
 
     HADOOP-7761. Improve the performance of raw comparisons. (todd)
@@ -505,6 +511,9 @@ Release 2.0.5-beta - UNRELEASED
 
     HADOOP-9283. Add support for running the Hadoop client on AIX. (atm)
 
+    HADOOP-8415. Add getDouble() and setDouble() in
+    org.apache.hadoop.conf.Configuration (Jan van der Lugt via harsh)
+
   IMPROVEMENTS
 
     HADOOP-9253. Capture ulimit info in the logs at service start time.
@@ -525,8 +534,13 @@ Release 2.0.5-beta - UNRELEASED
     HADOOP-9318. When exiting on a signal, print the signal name first. (Colin
     Patrick McCabe via atm)
 
+    HADOOP-9358. "Auth failed" log should include exception string (todd)
+
   OPTIMIZATIONS
 
+    HADOOP-9150. Avoid unnecessary DNS resolution attempts for logical URIs
+    (todd)
+
   BUG FIXES
 
     HADOOP-9294. GetGroupsTestBase fails on Windows. (Chris Nauroth via suresh)
@@ -585,6 +599,14 @@ Release 2.0.5-beta - UNRELEASED
     HADOOP-9299.  kerberos name resolution is kicking in even when kerberos
     is not configured (daryn)
 
+    HADOOP-9430. TestSSLFactory fails on IBM JVM. (Amir Sanjar via suresh)
+
+    HADOOP-9125. LdapGroupsMapping threw CommunicationException after some
+    idle time. (Kai Zheng via atm)
+
+    HADOOP-9357. Fallback to default authority if not specified in FileContext.
+    (Andrew Wang via eli)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -605,6 +627,9 @@ Release 2.0.4-alpha - UNRELEASED
     HADOOP-9408. misleading description for net.topology.table.file.name
     property in core-default.xml. (rajeshbabu via suresh)
 
+    HADOOP-9444. Modify hadoop-policy.xml to replace unexpanded variables to a
+    default value of '*'. (Roman Shaposhnik via vinodkv)
+
 Release 2.0.3-alpha - 2013-02-06 
 
   INCOMPATIBLE CHANGES

hadoop-common-project/hadoop-common/src/main/conf/hadoop-policy.xml

@@ -77,7 +77,7 @@
 
  <property>
     <name>security.admin.operations.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for AdminOperationsProtocol. Used for admin commands.
     The ACL is a comma-separated list of user and group names. The user and
     group list is separated by a blank. For e.g. "alice,bob users,wheel".
@@ -86,7 +86,7 @@
 
   <property>
     <name>security.refresh.usertogroups.mappings.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for RefreshUserMappingsProtocol. Used to refresh
     users mappings. The ACL is a comma-separated list of user and
     group names. The user and group list is separated by a blank. For
@@ -96,7 +96,7 @@
 
   <property>
     <name>security.refresh.policy.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for RefreshAuthorizationPolicyProtocol, used by the
     dfsadmin and mradmin commands to refresh the security policy in-effect.
     The ACL is a comma-separated list of user and group names. The user and
@@ -120,7 +120,7 @@
 
   <property>
     <name>security.qjournal.service.protocol.acl</name>
-    <value>${HADOOP_HDFS_USER}</value>
+    <value>*</value>
     <description>ACL for QJournalProtocol, used by the NN to communicate with
     JNs when using the QuorumJournalManager for edit logs.</description>
   </property>
@@ -139,7 +139,7 @@
 
   <property>
     <name>security.resourcetracker.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
+    <value>*</value>
     <description>ACL for ResourceTracker protocol, used by the
     ResourceManager and NodeManager to communicate with each other.
     The ACL is a comma-separated list of user and group names. The user and
@@ -149,7 +149,7 @@
 
   <property>
     <name>security.admin.protocol.acl</name>
-    <value>${HADOOP_YARN_USER}</value>
+    <value>*</value>
     <description>ACL for RMAdminProtocol, for admin commands. 
     The ACL is a comma-separated list of user and group names. The user and
     group list is separated by a blank. For e.g. "alice,bob users,wheel".

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java

@@ -244,17 +244,33 @@ public final class FileContext {
   }
  
   /* 
-   * Remove relative part - return "absolute":
+   * Resolve a relative path passed from the user.
-   * If input is relative path ("foo/bar") add wd: ie "/<workingDir>/foo/bar"
+   * 
-   * A fully qualified uri ("hdfs://nn:p/foo/bar") or a slash-relative path
+   * Relative paths are resolved against the current working directory
+   * (e.g. "foo/bar" becomes "/<workingDir>/foo/bar").
+   * Fully-qualified URIs (e.g. "hdfs://nn:p/foo/bar") and slash-relative paths
    * ("/foo/bar") are returned unchanged.
    * 
+   * Additionally, we fix malformed URIs that specify a scheme but not an 
+   * authority (e.g. "hdfs:///foo/bar"). Per RFC 2395, we remove the scheme
+   * if it matches the default FS, and let the default FS add in the default
+   * scheme and authority later (see {@link #AbstractFileSystem#checkPath}).
+   * 
    * Applications that use FileContext should use #makeQualified() since
-   * they really want a fully qualified URI.
+   * they really want a fully-qualified URI.
    * Hence this method is not called makeAbsolute() and 
    * has been deliberately declared private.
    */
   private Path fixRelativePart(Path p) {
+    // Per RFC 2396 5.2, drop schema if there is a scheme but no authority.
+    if (p.hasSchemeAndNoAuthority()) {
+      String scheme = p.toUri().getScheme();
+      if (scheme.equalsIgnoreCase(defaultFS.getUri().getScheme())) {
+        p = new Path(p.toUri().getSchemeSpecificPart());
+      }
+    }
+    // Absolute paths are unchanged. Relative paths are resolved against the
+    // current working directory.
     if (p.isUriPathAbsolute()) {
       return p;
     } else {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java

@@ -21,6 +21,7 @@ import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URI;
+import java.net.URISyntaxException;
 import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -211,12 +212,46 @@ public abstract class FileSystem extends Configured implements Closeable {
   public abstract URI getUri();
   
   /**
-   * Resolve the uri's hostname and add the default port if not in the uri
+   * Return a canonicalized form of this FileSystem's URI.
+   * 
+   * The default implementation simply calls {@link #canonicalizeUri(URI)}
+   * on the filesystem's own URI, so subclasses typically only need to
+   * implement that method.
+   *
+   * @see #canonicalizeUri(URI)
+   */
+  protected URI getCanonicalUri() {
+    return canonicalizeUri(getUri());
+  }
+  
+  /**
+   * Canonicalize the given URI.
+   * 
+   * This is filesystem-dependent, but may for example consist of
+   * canonicalizing the hostname using DNS and adding the default
+   * port if not specified.
+   * 
+   * The default implementation simply fills in the default port if
+   * not specified and if the filesystem has a default port.
+   *
    * @return URI
    * @see NetUtils#getCanonicalUri(URI, int)
    */
-  protected URI getCanonicalUri() {
+  protected URI canonicalizeUri(URI uri) {
-    return NetUtils.getCanonicalUri(getUri(), getDefaultPort());
+    if (uri.getPort() == -1 && getDefaultPort() > 0) {
+      // reconstruct the uri with the default port set
+      try {
+        uri = new URI(uri.getScheme(), uri.getUserInfo(),
+            uri.getHost(), getDefaultPort(),
+            uri.getPath(), uri.getQuery(), uri.getFragment());
+      } catch (URISyntaxException e) {
+        // Should never happen!
+        throw new AssertionError("Valid URI became unparseable: " +
+            uri);
+      }
+    }
+    
+    return uri;
   }
   
   /**
@@ -581,7 +616,7 @@ public abstract class FileSystem extends Configured implements Closeable {
       }
       if (uri != null) {
         // canonicalize uri before comparing with this fs
-        uri = NetUtils.getCanonicalUri(uri, getDefaultPort());
+        uri = canonicalizeUri(uri);
         thatAuthority = uri.getAuthority();
         if (thisAuthority == thatAuthority ||       // authorities match
             (thisAuthority != null &&

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java

@@ -95,16 +95,18 @@ public class FilterFileSystem extends FileSystem {
   public URI getUri() {
     return fs.getUri();
   }
-
+  
-  /**
+  
-   * Returns a qualified URI whose scheme and authority identify this
-   * FileSystem.
-   */
   @Override
   protected URI getCanonicalUri() {
     return fs.getCanonicalUri();
   }
-  
+
+  @Override
+  protected URI canonicalizeUri(URI uri) {
+    return fs.canonicalizeUri(uri);
+  }
+
   /** Make sure that a path specifies a FileSystem. */
   @Override
   public Path makeQualified(Path path) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java

@@ -256,6 +256,10 @@ public class Path implements Comparable {
     return  (isUriPathAbsolute() && 
         uri.getScheme() == null && uri.getAuthority() == null);
   }
+
+  public boolean hasSchemeAndNoAuthority() {
+    return uri.getScheme() != null && uri.getAuthority() == null;
+  }
   
   /**
    *  True if the path component (i.e. directory) of this URI is absolute.

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/Jets3tFileSystemStore.java

@@ -137,9 +137,15 @@ class Jets3tFileSystemStore implements FileSystemStore {
 
   @Override
   public boolean inodeExists(Path path) throws IOException {
-    InputStream in = get(pathToKey(path), true);
+    String key = pathToKey(path);
+    InputStream in = get(key, true);
     if (in == null) {
-      return false;
+      if (isRoot(key)) {
+        storeINode(path, INode.DIRECTORY_INODE);
+        return true;
+      } else {
+        return false;
+      }
     }
     in.close();
     return true;
@@ -211,7 +217,13 @@ class Jets3tFileSystemStore implements FileSystemStore {
 
   @Override
   public INode retrieveINode(Path path) throws IOException {
-    return INode.deserialize(get(pathToKey(path), true));
+    String key = pathToKey(path);
+    InputStream in = get(key, true);
+    if (in == null && isRoot(key)) {
+      storeINode(path, INode.DIRECTORY_INODE);
+      return INode.DIRECTORY_INODE;
+    }
+    return INode.deserialize(in);
   }
 
   @Override
@@ -366,6 +378,10 @@ class Jets3tFileSystemStore implements FileSystemStore {
     return blockToKey(block.getId());
   }
 
+  private boolean isRoot(String key) {
+    return key.isEmpty() || key.equals("/");
+  }
+
   @Override
   public void purge() throws IOException {
     try {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3/S3FileSystem.java

@@ -252,32 +252,125 @@ public class S3FileSystem extends FileSystem {
   @Override
   public boolean rename(Path src, Path dst) throws IOException {
     Path absoluteSrc = makeAbsolute(src);
+    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
     INode srcINode = store.retrieveINode(absoluteSrc);
+    boolean debugEnabled = LOG.isDebugEnabled();
     if (srcINode == null) {
       // src path doesn't exist
+      if (debugEnabled) {
+        LOG.debug(debugPreamble + "returning false as src does not exist");
+      }
       return false; 
     }
+
     Path absoluteDst = makeAbsolute(dst);
-    INode dstINode = store.retrieveINode(absoluteDst);
+
-    if (dstINode != null && dstINode.isDirectory()) {
+    //validate the parent dir of the destination
-      absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
-      dstINode = store.retrieveINode(absoluteDst);
-    }
-    if (dstINode != null) {
-      // dst path already exists - can't overwrite
-      return false;
-    }
     Path dstParent = absoluteDst.getParent();
     if (dstParent != null) {
+      //if the dst parent is not root, make sure it exists
       INode dstParentINode = store.retrieveINode(dstParent);
-      if (dstParentINode == null || dstParentINode.isFile()) {
+      if (dstParentINode == null) {
-        // dst parent doesn't exist or is a file
+        // dst parent doesn't exist
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "returning false as dst parent does not exist");
+        }
+        return false;
+      }
+      if (dstParentINode.isFile()) {
+        // dst parent exists but is a file
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "returning false as dst parent exists and is a file");
+        }
         return false;
       }
     }
+
+    //get status of source
+    boolean srcIsFile = srcINode.isFile();
+
+    INode dstINode = store.retrieveINode(absoluteDst);
+    boolean destExists = dstINode != null;
+    boolean destIsDir = destExists && !dstINode.isFile();
+    if (srcIsFile) {
+
+      //source is a simple file
+      if (destExists) {
+        if (destIsDir) {
+          //outcome #1 dest exists and is dir -filename to subdir of dest
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "copying src file under dest dir to " + absoluteDst);
+          }
+          absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+        } else {
+          //outcome #2 dest it's a file: fail iff different from src
+          boolean renamingOnToSelf = absoluteSrc.equals(absoluteDst);
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "copying file onto file, outcome is " + renamingOnToSelf);
+          }
+          return renamingOnToSelf;
+        }
+      } else {
+        // #3 dest does not exist: use dest as path for rename
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "copying file onto file");
+        }
+      }
+    } else {
+      //here the source exists and is a directory
+      // outcomes (given we know the parent dir exists if we get this far)
+      // #1 destination is a file: fail
+      // #2 destination is a directory: create a new dir under that one
+      // #3 destination doesn't exist: create a new dir with that name
+      // #3 and #4 are only allowed if the dest path is not == or under src
+
+      if (destExists) {
+        if (!destIsDir) {
+          // #1 destination is a file: fail
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "returning false as src is a directory, but not dest");
+          }
+          return false;
+        } else {
+          // the destination dir exists
+          // destination for rename becomes a subdir of the target name
+          absoluteDst = new Path(absoluteDst, absoluteSrc.getName());
+          if (debugEnabled) {
+            LOG.debug(debugPreamble +
+                      "copying src dir under dest dir to " + absoluteDst);
+          }
+        }
+      }
+      //the final destination directory is now know, so validate it for
+      //illegal moves
+
+      if (absoluteSrc.equals(absoluteDst)) {
+        //you can't rename a directory onto itself
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "Dest==source && isDir -failing");
+        }
+        return false;
+      }
+      if (absoluteDst.toString().startsWith(absoluteSrc.toString() + "/")) {
+        //you can't move a directory under itself
+        if (debugEnabled) {
+          LOG.debug(debugPreamble +
+                    "dst is equal to or under src dir -failing");
+        }
+        return false;
+      }
+    }
+    //here the dest path is set up -so rename
     return renameRecursive(absoluteSrc, absoluteDst);
   }
-  
+
   private boolean renameRecursive(Path src, Path dst) throws IOException {
     INode srcINode = store.retrieveINode(src);
     store.storeINode(dst, srcINode);

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/s3native/NativeS3FileSystem.java

@@ -582,35 +582,58 @@ public class NativeS3FileSystem extends FileSystem {
   public boolean rename(Path src, Path dst) throws IOException {
 
     String srcKey = pathToKey(makeAbsolute(src));
+    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
 
     if (srcKey.length() == 0) {
       // Cannot rename root of file system
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(debugPreamble +
+                  "returning false as cannot rename the root of a filesystem");
+      }
       return false;
     }
 
-    final String debugPreamble = "Renaming '" + src + "' to '" + dst + "' - ";
+    //get status of source
-
+    boolean srcIsFile;
+    try {
+      srcIsFile = getFileStatus(src).isFile();
+    } catch (FileNotFoundException e) {
+      //bail out fast if the source does not exist
+      if (LOG.isDebugEnabled()) {
+        LOG.debug(debugPreamble + "returning false as src does not exist");
+      }
+      return false;
+    }
     // Figure out the final destination
-    String dstKey;
+    String dstKey = pathToKey(makeAbsolute(dst));
+
     try {
       boolean dstIsFile = getFileStatus(dst).isFile();
       if (dstIsFile) {
+        //destination is a file.
+        //you can't copy a file or a directory onto an existing file
+        //except for the special case of dest==src, which is a no-op
         if(LOG.isDebugEnabled()) {
           LOG.debug(debugPreamble +
-              "returning false as dst is an already existing file");
+              "returning without rename as dst is an already existing file");
         }
-        return false;
+        //exit, returning true iff the rename is onto self
+        return srcKey.equals(dstKey);
       } else {
+        //destination exists and is a directory
         if(LOG.isDebugEnabled()) {
           LOG.debug(debugPreamble + "using dst as output directory");
         }
+        //destination goes under the dst path, with the name of the
+        //source entry
         dstKey = pathToKey(makeAbsolute(new Path(dst, src.getName())));
       }
     } catch (FileNotFoundException e) {
+      //destination does not exist => the source file or directory
+      //is copied over with the name of the destination
       if(LOG.isDebugEnabled()) {
         LOG.debug(debugPreamble + "using dst as output destination");
       }
-      dstKey = pathToKey(makeAbsolute(dst));
       try {
         if (getFileStatus(dst.getParent()).isFile()) {
           if(LOG.isDebugEnabled()) {
@@ -628,16 +651,17 @@ public class NativeS3FileSystem extends FileSystem {
       }
     }
 
-    boolean srcIsFile;
+    //rename to self behavior follows Posix rules and is different
-    try {
+    //for directories and files -the return code is driven by src type
-      srcIsFile = getFileStatus(src).isFile();
+    if (srcKey.equals(dstKey)) {
-    } catch (FileNotFoundException e) {
+      //fully resolved destination key matches source: fail
-      if(LOG.isDebugEnabled()) {
+      if (LOG.isDebugEnabled()) {
-        LOG.debug(debugPreamble + "returning false as src does not exist");
+        LOG.debug(debugPreamble + "renamingToSelf; returning true");
       }
-      return false;
+      return true;
     }
     if (srcIsFile) {
+      //source is a file; COPY then DELETE
       if(LOG.isDebugEnabled()) {
         LOG.debug(debugPreamble +
             "src is file, so doing copy then delete in S3");
@@ -645,9 +669,19 @@ public class NativeS3FileSystem extends FileSystem {
       store.copy(srcKey, dstKey);
       store.delete(srcKey);
     } else {
+      //src is a directory
       if(LOG.isDebugEnabled()) {
         LOG.debug(debugPreamble + "src is directory, so copying contents");
       }
+      //Verify dest is not a child of the parent
+      if (dstKey.startsWith(srcKey + "/")) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(
+            debugPreamble + "cannot rename a directory to a subdirectory of self");
+        }
+        return false;
+      }
+      //create the subdir under the destination
       store.storeEmptyFile(dstKey + FOLDER_SUFFIX);
 
       List<String> keysToDelete = new ArrayList<String>();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java

@@ -257,6 +257,7 @@ public class Client {
     private final ConnectionId remoteId;                // connection id
     private AuthMethod authMethod; // authentication method
     private Token<? extends TokenIdentifier> token;
+    private int serviceClass;
     private SaslRpcClient saslRpcClient;
     
     private Socket socket = null;                 // connected socket
@@ -279,7 +280,7 @@ public class Client {
     
     private final Object sendRpcRequestLock = new Object();
 
-    public Connection(ConnectionId remoteId) throws IOException {
+    public Connection(ConnectionId remoteId, int serviceClass) throws IOException {
       this.remoteId = remoteId;
       this.server = remoteId.getAddress();
       if (server.isUnresolved()) {
@@ -296,6 +297,7 @@ public class Client {
       this.tcpNoDelay = remoteId.getTcpNoDelay();
       this.doPing = remoteId.getDoPing();
       this.pingInterval = remoteId.getPingInterval();
+      this.serviceClass = serviceClass;
       if (LOG.isDebugEnabled()) {
         LOG.debug("The ping interval is " + this.pingInterval + " ms.");
       }
@@ -747,7 +749,9 @@ public class Client {
      * +----------------------------------+
      * |  "hrpc" 4 bytes                  |      
      * +----------------------------------+
-     * |  Version (1 bytes)               |      
+     * |  Version (1 byte)                |
+     * +----------------------------------+
+     * |  Service Class (1 byte)          |
      * +----------------------------------+
      * |  Authmethod (1 byte)             |      
      * +----------------------------------+
@@ -760,6 +764,7 @@ public class Client {
       // Write out the header, version and authentication method
       out.write(Server.HEADER.array());
       out.write(Server.CURRENT_VERSION);
+      out.write(serviceClass);
       authMethod.write(out);
       Server.IpcSerializationType.PROTOBUF.write(out);
       out.flush();
@@ -1179,19 +1184,33 @@ public class Client {
 
   
   /**
-   * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress, 
+   * Same as {@link #call(RPC.RpcKind, Writable, InetSocketAddress,
    * Class, UserGroupInformation, int, Configuration)}
    * except that rpcKind is writable.
    */
-  public Writable call(Writable param, InetSocketAddress addr, 
+  public Writable call(Writable param, InetSocketAddress addr,
       Class<?> protocol, UserGroupInformation ticket,
-      int rpcTimeout, Configuration conf)  
+      int rpcTimeout, Configuration conf)
       throws InterruptedException, IOException {
-        ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
+    ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
         ticket, rpcTimeout, conf);
     return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId);
   }
   
+  /**
+   * Same as {@link #call(Writable, InetSocketAddress,
+   * Class, UserGroupInformation, int, Configuration)}
+   * except that specifying serviceClass.
+   */
+  public Writable call(Writable param, InetSocketAddress addr,
+      Class<?> protocol, UserGroupInformation ticket,
+      int rpcTimeout, int serviceClass, Configuration conf)
+      throws InterruptedException, IOException {
+    ConnectionId remoteId = ConnectionId.getConnectionId(addr, protocol,
+        ticket, rpcTimeout, conf);
+    return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId, serviceClass);
+  }
+
   /**
    * Make a call, passing <code>param</code>, to the IPC server running at
    * <code>address</code> which is servicing the <code>protocol</code> protocol,
@@ -1218,10 +1237,10 @@ public class Client {
      return call(RPC.RpcKind.RPC_BUILTIN, param, remoteId);
   }
   
-  /** 
+  /**
    * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
    * <code>remoteId</code>, returning the rpc respond.
-   * 
+   *
    * @param rpcKind
    * @param rpcRequest -  contains serialized method and method parameters
    * @param remoteId - the target rpc server
@@ -1231,8 +1250,26 @@ public class Client {
    */
   public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
       ConnectionId remoteId) throws InterruptedException, IOException {
+    return call(rpcKind, rpcRequest, remoteId, RPC.RPC_SERVICE_CLASS_DEFAULT);
+  }
+
+  /** 
+   * Make a call, passing <code>rpcRequest</code>, to the IPC server defined by
+   * <code>remoteId</code>, returning the rpc respond.
+   * 
+   * @param rpcKind
+   * @param rpcRequest -  contains serialized method and method parameters
+   * @param remoteId - the target rpc server
+   * @param serviceClass - service class for RPC
+   * @returns the rpc response
+   * Throws exceptions if there are network problems or if the remote code 
+   * threw an exception.
+   */
+  public Writable call(RPC.RpcKind rpcKind, Writable rpcRequest,
+      ConnectionId remoteId, int serviceClass)
+      throws InterruptedException, IOException {
     Call call = new Call(rpcKind, rpcRequest);
-    Connection connection = getConnection(remoteId, call);
+    Connection connection = getConnection(remoteId, call, serviceClass);
     try {
       connection.sendRpcRequest(call);                 // send the rpc request
     } catch (RejectedExecutionException e) {
@@ -1289,7 +1326,7 @@ public class Client {
   /** Get a connection from the pool, or create a new one and add it to the
    * pool.  Connections to a given ConnectionId are reused. */
   private Connection getConnection(ConnectionId remoteId,
-                                   Call call)
+                                   Call call, int serviceClass)
                                    throws IOException, InterruptedException {
     if (!running.get()) {
       // the client is stopped
@@ -1304,7 +1341,7 @@ public class Client {
       synchronized (connections) {
         connection = connections.get(remoteId);
         if (connection == null) {
-          connection = new Connection(remoteId);
+          connection = new Connection(remoteId, serviceClass);
           connections.put(remoteId, connection);
         }
       }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java

@@ -77,12 +77,12 @@ import com.google.protobuf.BlockingService;
 @InterfaceAudience.LimitedPrivate(value = { "Common", "HDFS", "MapReduce", "Yarn" })
 @InterfaceStability.Evolving
 public class RPC {
+  final static int RPC_SERVICE_CLASS_DEFAULT = 0;
   public enum RpcKind {
     RPC_BUILTIN ((short) 1),         // Used for built in calls by tests
     RPC_WRITABLE ((short) 2),        // Use WritableRpcEngine 
     RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
     final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
-    private static final short FIRST_INDEX = RPC_BUILTIN.value;    
     public final short value; //TODO make it private
 
     RpcKind(short val) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java

@@ -438,6 +438,11 @@ public abstract class Server {
     return Arrays.asList(handlers);
   }
 
+  @VisibleForTesting
+  List<Connection> getConnections() {
+    return connectionList;
+  }
+
   /**
    * Refresh the service authorization ACL for the service handled by this server.
    */
@@ -1104,6 +1109,7 @@ public abstract class Server {
     private ByteBuffer connectionHeaderBuf = null;
     private ByteBuffer unwrappedData;
     private ByteBuffer unwrappedDataLengthBuffer;
+    private int serviceClass;
     
     UserGroupInformation user = null;
     public UserGroupInformation attemptingUser = null; // user name before auth
@@ -1231,7 +1237,8 @@ public abstract class Server {
           rpcMetrics.incrAuthenticationFailures();
           String clientIP = this.toString();
           // attempting user could be null
-          AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser);
+          AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser +
+            " (" + e.getLocalizedMessage() + ")");
           throw e;
         }
         if (saslServer.isComplete() && replyToken == null) {
@@ -1314,14 +1321,17 @@ public abstract class Server {
         if (!connectionHeaderRead) {
           //Every connection is expected to send the header.
           if (connectionHeaderBuf == null) {
-            connectionHeaderBuf = ByteBuffer.allocate(3);
+            connectionHeaderBuf = ByteBuffer.allocate(4);
           }
           count = channelRead(channel, connectionHeaderBuf);
           if (count < 0 || connectionHeaderBuf.remaining() > 0) {
             return count;
           }
           int version = connectionHeaderBuf.get(0);
-          byte[] method = new byte[] {connectionHeaderBuf.get(1)};
+          // TODO we should add handler for service class later
+          this.setServiceClass(connectionHeaderBuf.get(1));
+
+          byte[] method = new byte[] {connectionHeaderBuf.get(2)};
           authMethod = AuthMethod.read(new DataInputStream(
               new ByteArrayInputStream(method)));
           dataLengthBuffer.flip();
@@ -1345,7 +1355,7 @@ public abstract class Server {
           }
           
           IpcSerializationType serializationType = IpcSerializationType
-              .fromByte(connectionHeaderBuf.get(2));
+              .fromByte(connectionHeaderBuf.get(3));
           if (serializationType != IpcSerializationType.PROTOBUF) {
             respondUnsupportedSerialization(serializationType);
             return -1;
@@ -1735,6 +1745,22 @@ public abstract class Server {
       return true;
     }
     
+    /**
+     * Get service class for connection
+     * @return the serviceClass
+     */
+    public int getServiceClass() {
+      return serviceClass;
+    }
+
+    /**
+     * Set service class for connection
+     * @param serviceClass the serviceClass to set
+     */
+    public void setServiceClass(int serviceClass) {
+      this.serviceClass = serviceClass;
+    }
+
     private synchronized void close() throws IOException {
       disposeSasl();
       data = null;

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java

@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.Hashtable;
 import java.util.List;
 
+import javax.naming.CommunicationException;
 import javax.naming.Context;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
@@ -166,6 +167,8 @@ public class LdapGroupsMapping
   private String groupMemberAttr;
   private String groupNameAttr;
 
+  public static int RECONNECT_RETRY_COUNT = 3;
+  
   /**
    * Returns list of groups for a user.
    * 
@@ -178,34 +181,63 @@ public class LdapGroupsMapping
    */
   @Override
   public synchronized List<String> getGroups(String user) throws IOException {
-    List<String> groups = new ArrayList<String>();
+    List<String> emptyResults = new ArrayList<String>();
-
+    /*
+     * Normal garbage collection takes care of removing Context instances when they are no longer in use. 
+     * Connections used by Context instances being garbage collected will be closed automatically.
+     * So in case connection is closed and gets CommunicationException, retry some times with new new DirContext/connection. 
+     */
     try {
-      DirContext ctx = getDirContext();
+      return doGetGroups(user);
-
+    } catch (CommunicationException e) {
-      // Search for the user. We'll only ever need to look at the first result
+      LOG.warn("Connection is closed, will try to reconnect");
-      NamingEnumeration<SearchResult> results = ctx.search(baseDN,
-                                                           userSearchFilter,
-                                                           new Object[]{user},
-                                                           SEARCH_CONTROLS);
-      if (results.hasMoreElements()) {
-        SearchResult result = results.nextElement();
-        String userDn = result.getNameInNamespace();
-
-        NamingEnumeration<SearchResult> groupResults =
-          ctx.search(baseDN,
-                     "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))",
-                     new Object[]{userDn},
-                     SEARCH_CONTROLS);
-        while (groupResults.hasMoreElements()) {
-          SearchResult groupResult = groupResults.nextElement();
-          Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
-          groups.add(groupName.get().toString());
-        }
-      }
     } catch (NamingException e) {
       LOG.warn("Exception trying to get groups for user " + user, e);
-      return new ArrayList<String>();
+      return emptyResults;
+    }
+
+    int retryCount = 0;
+    while (retryCount ++ < RECONNECT_RETRY_COUNT) {
+      //reset ctx so that new DirContext can be created with new connection
+      this.ctx = null;
+      
+      try {
+        return doGetGroups(user);
+      } catch (CommunicationException e) {
+        LOG.warn("Connection being closed, reconnecting failed, retryCount = " + retryCount);
+      } catch (NamingException e) {
+        LOG.warn("Exception trying to get groups for user " + user, e);
+        return emptyResults;
+      }
+    }
+    
+    return emptyResults;
+  }
+  
+  List<String> doGetGroups(String user) throws NamingException {
+    List<String> groups = new ArrayList<String>();
+
+    DirContext ctx = getDirContext();
+
+    // Search for the user. We'll only ever need to look at the first result
+    NamingEnumeration<SearchResult> results = ctx.search(baseDN,
+        userSearchFilter,
+        new Object[]{user},
+        SEARCH_CONTROLS);
+    if (results.hasMoreElements()) {
+      SearchResult result = results.nextElement();
+      String userDn = result.getNameInNamespace();
+
+      NamingEnumeration<SearchResult> groupResults =
+          ctx.search(baseDN,
+              "(&" + groupSearchFilter + "(" + groupMemberAttr + "={0}))",
+              new Object[]{userDn},
+              SEARCH_CONTROLS);
+      while (groupResults.hasMoreElements()) {
+        SearchResult groupResult = groupResults.nextElement();
+        Attribute groupName = groupResult.getAttributes().get(groupNameAttr);
+        groups.add(groupName.get().toString());
+      }
     }
 
     return groups;
@@ -236,7 +268,7 @@ public class LdapGroupsMapping
 
     return ctx;
   }
-
+  
   /**
    * Caches groups, no need to do that for this provider
    */

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/FileBasedKeyStoresFactory.java

@@ -164,7 +164,9 @@ public class FileBasedKeyStoresFactory implements KeyStoresFactory {
     } else {
       keystore.load(null, null);
     }
-    KeyManagerFactory keyMgrFactory = KeyManagerFactory.getInstance("SunX509");
+    KeyManagerFactory keyMgrFactory = KeyManagerFactory
+        .getInstance(SSLFactory.SSLCERTIFICATE);
+      
     keyMgrFactory.init(keystore, (keystorePassword != null) ?
                                  keystorePassword.toCharArray() : null);
     keyManagers = keyMgrFactory.getKeyManagers();

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/ReloadingX509TrustManager.java

@@ -169,8 +169,8 @@ public final class ReloadingX509TrustManager
       in.close();
     }
 
-    TrustManagerFactory trustManagerFactory =
+    TrustManagerFactory trustManagerFactory = 
-      TrustManagerFactory.getInstance("SunX509");
+      TrustManagerFactory.getInstance(SSLFactory.SSLCERTIFICATE);
     trustManagerFactory.init(ks);
     TrustManager[] trustManagers = trustManagerFactory.getTrustManagers();
     for (TrustManager trustManager1 : trustManagers) {

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/SSLFactory.java

@@ -58,6 +58,9 @@ public class SSLFactory implements ConnectionConfigurator {
     "hadoop.ssl.client.conf";
   public static final String SSL_SERVER_CONF_KEY =
     "hadoop.ssl.server.conf";
+  private static final boolean IBMJAVA = 
+      System.getProperty("java.vendor").contains("IBM");
+  public static final String SSLCERTIFICATE = IBMJAVA?"ibmX509":"SunX509"; 
 
   public static final boolean DEFAULT_SSL_REQUIRE_CLIENT_CERT = false;
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.fs;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.EnumSet;
 
 import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -1164,6 +1166,40 @@ public abstract class FileContextMainOperationsBaseTest  {
       Assert.assertEquals(fc.getFileStatus(file), fc.getFileLinkStatus(file));
     }
   }
+
+  /**
+   * Test that URIs with a scheme, no authority, and absolute path component
+   * resolve with the authority of the default FS.
+   */
+  @Test(timeout=30000)
+  public void testAbsolutePathSchemeNoAuthority() throws IOException,
+      URISyntaxException {
+    Path file = getTestRootPath(fc, "test/file");
+    createFile(file);
+    URI uri = file.toUri();
+    URI noAuthorityUri = new URI(uri.getScheme(), null, uri.getPath(),
+        uri.getQuery(), uri.getFragment());
+    Path noAuthority = new Path(noAuthorityUri);
+    Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
+  }
+
+  /**
+   * Test that URIs with a scheme, no authority, and relative path component
+   * resolve with the authority of the default FS.
+   */
+  @Test(timeout=30000)
+  public void testRelativePathSchemeNoAuthority() throws IOException,
+      URISyntaxException {
+    Path workDir = new Path(getAbsoluteTestRootPath(fc), new Path("test"));
+    fc.setWorkingDirectory(workDir);
+    Path file = new Path(workDir, "file");
+    createFile(file);
+    URI uri = file.toUri();
+    URI noAuthorityUri = new URI(uri.getScheme() + ":file");
+    System.out.println(noAuthorityUri);
+    Path noAuthority = new Path(noAuthorityUri);
+    Assert.assertEquals(fc.getFileStatus(file), fc.getFileStatus(noAuthority));
+  }
   
   protected void createFile(Path path) throws IOException {
     FSDataOutputStream out = fc.create(path, EnumSet.of(CREATE),

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.Locale;
 
 import junit.framework.TestCase;
 
@@ -51,7 +52,13 @@ public abstract class FileSystemContractBaseTest extends TestCase {
 
   @Override
   protected void tearDown() throws Exception {
-    fs.delete(path("/test"), true);
+    try {
+      if (fs != null) {
+        fs.delete(path("/test"), true);
+      }
+    } catch (IOException e) {
+      LOG.error("Error deleting /test: " + e, e);
+    }
   }
   
   protected int getBlockSize() {
@@ -62,10 +69,23 @@ public abstract class FileSystemContractBaseTest extends TestCase {
     return "/user/" + System.getProperty("user.name");
   }
 
+  /**
+   * Override this if the filesystem does not support rename
+   * @return true if the FS supports rename -and rename related tests
+   * should be run
+   */
   protected boolean renameSupported() {
     return true;
   }
 
+  /**
+   * Override this if the filesystem is not case sensitive
+   * @return true if the case detection/preservation tests should run
+   */
+  protected boolean filesystemIsCaseSensitive() {
+    return true;
+  }
+
   public void testFsStatus() throws Exception {
     FsStatus fsStatus = fs.getStatus();
     assertNotNull(fsStatus);
@@ -109,6 +129,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
     assertTrue(fs.mkdirs(testDir));
 
     assertTrue(fs.exists(testDir));
+    assertTrue("Should be a directory", fs.isDirectory(testDir));
     assertFalse(fs.isFile(testDir));
 
     Path parentDir = testDir.getParent();
@@ -118,17 +139,17 @@ public abstract class FileSystemContractBaseTest extends TestCase {
     Path grandparentDir = parentDir.getParent();
     assertTrue(fs.exists(grandparentDir));
     assertFalse(fs.isFile(grandparentDir));
-    
+
   }
-  
+
   public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
     Path testDir = path("/test/hadoop");
     assertFalse(fs.exists(testDir));
     assertTrue(fs.mkdirs(testDir));
     assertTrue(fs.exists(testDir));
-    
+
     createFile(path("/test/hadoop/file"));
-    
+
     Path testSubDir = path("/test/hadoop/file/subdir");
     try {
       fs.mkdirs(testSubDir);
@@ -137,7 +158,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
       // expected
     }
     assertFalse(fs.exists(testSubDir));
-    
+
     Path testDeepSubDir = path("/test/hadoop/file/deep/sub/dir");
     try {
       fs.mkdirs(testDeepSubDir);
@@ -146,7 +167,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
       // expected
     }
     assertFalse(fs.exists(testDeepSubDir));
-    
+
   }
 
   public void testMkdirsWithUmask() throws Exception {
@@ -177,7 +198,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
       // expected
     }
   }
-  
+
   public void testListStatusThrowsExceptionForNonExistentFile() throws Exception {
     try {
       fs.listStatus(path("/test/hadoop/file"));
@@ -186,7 +207,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
       // expected
     }
   }
-  
+
   public void testListStatus() throws Exception {
     Path[] testDirs = { path("/test/hadoop/a"),
                         path("/test/hadoop/b"),
@@ -210,7 +231,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
     paths = fs.listStatus(path("/test/hadoop/a"));
     assertEquals(0, paths.length);
   }
-  
+
   public void testWriteReadAndDeleteEmptyFile() throws Exception {
     writeReadAndDelete(0);
   }
@@ -222,7 +243,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
   public void testWriteReadAndDeleteOneBlock() throws Exception {
     writeReadAndDelete(getBlockSize());
   }
-  
+
   public void testWriteReadAndDeleteOneAndAHalfBlocks() throws Exception {
     writeReadAndDelete(getBlockSize() + (getBlockSize() / 2));
   }
@@ -365,8 +386,7 @@ public abstract class FileSystemContractBaseTest extends TestCase {
     Path dst = path("/test/new/newdir");
     fs.mkdirs(dst);
     rename(src, dst, true, false, true);
-    assertTrue("Destination changed",
+    assertIsFile(path("/test/new/newdir/file"));
-        fs.exists(path("/test/new/newdir/file")));
   }
   
   public void testRenameDirectoryMoveToNonExistentDirectory() 
@@ -466,9 +486,9 @@ public abstract class FileSystemContractBaseTest extends TestCase {
   
   private void rename(Path src, Path dst, boolean renameSucceeded,
       boolean srcExists, boolean dstExists) throws IOException {
-    assertEquals("Rename result", renameSucceeded, fs.rename(src, dst));
+    assertEquals("mv " + src + " " + dst,renameSucceeded, fs.rename(src, dst));
-    assertEquals("Source exists", srcExists, fs.exists(src));
+    assertEquals("Source exists: " + src, srcExists, fs.exists(src));
-    assertEquals("Destination exists", dstExists, fs.exists(dst));
+    assertEquals("Destination exists" + dst, dstExists, fs.exists(dst));
   }
 
   /**
@@ -494,6 +514,253 @@ public abstract class FileSystemContractBaseTest extends TestCase {
     writeAndRead(path, filedata2, blockSize * 2, true, false);
   }
 
+  /**
+   * Assert that a filesystem is case sensitive.
+   * This is done by creating a mixed-case filename and asserting that
+   * its lower case version is not there.
+   * @throws Exception
+   */
+  public void testFilesystemIsCaseSensitive() throws Exception {
+    if (!filesystemIsCaseSensitive()) {
+      LOG.info("Skipping test");
+      return;
+    }
+    String mixedCaseFilename = "/test/UPPER.TXT";
+    Path upper = path(mixedCaseFilename);
+    Path lower = path(mixedCaseFilename.toLowerCase(Locale.ENGLISH));
+    assertFalse("File exists" + upper, fs.exists(upper));
+    assertFalse("File exists" + lower, fs.exists(lower));
+    FSDataOutputStream out = fs.create(upper);
+    out.writeUTF("UPPER");
+    out.close();
+    FileStatus upperStatus = fs.getFileStatus(upper);
+    assertTrue("File does not exist" + upper, fs.exists(upper));
+    //verify the lower-case version of the filename doesn't exist
+    assertFalse("File exists" + lower, fs.exists(lower));
+    //now overwrite the lower case version of the filename with a
+    //new version.
+    out = fs.create(lower);
+    out.writeUTF("l");
+    out.close();
+    assertTrue("File does not exist" + lower, fs.exists(lower));
+    //verify the length of the upper file hasn't changed
+    FileStatus newStatus = fs.getFileStatus(upper);
+    assertEquals("Expected status:" + upperStatus
+                 + " actual status " + newStatus,
+                 upperStatus.getLen(),
+                 newStatus.getLen()); }
+
+  /**
+   * Asserts that a zero byte file has a status of file and not
+   * directory or symlink
+   * @throws Exception on failures
+   */
+  public void testZeroByteFilesAreFiles() throws Exception {
+    Path src = path("/test/testZeroByteFilesAreFiles");
+    //create a zero byte file
+    FSDataOutputStream out = fs.create(src);
+    out.close();
+    assertIsFile(src);
+  }
+
+  /**
+   * Asserts that a zero byte file has a status of file and not
+   * directory or symlink
+   * @throws Exception on failures
+   */
+  public void testMultiByteFilesAreFiles() throws Exception {
+    Path src = path("/test/testMultiByteFilesAreFiles");
+    FSDataOutputStream out = fs.create(src);
+    out.writeUTF("testMultiByteFilesAreFiles");
+    out.close();
+    assertIsFile(src);
+  }
+
+  /**
+   * Assert that root directory renames are not allowed
+   * @throws Exception on failures
+   */
+  public void testRootDirAlwaysExists() throws Exception {
+    //this will throw an exception if the path is not found
+    fs.getFileStatus(path("/"));
+    //this catches overrides of the base exists() method that don't
+    //use getFileStatus() as an existence probe
+    assertTrue("FileSystem.exists() fails for root", fs.exists(path("/")));
+  }
+
+  /**
+   * Assert that root directory renames are not allowed
+   * @throws Exception on failures
+   */
+  public void testRenameRootDirForbidden() throws Exception {
+    if (!renameSupported()) return;
+
+    rename(path("/"),
+           path("/test/newRootDir"),
+           false, true, false);
+  }
+
+  /**
+   * Assert that renaming a parent directory to be a child
+   * of itself is forbidden
+   * @throws Exception on failures
+   */
+  public void testRenameChildDirForbidden() throws Exception {
+    if (!renameSupported()) return;
+    LOG.info("testRenameChildDirForbidden");
+    Path parentdir = path("/test/parentdir");
+    fs.mkdirs(parentdir);
+    Path childFile = new Path(parentdir, "childfile");
+    createFile(childFile);
+    //verify one level down
+    Path childdir = new Path(parentdir, "childdir");
+    rename(parentdir, childdir, false, true, false);
+    //now another level
+    fs.mkdirs(childdir);
+    Path childchilddir = new Path(childdir, "childdir");
+    rename(parentdir, childchilddir, false, true, false);
+  }
+
+  /**
+   * This a sanity check to make sure that any filesystem's handling of
+   * renames doesn't cause any regressions
+   */
+  public void testRenameToDirWithSamePrefixAllowed() throws Throwable {
+    if (!renameSupported()) return;
+    Path parentdir = path("test/parentdir");
+    fs.mkdirs(parentdir);
+    Path dest = path("test/parentdirdest");
+    rename(parentdir, dest, true, false, true);
+  }
+
+  /**
+   * trying to rename a directory onto itself should fail,
+   * preserving everything underneath.
+   */
+  public void testRenameDirToSelf() throws Throwable {
+    if (!renameSupported()) {
+      return;
+    }
+    Path parentdir = path("test/parentdir");
+    fs.mkdirs(parentdir);
+    Path child = new Path(parentdir, "child");
+    createFile(child);
+
+    rename(parentdir, parentdir, false, true, true);
+    //verify the child is still there
+    assertIsFile(child);
+  }
+
+  /**
+   * trying to rename a directory onto its parent dir will build
+   * a destination path of its original name, which should then fail.
+   * The source path and the destination path should still exist afterwards
+   */
+  public void testMoveDirUnderParent() throws Throwable {
+    if (!renameSupported()) {
+      return;
+    }
+    Path testdir = path("test/dir");
+    fs.mkdirs(testdir);
+    Path parent = testdir.getParent();
+    //the outcome here is ambiguous, so is not checked
+    fs.rename(testdir, parent);
+    assertEquals("Source exists: " + testdir, true, fs.exists(testdir));
+    assertEquals("Destination exists" + parent, true, fs.exists(parent));
+  }
+
+  /**
+   * trying to rename a file onto itself should succeed (it's a no-op)
+   *
+   */
+  public void testRenameFileToSelf() throws Throwable {
+    if (!renameSupported()) return;
+    Path filepath = path("test/file");
+    createFile(filepath);
+    //HDFS expects rename src, src -> true
+    rename(filepath, filepath, true, true, true);
+    //verify the file is still there
+    assertIsFile(filepath);
+  }
+
+  /**
+   * trying to move a file into it's parent dir should succeed
+   * again: no-op
+   */
+  public void testMoveFileUnderParent() throws Throwable {
+    if (!renameSupported()) return;
+    Path filepath = path("test/file");
+    createFile(filepath);
+    //HDFS expects rename src, src -> true
+    rename(filepath, filepath, true, true, true);
+    //verify the file is still there
+    assertIsFile(filepath);
+  }
+
+  public void testLSRootDir() throws Throwable {
+    Path dir = path("/");
+    Path child = path("/test");
+    createFile(child);
+    assertListFilesFinds(dir, child);
+  }
+
+  public void testListStatusRootDir() throws Throwable {
+    Path dir = path("/");
+    Path child  = path("/test");
+    createFile(child);
+    assertListStatusFinds(dir, child);
+  }
+
+  private void assertListFilesFinds(Path dir, Path subdir) throws IOException {
+    RemoteIterator<LocatedFileStatus> iterator =
+      fs.listFiles(dir, true);
+    boolean found = false;
+    StringBuilder builder = new StringBuilder();
+    while (iterator.hasNext()) {
+      LocatedFileStatus next =  iterator.next();
+      builder.append(next.toString()).append('\n');
+      if (next.getPath().equals(subdir)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + subdir
+               + " not found in directory " + dir + ":" + builder,
+               found);
+  }
+
+  private void assertListStatusFinds(Path dir, Path subdir) throws IOException {
+    FileStatus[] stats = fs.listStatus(dir);
+    boolean found = false;
+    StringBuilder builder = new StringBuilder();
+    for (FileStatus stat : stats) {
+      builder.append(stat.toString()).append('\n');
+      if (stat.getPath().equals(subdir)) {
+        found = true;
+      }
+    }
+    assertTrue("Path " + subdir
+               + " not found in directory " + dir + ":" + builder,
+               found);
+  }
+
+
+  /**
+   * Assert that a file exists and whose {@link FileStatus} entry
+   * declares that this is a file and not a symlink or directory.
+   * @param filename name of the file
+   * @throws IOException IO problems during file operations
+   */
+  private void assertIsFile(Path filename) throws IOException {
+    assertTrue("Does not exist: " + filename, fs.exists(filename));
+    FileStatus status = fs.getFileStatus(filename);
+    String fileInfo = filename + "  " + status;
+    assertTrue("Not a file " + fileInfo, status.isFile());
+    assertFalse("File claims to be a symlink " + fileInfo,
+                status.isSymlink());
+    assertFalse("File claims to be a directory " + fileInfo,
+                status.isDirectory());
+  }
+
   /**
    *
    * Write a file and read it in, validating the result. Optional flags control

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java

@@ -26,6 +26,7 @@ import java.net.URI;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.NetUtilsTestResolver;
 import org.apache.hadoop.util.Progressable;
 import org.junit.BeforeClass;
@@ -312,6 +313,11 @@ public class TestFileSystemCanonicalization {
       return defaultPort;
     }
     
+    @Override
+    protected URI canonicalizeUri(URI uri) {
+      return NetUtils.getCanonicalUri(uri, getDefaultPort());
+    }
+
     @Override
     public FSDataInputStream open(Path f, int bufferSize) throws IOException {
       throw new IOException("not supposed to be here");

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.fs;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.util.Shell;
 
 import static org.apache.hadoop.fs.FileSystemTestHelper.*;
@@ -266,9 +267,14 @@ public class TestLocalFileSystem {
     LocalFileSystem fs = FileSystem.getLocal(conf);
     Path path = new Path(TEST_ROOT_DIR, "test-file");
     writeFile(fs, path, 1);
-    BufferedFSInputStream bis = new BufferedFSInputStream(
+    BufferedFSInputStream bis = null;
-        new RawLocalFileSystem().new LocalFSFileInputStream(path), 1024);
+    try {
-    assertNotNull(bis.getFileDescriptor());
+      bis = new BufferedFSInputStream(new RawLocalFileSystem()
+        .new LocalFSFileInputStream(path), 1024);
+      assertNotNull(bis.getFileDescriptor());
+    } finally {
+      IOUtils.cleanup(null, bis);
+    }
   }
 
   @Test

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3/InMemoryFileSystemStore.java

@@ -50,6 +50,7 @@ class InMemoryFileSystemStore implements FileSystemStore {
   @Override
   public void initialize(URI uri, Configuration conf) {
     this.conf = conf;
+    inodes.put(new Path("/"), INode.DIRECTORY_INODE);
   }
   
   @Override

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/s3native/NativeS3FileSystemContractBaseTest.java

@@ -51,7 +51,7 @@ public abstract class NativeS3FileSystemContractBaseTest
   
   public void testListStatusForRoot() throws Exception {
     FileStatus[] paths = fs.listStatus(path("/"));
-    assertEquals(0, paths.length);
+    assertEquals("Root directory is not empty; ", 0, paths.length);
     
     Path testDir = path("/test");
     assertTrue(fs.mkdirs(testDir));

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java

@@ -25,6 +25,7 @@ import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.ipc.Server.Connection;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.net.ConnectTimeoutException;
 import org.apache.hadoop.net.NetUtils;
@@ -520,11 +521,53 @@ public class TestIPC {
     }
   }
   
+  /**
+   * Check service class byte in IPC header is correct on wire.
+   */
+  @Test(timeout=60000)
+  public void testIpcWithServiceClass() throws Exception {
+    // start server
+    Server server = new TestServer(5, false);
+    InetSocketAddress addr = NetUtils.getConnectAddress(server);
+    server.start();
+
+    // start client
+    Client.setConnectTimeout(conf, 10000);
+
+    callAndVerify(server, addr, 0, true);
+    // Service Class is low to -128 as byte on wire.
+    // -128 shouldn't be casted on wire but -129 should.
+    callAndVerify(server, addr, -128, true);
+    callAndVerify(server, addr, -129, false);
+
+    // Service Class is up to 127.
+    // 127 shouldn't be casted on wire but 128 should.
+    callAndVerify(server, addr, 127, true);
+    callAndVerify(server, addr, 128, false);
+
+    server.stop();
+  }
+
+  /**
+   * Make a call from a client and verify if header info is changed in server side
+   */
+  private void callAndVerify(Server server, InetSocketAddress addr,
+      int serviceClass, boolean noChanged) throws Exception{
+    Client client = new Client(LongWritable.class, conf);
+
+    client.call(new LongWritable(RANDOM.nextLong()),
+        addr, null, null, MIN_SLEEP_TIME, serviceClass, conf);
+    Connection connection = server.getConnections().get(0);
+    int serviceClass2 = connection.getServiceClass();
+    assertFalse(noChanged ^ serviceClass == serviceClass2);
+    client.stop();
+  }
+
   /**
    * Check that file descriptors aren't leaked by starting
    * and stopping IPC servers.
    */
-  @Test
+  @Test(timeout=60000)
   public void testSocketLeak() throws Exception {
     Assume.assumeTrue(FD_DIR.exists()); // only run on Linux
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java

@@ -26,6 +26,7 @@ import java.io.Writer;
 import java.util.Arrays;
 import java.util.List;
 
+import javax.naming.CommunicationException;
 import javax.naming.NamingEnumeration;
 import javax.naming.NamingException;
 import javax.naming.directory.Attribute;
@@ -46,21 +47,15 @@ public class TestLdapGroupsMapping {
   private DirContext mockContext;
   
   private LdapGroupsMapping mappingSpy = spy(new LdapGroupsMapping());
+  private NamingEnumeration mockUserNamingEnum = mock(NamingEnumeration.class);
+  private NamingEnumeration mockGroupNamingEnum = mock(NamingEnumeration.class);
+  private String[] testGroups = new String[] {"group1", "group2"};
   
   @Before
   public void setupMocks() throws NamingException {
     mockContext = mock(DirContext.class);
     doReturn(mockContext).when(mappingSpy).getDirContext();
-    
+            
-    NamingEnumeration mockUserNamingEnum = mock(NamingEnumeration.class);
-    NamingEnumeration mockGroupNamingEnum = mock(NamingEnumeration.class);
-    
-    // The search functionality of the mock context is reused, so we will
-    // return the user NamingEnumeration first, and then the group
-    when(mockContext.search(anyString(), anyString(), any(Object[].class),
-        any(SearchControls.class)))
-        .thenReturn(mockUserNamingEnum, mockGroupNamingEnum);
-    
     SearchResult mockUserResult = mock(SearchResult.class);
     // We only ever call hasMoreElements once for the user NamingEnum, so 
     // we can just have one return value
@@ -76,23 +71,57 @@ public class TestLdapGroupsMapping {
     
     // Define the attribute for the name of the first group
     Attribute group1Attr = new BasicAttribute("cn");
-    group1Attr.add("group1");
+    group1Attr.add(testGroups[0]);
     Attributes group1Attrs = new BasicAttributes();
     group1Attrs.put(group1Attr);
     
     // Define the attribute for the name of the second group
     Attribute group2Attr = new BasicAttribute("cn");
-    group2Attr.add("group2");
+    group2Attr.add(testGroups[1]);
     Attributes group2Attrs = new BasicAttributes();
     group2Attrs.put(group2Attr);
     
     // This search result gets reused, so return group1, then group2
     when(mockGroupResult.getAttributes()).thenReturn(group1Attrs, group2Attrs);
-    
   }
   
   @Test
   public void testGetGroups() throws IOException, NamingException {
+    // The search functionality of the mock context is reused, so we will
+    // return the user NamingEnumeration first, and then the group
+    when(mockContext.search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class)))
+        .thenReturn(mockUserNamingEnum, mockGroupNamingEnum);
+    
+    doTestGetGroups(Arrays.asList(testGroups), 2);
+  }
+
+  @Test
+  public void testGetGroupsWithConnectionClosed() throws IOException, NamingException {
+    // The case mocks connection is closed/gc-ed, so the first search call throws CommunicationException,
+    // then after reconnected return the user NamingEnumeration first, and then the group
+    when(mockContext.search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class)))
+        .thenThrow(new CommunicationException("Connection is closed"))
+        .thenReturn(mockUserNamingEnum, mockGroupNamingEnum);
+    
+    // Although connection is down but after reconnected it still should retrieve the result groups
+    doTestGetGroups(Arrays.asList(testGroups), 1 + 2); // 1 is the first failure call 
+  }
+
+  @Test
+  public void testGetGroupsWithLdapDown() throws IOException, NamingException {
+    // This mocks the case where Ldap server is down, and always throws CommunicationException 
+    when(mockContext.search(anyString(), anyString(), any(Object[].class),
+        any(SearchControls.class)))
+        .thenThrow(new CommunicationException("Connection is closed"));
+    
+    // Ldap server is down, no groups should be retrieved
+    doTestGetGroups(Arrays.asList(new String[] {}), 
+        1 + LdapGroupsMapping.RECONNECT_RETRY_COUNT); // 1 is the first normal call
+  }
+  
+  private void doTestGetGroups(List<String> expectedGroups, int searchTimes) throws IOException, NamingException {  
     Configuration conf = new Configuration();
     // Set this, so we don't throw an exception
     conf.set(LdapGroupsMapping.LDAP_URL_KEY, "ldap://test");
@@ -102,10 +131,10 @@ public class TestLdapGroupsMapping {
     // regardless of input
     List<String> groups = mappingSpy.getGroups("some_user");
     
-    Assert.assertEquals(Arrays.asList("group1", "group2"), groups);
+    Assert.assertEquals(expectedGroups, groups);
     
     // We should have searched for a user, and then two groups
-    verify(mockContext, times(2)).search(anyString(),
+    verify(mockContext, times(searchTimes)).search(anyString(),
                                          anyString(),
                                          any(Object[].class),
                                          any(SearchControls.class));

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.test;
 import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
+import java.lang.reflect.InvocationTargetException;
 import java.util.Arrays;
 import java.util.Random;
 import java.util.Set;
@@ -266,15 +267,29 @@ public abstract class GenericTestUtils {
    */
   public static class DelegateAnswer implements Answer<Object> { 
     private final Object delegate;
+    private final Log log;
     
     public DelegateAnswer(Object delegate) {
+      this(null, delegate);
+    }
+    
+    public DelegateAnswer(Log log, Object delegate) {
+      this.log = log;
       this.delegate = delegate;
     }
 
     @Override
     public Object answer(InvocationOnMock invocation) throws Throwable {
-      return invocation.getMethod().invoke(
+      try {
-          delegate, invocation.getArguments());
+        if (log != null) {
+          log.info("Call to " + invocation + " on " + delegate,
+              new Exception("TRACE"));
+        }
+        return invocation.getMethod().invoke(
+            delegate, invocation.getArguments());
+      } catch (InvocationTargetException ite) {
+        throw ite.getCause();
+      }
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -181,6 +181,9 @@ Trunk (Unreleased)
     HDFS-4346. Add SequentialNumber as a base class for INodeId and
     GenerationStamp.  (szetszwo)
 
+    HDFS-4633 TestDFSClientExcludedNodes fails sporadically if excluded nodes
+    cache expires too quickly  (Chris Nauroth via Sanjay)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -369,6 +372,12 @@ Release 2.0.5-beta - UNRELEASED
     HDFS-4246. The exclude node list should be more forgiving, for each output
     stream. (harsh via atm)
 
+    HDFS-4635. Move BlockManager#computeCapacity to LightWeightGSet. (suresh)
+
+    HDFS-4621. Additional logging to help diagnose slow QJM syncs. (todd)
+
+    HDFS-4618. Default transaction interval for checkpoints is too low. (todd)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -448,6 +457,9 @@ Release 2.0.5-beta - UNRELEASED
     HDFS-4584. Skip TestNNWithQJM.testNewNamenodeTakesOverWriter() on Windows.
     (Arpit Agarwal via szetszwo)
 
+    HDFS-4598. Fix the default value of ConcatSourcesParam and the WebHDFS doc.
+    (szetszwo)
+
 Release 2.0.4-alpha - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -2462,6 +2474,9 @@ Release 0.23.7 - UNRELEASED
     HDFS-3367. WebHDFS doesn't use the logged in user when opening
     connections (daryn)
 
+    HDFS-4581. checkDiskError should not be called on network errors (Rohit
+    Kochar via kihwal)
+
 Release 0.23.6 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

@@ -122,7 +122,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String  DFS_NAMENODE_CHECKPOINT_PERIOD_KEY = "dfs.namenode.checkpoint.period";
   public static final long    DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT = 3600;
   public static final String  DFS_NAMENODE_CHECKPOINT_TXNS_KEY = "dfs.namenode.checkpoint.txns";
-  public static final long    DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT = 40000;
+  public static final long    DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT = 1000000;
   public static final String  DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY = "dfs.namenode.checkpoint.max-retries";
   public static final int     DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT = 3;
   public static final String  DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY = "dfs.namenode.heartbeat.recheck-interval";

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java

@@ -65,6 +65,7 @@ import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.Token;
@@ -315,13 +316,14 @@ public class DistributedFileSystem extends FileSystem {
   }
   
   /**
-   * Move blocks from srcs to trg
+   * Move blocks from srcs to trg and delete srcs afterwards.
-   * and delete srcs afterwards
+   * The file block sizes must be the same.
-   * RESTRICTION: all blocks should be the same size
+   * 
    * @param trg existing file to append to
    * @param psrcs list of files (same block size, same replication)
    * @throws IOException
    */
+  @Override
   public void concat(Path trg, Path [] psrcs) throws IOException {
     String [] srcs = new String [psrcs.length];
     for(int i=0; i<psrcs.length; i++) {
@@ -895,6 +897,17 @@ public class DistributedFileSystem extends FileSystem {
   public String getCanonicalServiceName() {
     return dfs.getCanonicalServiceName();
   }
+  
+  @Override
+  protected URI canonicalizeUri(URI uri) {
+    if (HAUtil.isLogicalUri(getConf(), uri)) {
+      // Don't try to DNS-resolve logical URIs, since the 'authority'
+      // portion isn't a proper hostname
+      return uri;
+    } else {
+      return NetUtils.getCanonicalUri(uri, getDefaultPort());
+    }
+  }
 
   /**
    * Utility function that returns if the NameNode is in safemode or not. In HA

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java

@@ -161,6 +161,11 @@ public class HftpFileSystem extends FileSystem
     // actual port in the uri
     return SecurityUtil.buildTokenService(nnSecureUri).toString();
   }
+  
+  @Override
+  protected URI canonicalizeUri(URI uri) {
+    return NetUtils.getCanonicalUri(uri, getDefaultPort());
+  }
 
   /**
    * Return the protocol scheme for the FileSystem.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java

@@ -133,6 +133,8 @@ public class IPCLoggerChannel implements AsyncLogger {
   private Stopwatch lastHeartbeatStopwatch = new Stopwatch();
   
   private static final long HEARTBEAT_INTERVAL_MILLIS = 1000;
+
+  private static final long WARN_JOURNAL_MILLIS_THRESHOLD = 1000;
   
   static final Factory FACTORY = new AsyncLogger.Factory() {
     @Override
@@ -371,6 +373,12 @@ public class IPCLoggerChannel implements AsyncLogger {
                 now - submitNanos, TimeUnit.NANOSECONDS);
             metrics.addWriteEndToEndLatency(endToEndTime);
             metrics.addWriteRpcLatency(rpcTime);
+            if (rpcTime / 1000 > WARN_JOURNAL_MILLIS_THRESHOLD) {
+              QuorumJournalManager.LOG.warn(
+                  "Took " + (rpcTime / 1000) + "ms to send a batch of " +
+                  numTxns + " edits (" + data.length + " bytes) to " +
+                  "remote journal " + IPCLoggerChannel.this);
+            }
           }
           synchronized (IPCLoggerChannel.this) {
             highestAckedTxId = firstTxnId + numTxns - 1;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumCall.java

@@ -24,6 +24,7 @@ import java.util.concurrent.TimeoutException;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.util.Time;
 
+import com.google.common.base.Joiner;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 import com.google.common.util.concurrent.FutureCallback;
@@ -120,6 +121,15 @@ class QuorumCall<KEY, RESULT> {
         String msg = String.format(
             "Waited %s ms (timeout=%s ms) for a response for %s",
             waited, millis, operationName);
+        if (!successes.isEmpty()) {
+          msg += ". Succeeded so far: [" + Joiner.on(",").join(successes.keySet()) + "]";
+        }
+        if (!exceptions.isEmpty()) {
+          msg += ". Exceptions so far: [" + getExceptionMapString() + "]";
+        }
+        if (successes.isEmpty() && exceptions.isEmpty()) {
+          msg += ". No responses yet.";
+        }
         if (waited > millis * WAIT_PROGRESS_WARN_THRESHOLD) {
           QuorumJournalManager.LOG.warn(msg);
         } else {
@@ -227,4 +237,22 @@ class QuorumCall<KEY, RESULT> {
     }
     return sb.toString();
   }
+
+  /**
+   * Return a string suitable for displaying to the user, containing
+   * any exceptions that have been received so far.
+   */
+  private String getExceptionMapString() {
+    StringBuilder sb = new StringBuilder();
+    boolean first = true;
+    for (Map.Entry<KEY, Throwable> e : exceptions.entrySet()) {
+      if (!first) {
+        sb.append(", ");
+      }
+      first = false;
+      sb.append(e.getKey()).append(": ")
+        .append(e.getValue().getLocalizedMessage());
+    }
+    return sb.toString();
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java

@@ -128,6 +128,10 @@ class Journal implements Closeable {
 
   private final JournalMetrics metrics;
 
+  /**
+   * Time threshold for sync calls, beyond which a warning should be logged to the console.
+   */
+  private static final int WARN_SYNC_MILLIS_THRESHOLD = 1000;
 
   Journal(File logDir, String journalId,
       StorageErrorReporter errorReporter) throws IOException {
@@ -370,6 +374,10 @@ class Journal implements Closeable {
     sw.stop();
     
     metrics.addSync(sw.elapsedTime(TimeUnit.MICROSECONDS));
+    if (sw.elapsedTime(TimeUnit.MILLISECONDS) > WARN_SYNC_MILLIS_THRESHOLD) {
+      LOG.warn("Sync of transaction range " + firstTxnId + "-" + lastTxnId +
+               " took " + sw.elapsedTime(TimeUnit.MILLISECONDS) + "ms");
+    }
 
     if (isLagging) {
       // This batch of edits has already been committed on a quorum of other

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -235,6 +235,7 @@ public class BlockManager {
     heartbeatManager = datanodeManager.getHeartbeatManager();
     invalidateBlocks = new InvalidateBlocks(datanodeManager);
 
+    // Compute the map capacity by allocating 2% of total memory
     blocksMap = new BlocksMap(DEFAULT_MAP_LOAD_FACTOR);
     blockplacement = BlockPlacementPolicy.getInstance(
         conf, stats, datanodeManager.getNetworkTopology());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java

@@ -60,38 +60,11 @@ class BlocksMap {
   private GSet<Block, BlockInfo> blocks;
 
   BlocksMap(final float loadFactor) {
-    this.capacity = computeCapacity();
+    // Use 2% of total memory to size the GSet capacity
+    this.capacity = LightWeightGSet.computeCapacity(2.0, "BlocksMap");
     this.blocks = new LightWeightGSet<Block, BlockInfo>(capacity);
   }
 
-  /**
-   * Let t = 2% of max memory.
-   * Let e = round(log_2 t).
-   * Then, we choose capacity = 2^e/(size of reference),
-   * unless it is outside the close interval [1, 2^30].
-   */
-  private static int computeCapacity() {
-    //VM detection
-    //See http://java.sun.com/docs/hotspot/HotSpotFAQ.html#64bit_detection
-    final String vmBit = System.getProperty("sun.arch.data.model");
-
-    //2% of max memory
-    final double twoPC = Runtime.getRuntime().maxMemory()/50.0;
-
-    //compute capacity
-    final int e1 = (int)(Math.log(twoPC)/Math.log(2.0) + 0.5);
-    final int e2 = e1 - ("32".equals(vmBit)? 2: 3);
-    final int exponent = e2 < 0? 0: e2 > 30? 30: e2;
-    final int c = 1 << exponent;
-
-    if (LightWeightGSet.LOG.isDebugEnabled()) {
-      LightWeightGSet.LOG.debug("VM type       = " + vmBit + "-bit");
-      LightWeightGSet.LOG.debug("2% max memory = " + twoPC/(1 << 20) + " MB");
-      LightWeightGSet.LOG.debug("capacity      = 2^" + exponent
-          + " = " + c + " entries");
-    }
-    return c;
-  }
 
   void close() {
     // Empty blocks once GSet#clear is implemented (HDFS-3940)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

@@ -60,8 +60,11 @@ import java.io.PrintStream;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
 import java.net.Socket;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.net.UnknownHostException;
+import java.nio.channels.ClosedByInterruptException;
 import java.nio.channels.ServerSocketChannel;
 import java.nio.channels.SocketChannel;
 import java.security.PrivilegedExceptionAction;
@@ -1172,7 +1175,13 @@ public class DataNode extends Configured
   protected void checkDiskError(Exception e ) throws IOException {
     
     LOG.warn("checkDiskError: exception: ", e);
-    
+    if (e instanceof SocketException || e instanceof SocketTimeoutException
+    	  || e instanceof ClosedByInterruptException 
+    	  || e.getMessage().startsWith("Broken pipe")) {
+      LOG.info("Not checking disk as checkDiskError was called on a network" +
+      		" related exception");	
+      return;
+    }
     if (e.getMessage() != null &&
         e.getMessage().startsWith("No space left on device")) {
       throw new DiskOutOfSpaceException("No space left on device");
@@ -1484,8 +1493,12 @@ public class DataNode extends Configured
       } catch (IOException ie) {
         LOG.warn(bpReg + ":Failed to transfer " + b + " to " +
             targets[0] + " got ", ie);
-        // check if there are any disk problem
+          // check if there are any disk problem
-        checkDiskError();
+        try{
+          checkDiskError(ie);
+        } catch(IOException e) {
+            LOG.warn("DataNode.checkDiskError failed in run() with: ", e);
+        }
         
       } finally {
         xmitsInProgress.getAndDecrement();

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/LightWeightGSet.java

@@ -24,8 +24,11 @@ import java.util.Iterator;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.HadoopIllegalArgumentException;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * A low memory footprint {@link GSet} implementation,
  * which uses an array for storing the elements
@@ -285,4 +288,54 @@ public class LightWeightGSet<K, E extends K> implements GSet<K, E> {
       throw new UnsupportedOperationException("Remove is not supported.");
     }
   }
+  
+  /**
+   * Let t = percentage of max memory.
+   * Let e = round(log_2 t).
+   * Then, we choose capacity = 2^e/(size of reference),
+   * unless it is outside the close interval [1, 2^30].
+   */
+  public static int computeCapacity(double percentage, String mapName) {
+    return computeCapacity(Runtime.getRuntime().maxMemory(), percentage,
+        mapName);
+  }
+  
+  @VisibleForTesting
+  static int computeCapacity(long maxMemory, double percentage,
+      String mapName) {
+    if (percentage > 100.0 || percentage < 0.0) {
+      throw new HadoopIllegalArgumentException("Percentage " + percentage
+          + " must be greater than or equal to 0 "
+          + " and less than or equal to 100");
+    }
+    if (maxMemory < 0) {
+      throw new HadoopIllegalArgumentException("Memory " + maxMemory
+          + " must be greater than or equal to 0");
+    }
+    if (percentage == 0.0 || maxMemory == 0) {
+      return 0;
+    }
+    //VM detection
+    //See http://java.sun.com/docs/hotspot/HotSpotFAQ.html#64bit_detection
+    final String vmBit = System.getProperty("sun.arch.data.model");
+
+    //Percentage of max memory
+    final double percentDivisor = 100.0/percentage;
+    final double percentMemory = maxMemory/percentDivisor;
+    
+    //compute capacity
+    final int e1 = (int)(Math.log(percentMemory)/Math.log(2.0) + 0.5);
+    final int e2 = e1 - ("32".equals(vmBit)? 2: 3);
+    final int exponent = e2 < 0? 0: e2 > 30? 30: e2;
+    final int c = 1 << exponent;
+
+    if (LightWeightGSet.LOG.isDebugEnabled()) {
+      LOG.debug("Computing capacity for map " + mapName);
+      LOG.debug("VM type       = " + vmBit + "-bit");
+      LOG.debug(percentage + "% max memory = "
+          + StringUtils.TraditionalBinaryPrefix.long2String(maxMemory, "B", 1));
+      LOG.debug("capacity      = 2^" + exponent + " = " + c + " entries");
+    }
+    return c;
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java

@@ -30,7 +30,6 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -64,7 +63,33 @@ import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
-import org.apache.hadoop.hdfs.web.resources.*;
+import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
+import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
+import org.apache.hadoop.hdfs.web.resources.BufferSizeParam;
+import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam;
+import org.apache.hadoop.hdfs.web.resources.CreateParentParam;
+import org.apache.hadoop.hdfs.web.resources.DelegationParam;
+import org.apache.hadoop.hdfs.web.resources.DeleteOpParam;
+import org.apache.hadoop.hdfs.web.resources.DestinationParam;
+import org.apache.hadoop.hdfs.web.resources.DoAsParam;
+import org.apache.hadoop.hdfs.web.resources.GetOpParam;
+import org.apache.hadoop.hdfs.web.resources.GroupParam;
+import org.apache.hadoop.hdfs.web.resources.HttpOpParam;
+import org.apache.hadoop.hdfs.web.resources.LengthParam;
+import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam;
+import org.apache.hadoop.hdfs.web.resources.OffsetParam;
+import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
+import org.apache.hadoop.hdfs.web.resources.OwnerParam;
+import org.apache.hadoop.hdfs.web.resources.Param;
+import org.apache.hadoop.hdfs.web.resources.PermissionParam;
+import org.apache.hadoop.hdfs.web.resources.PostOpParam;
+import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.hdfs.web.resources.RecursiveParam;
+import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam;
+import org.apache.hadoop.hdfs.web.resources.RenewerParam;
+import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
+import org.apache.hadoop.hdfs.web.resources.UserParam;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.retry.RetryPolicy;
 import org.apache.hadoop.io.retry.RetryUtils;
@@ -82,7 +107,6 @@ import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.security.token.TokenRenewer;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
 import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.StringUtils;
 import org.mortbay.util.ajax.JSON;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -214,6 +238,11 @@ public class WebHdfsFileSystem extends FileSystem
   public URI getUri() {
     return this.uri;
   }
+  
+  @Override
+  protected URI canonicalizeUri(URI uri) {
+    return NetUtils.getCanonicalUri(uri, getDefaultPort());
+  }
 
   /** @return the home directory. */
   public static String getHomeDirectoryString(final UserGroupInformation ugi) {
@@ -729,17 +758,10 @@ public class WebHdfsFileSystem extends FileSystem
   }
 
   @Override
-  public void concat(final Path trg, final Path [] psrcs) throws IOException {
+  public void concat(final Path trg, final Path [] srcs) throws IOException {
     statistics.incrementWriteOps(1);
     final HttpOpParam.Op op = PostOpParam.Op.CONCAT;
 
-    List<String> strPaths = new ArrayList<String>(psrcs.length);
-    for(Path psrc : psrcs) {
-       strPaths.add(psrc.toUri().getPath());
-    }
-
-    String srcs = StringUtils.join(",", strPaths);
-
     ConcatSourcesParam param = new ConcatSourcesParam(srcs);
     run(op, trg, param);
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java

@@ -18,15 +18,28 @@
 
 package org.apache.hadoop.hdfs.web.resources;
 
+import org.apache.hadoop.fs.Path;
+
 /** The concat source paths parameter. */
 public class ConcatSourcesParam extends StringParam {
   /** Parameter name. */
   public static final String NAME = "sources";
 
-  public static final String DEFAULT = NULL;
+  public static final String DEFAULT = "";
 
   private static final Domain DOMAIN = new Domain(NAME, null);
 
+  private static String paths2String(Path[] paths) {
+    if (paths == null || paths.length == 0) {
+      return "";
+    }
+    final StringBuilder b = new StringBuilder(paths[0].toUri().getPath());
+    for(int i = 1; i < paths.length; i++) {
+      b.append(',').append(paths[i].toUri().getPath());
+    }
+    return b.toString();
+  }
+
   /**
    * Constructor.
    * @param str a string representation of the parameter value.
@@ -35,6 +48,10 @@ public class ConcatSourcesParam extends StringParam {
     super(DOMAIN, str);
   }
 
+  public ConcatSourcesParam(Path[] paths) {
+    this(paths2String(paths));
+  }
+
   @Override
   public String getName() {
     return NAME;

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -635,7 +635,7 @@
 
 <property>
   <name>dfs.namenode.checkpoint.txns</name>
-  <value>40000</value>
+  <value>1000000</value>
   <description>The Secondary NameNode or CheckpointNode will create a checkpoint
   of the namespace every 'dfs.namenode.checkpoint.txns' transactions, regardless
   of whether 'dfs.namenode.checkpoint.period' has expired.

hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm

@@ -193,7 +193,7 @@ HDFS Users Guide
      * <<<dfs.namenode.checkpoint.period>>>, set to 1 hour by default, specifies
        the maximum delay between two consecutive checkpoints, and
 
-     * <<<dfs.namenode.checkpoint.txns>>>, set to 40000 default, defines the
+     * <<<dfs.namenode.checkpoint.txns>>>, set to 1 million by default, defines the
        number of uncheckpointed transactions on the NameNode which will
        force an urgent checkpoint, even if the checkpoint period has not
        been reached.
@@ -232,7 +232,7 @@ HDFS Users Guide
      * <<<dfs.namenode.checkpoint.period>>>, set to 1 hour by default, specifies
        the maximum delay between two consecutive checkpoints
 
-     * <<<dfs.namenode.checkpoint.txns>>>, set to 40000 default, defines the
+     * <<<dfs.namenode.checkpoint.txns>>>, set to 1 million by default, defines the
        number of uncheckpointed transactions on the NameNode which will
        force an urgent checkpoint, even if the checkpoint period has not
        been reached.

hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm

@@ -109,7 +109,7 @@ WebHDFS REST API
     * {{{Append to a File}<<<APPEND>>>}}
         (see  {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append)
 
-    * {{{Concat File(s)}<<<CONCAT>>>}}
+    * {{{Concatenate Files}<<<CONCAT>>>}}
         (see  {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat)
 
   * HTTP DELETE
@@ -307,7 +307,7 @@ Content-Length: 0
   * Submit a HTTP POST request.
 
 +---------------------------------
-curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CONCAT&sources=<SOURCES>"
+curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CONCAT&sources=<PATHS>"
 +---------------------------------
 
   The client receives a response with zero content length:
@@ -319,10 +319,6 @@ Content-Length: 0
 
   []
 
-  This REST API call is available as of Hadoop version 2.0.3.
-  Please note that <SOURCES> is a comma seperated list of absolute paths.
-  (Example: sources=/test/file1,/test/file2,/test/file3)
-
   See also:
   {{{Sources}<<<sources>>>}},
    {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat
@@ -1761,7 +1757,7 @@ var tokenProperties =
 *----------------+-------------------------------------------------------------------+
 || Name          | <<<sources>>> |
 *----------------+-------------------------------------------------------------------+
-|| Description   | The comma seperated absolute paths used for concatenation. |
+|| Description   | A list of source paths. |
 *----------------+-------------------------------------------------------------------+
 || Type          | String |
 *----------------+-------------------------------------------------------------------+
@@ -1769,12 +1765,9 @@ var tokenProperties =
 *----------------+-------------------------------------------------------------------+
 || Valid Values  | A list of comma seperated absolute FileSystem paths without scheme and authority. |
 *----------------+-------------------------------------------------------------------+
-|| Syntax        | See the note in {{Delegation}}. |
+|| Syntax        | Any string. |
 *----------------+-------------------------------------------------------------------+
 
-  <<Note>> that sources are absolute FileSystem paths.
-
-
   See also:
   {{{Concat File(s)}<<<CONCAT>>>}}
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientExcludedNodes.java

@@ -32,6 +32,8 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
 import org.apache.hadoop.util.ThreadUtil;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 
@@ -41,10 +43,25 @@ import org.junit.Test;
  */
 public class TestDFSClientExcludedNodes {
 
-  @Test(timeout=10000)
+  private MiniDFSCluster cluster;
+  private Configuration conf;
+
+  @Before
+  public void setUp() {
+    cluster = null;
+    conf = new HdfsConfiguration();
+  }
+
+  @After
+  public void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  @Test(timeout=60000)
   public void testExcludedNodes() throws IOException {
-    Configuration conf = new HdfsConfiguration();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
     FileSystem fs = cluster.getFileSystem();
     Path filePath = new Path("/testExcludedNodes");
 
@@ -67,17 +84,16 @@ public class TestDFSClientExcludedNodes {
     }
   }
 
-  @Test(timeout=10000)
+  @Test(timeout=60000)
   public void testExcludedNodesForgiveness() throws IOException {
-    Configuration conf = new HdfsConfiguration();
+    // Forgive nodes in under 2.5s for this test case.
-    // Forgive nodes in under 1s for this test case.
     conf.setLong(
         DFSConfigKeys.DFS_CLIENT_WRITE_EXCLUDE_NODES_CACHE_EXPIRY_INTERVAL,
-        1000);
+        2500);
     // We'll be using a 512 bytes block size just for tests
     // so making sure the checksum bytes too match it.
     conf.setInt("io.bytes.per.checksum", 512);
-    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
     List<DataNodeProperties> props = cluster.dataNodes;
     FileSystem fs = cluster.getFileSystem();
     Path filePath = new Path("/testForgivingExcludedNodes");
@@ -112,11 +128,11 @@ public class TestDFSClientExcludedNodes {
     Assert.assertEquals(true, cluster.restartDataNode(two, true));
     cluster.waitActive();
 
-    // Sleep for 2s, to let the excluded nodes be expired
+    // Sleep for 5s, to let the excluded nodes be expired
     // from the excludes list (i.e. forgiven after the configured wait period).
-    // [Sleeping just in case the restart of the DNs completed < 2s cause
+    // [Sleeping just in case the restart of the DNs completed < 5s cause
     // otherwise, we'll end up quickly excluding those again.]
-    ThreadUtil.sleepAtLeastIgnoreInterrupts(2000);
+    ThreadUtil.sleepAtLeastIgnoreInterrupts(5000);
 
     // Terminate the last good DN, to assert that there's no
     // single-DN-available scenario, caused by not forgiving the other

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientFailover.java

@@ -26,8 +26,11 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketAddress;
+import java.lang.reflect.Field;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.List;
 
 import javax.net.SocketFactory;
 
@@ -35,6 +38,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.server.namenode.NameNode;
@@ -48,10 +52,13 @@ import org.apache.hadoop.util.StringUtils;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.Description;
 import org.junit.After;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mockito;
 
+import sun.net.spi.nameservice.NameService;
+
 public class TestDFSClientFailover {
   
   private static final Log LOG = LogFactory.getLog(TestDFSClientFailover.class);
@@ -201,4 +208,74 @@ public class TestDFSClientFailover {
           "Could not find any configured addresses for URI " + uri));
     }
   }
+
+  /**
+   * Spy on the Java DNS infrastructure.
+   * This likely only works on Sun-derived JDKs, but uses JUnit's
+   * Assume functionality so that any tests using it are skipped on
+   * incompatible JDKs.
+   */
+  private NameService spyOnNameService() {
+    try {
+      Field f = InetAddress.class.getDeclaredField("nameServices");
+      f.setAccessible(true);
+      Assume.assumeNotNull(f);
+      @SuppressWarnings("unchecked")
+      List<NameService> nsList = (List<NameService>) f.get(null);
+
+      NameService ns = nsList.get(0);
+      Log log = LogFactory.getLog("NameServiceSpy");
+      
+      ns = Mockito.mock(NameService.class,
+          new GenericTestUtils.DelegateAnswer(log, ns));
+      nsList.set(0, ns);
+      return ns;
+    } catch (Throwable t) {
+      LOG.info("Unable to spy on DNS. Skipping test.", t);
+      // In case the JDK we're testing on doesn't work like Sun's, just
+      // skip the test.
+      Assume.assumeNoException(t);
+      throw new RuntimeException(t);
+    }
+  }
+  
+  /**
+   * Test that the client doesn't ever try to DNS-resolve the logical URI.
+   * Regression test for HADOOP-9150.
+   */
+  @Test
+  public void testDoesntDnsResolveLogicalURI() throws Exception {
+    NameService spyNS = spyOnNameService();
+    
+    FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
+    String logicalHost = fs.getUri().getHost();
+    Path qualifiedRoot = fs.makeQualified(new Path("/"));
+    
+    // Make a few calls against the filesystem.
+    fs.getCanonicalServiceName();
+    fs.listStatus(qualifiedRoot);
+    
+    // Ensure that the logical hostname was never resolved.
+    Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost));
+  }
+  
+  /**
+   * Same test as above, but for FileContext.
+   */
+  @Test
+  public void testFileContextDoesntDnsResolveLogicalURI() throws Exception {
+    NameService spyNS = spyOnNameService();
+    FileSystem fs = HATestUtil.configureFailoverFs(cluster, conf);
+    String logicalHost = fs.getUri().getHost();
+    Configuration haClientConf = fs.getConf();
+    
+    FileContext fc = FileContext.getFileContext(haClientConf);
+    Path root = new Path("/");
+    fc.listStatus(root);
+    fc.listStatus(fc.makeQualified(root));
+    fc.getDefaultFileSystem().getCanonicalServiceName();
+
+    // Ensure that the logical hostname was never resolved.
+    Mockito.verify(spyNS, Mockito.never()).lookupAllHostAddr(Mockito.eq(logicalHost));
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestGSet.java

@@ -21,6 +21,7 @@ import java.util.ConcurrentModificationException;
 import java.util.Iterator;
 import java.util.Random;
 
+import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.util.Time;
 import org.junit.Assert;
 import org.junit.Test;
@@ -452,4 +453,81 @@ public class TestGSet {
       next = e;
     }
   }
+  
+  /** 
+   * Test for {@link LightWeightGSet#computeCapacity(double, String)}
+   * with invalid percent less than 0.
+   */
+  @Test(expected=HadoopIllegalArgumentException.class)
+  public void testComputeCapacityNegativePercent() {
+    LightWeightGSet.computeCapacity(1024, -1.0, "testMap");
+  }
+  
+  /** 
+   * Test for {@link LightWeightGSet#computeCapacity(double, String)}
+   * with invalid percent greater than 100.
+   */
+  @Test(expected=HadoopIllegalArgumentException.class)
+  public void testComputeCapacityInvalidPercent() {
+    LightWeightGSet.computeCapacity(1024, 101.0, "testMap");
+  }
+  
+  /** 
+   * Test for {@link LightWeightGSet#computeCapacity(double, String)}
+   * with invalid negative max memory
+   */
+  @Test(expected=HadoopIllegalArgumentException.class)
+  public void testComputeCapacityInvalidMemory() {
+    LightWeightGSet.computeCapacity(-1, 50.0, "testMap");
+  }
+  
+  private static boolean isPowerOfTwo(int num) {
+    return num == 0 || (num > 0 && Integer.bitCount(num) == 1);
+  }
+  
+  /** Return capacity as percentage of total memory */
+  private static int getPercent(long total, int capacity) {
+    // Reference size in bytes
+    double referenceSize = 
+        System.getProperty("sun.arch.data.model").equals("32") ? 4.0 : 8.0;
+    return (int)(((capacity * referenceSize)/total) * 100.0);
+  }
+  
+  /** Return capacity as percentage of total memory */
+  private static void testCapacity(long maxMemory, double percent) {
+    int capacity = LightWeightGSet.computeCapacity(maxMemory, percent, "map");
+    LightWeightGSet.LOG.info("Validating - total memory " + maxMemory + " percent "
+        + percent + " returned capacity " + capacity);
+    // Returned capacity is zero or power of two
+    Assert.assertTrue(isPowerOfTwo(capacity));
+
+    // Ensure the capacity returned is the nearest to the asked perecentage
+    int capacityPercent = getPercent(maxMemory, capacity);
+    if (capacityPercent == percent) {
+      return;
+    } else if (capacityPercent > percent) {
+      Assert.assertTrue(getPercent(maxMemory, capacity * 2) > percent);
+    } else {
+      Assert.assertTrue(getPercent(maxMemory, capacity / 2) < percent);
+    }
+  }
+  
+  /** 
+   * Test for {@link LightWeightGSet#computeCapacity(double, String)}
+   */
+  @Test
+  public void testComputeCapacity() {
+    // Tests for boundary conditions where percent or memory are zero
+    testCapacity(0, 0.0);
+    testCapacity(100, 0.0);
+    testCapacity(0, 100.0);
+    
+    // Compute capacity for some 100 random max memory and percentage
+    Random r = new Random();
+    for (int i = 0; i < 100; i++) {
+      long maxMemory = r.nextInt(Integer.MAX_VALUE);
+      double percent = r.nextInt(101);
+      testCapacity(maxMemory, percent);
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java

@@ -17,18 +17,22 @@
  */
 package org.apache.hadoop.hdfs.web.resources;
 
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.util.Arrays;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.util.StringUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-
 public class TestParam {
   public static final Log LOG = LogFactory.getLog(TestParam.class);
 
@@ -265,4 +269,20 @@ public class TestParam {
     UserParam userParam = new UserParam("a$");
     assertNotNull(userParam.getValue());
   }
+  
+  @Test
+  public void testConcatSourcesParam() {
+    final String[] strings = {"/", "/foo", "/bar"};
+    for(int n = 0; n < strings.length; n++) {
+      final String[] sub = new String[n]; 
+      final Path[] paths = new Path[n];
+      for(int i = 0; i < paths.length; i++) {
+        paths[i] = new Path(sub[i] = strings[i]);
+      }
+
+      final String expected = StringUtils.join(",", Arrays.asList(sub));
+      final ConcatSourcesParam computed = new ConcatSourcesParam(paths);
+      Assert.assertEquals(expected, computed.getValue());
+    }
+  }
 }

hadoop-mapreduce-project/CHANGES.txt

@@ -14,10 +14,6 @@ Trunk (Unreleased)
     MAPREDUCE-4887. Add RehashPartitioner, to smooth distributions
     with poor implementations of Object#hashCode().  (Radim Kolar via cutting)
 
-    HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows
-    Azure environments. (See breakdown of tasks below for subtasks and
-    contributors)
-
   IMPROVEMENTS
 
     MAPREDUCE-3787. [Gridmix] Optimize job monitoring and STRESS mode for
@@ -75,9 +71,6 @@ Trunk (Unreleased)
     MAPREDUCE-4735. Make arguments in TestDFSIO case insensitive.
     (Brandon Li via suresh)
 
-    MAPREDUCE-5014. Extend Distcp to accept a custom CopyListing. 
-    (Srikanth Sundarrajan via amareshwari)
-
   BUG FIXES
 
     MAPREDUCE-4272. SortedRanges.Range#compareTo is not spec compliant.
@@ -161,32 +154,11 @@ Trunk (Unreleased)
 
     MAPREDUCE-5012. Typo in javadoc for IdentityMapper class. (Adam Monsen
     via suresh)
-    
-    MAPREDUCE-5078. TestMRAppMaster fails on Windows due to mismatched path
-    separators. (Chris Nauroth via sseth)
 
-  BREAKDOWN OF HADOOP-8562 SUBTASKS
+    MAPREDUCE-5006. Fix failing streaming tests due to MAPREDUCE-4994.
+    (Sandy Ryza via tomwhite)
 
-    MAPREDUCE-4739. Some MapReduce tests fail to find winutils.
+Release 2.0.4-beta - UNRELEASED
-    (Chris Nauroth via suresh)
-
-    MAPREDUCE-4780. MapReduce distribution build fails on Windows.
-    (Chris Nauroth via suresh)
-
-    MAPREDUCE-4790. MapReduce build script would be more readable using abspath.
-    (Chris Nauroth via suresh)
-
-    MAPREDUCE-4869. Fix TestMapReduceChildJVM. (Chris Nauroth via acmurthy)
-
-    MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy)
-
-    MAPREDUCE-4983. Fixed various platform specific assumptions in various tests,
-    so that they can pass on Windows too. (Chris Nauroth via vinodkv)
-
-    HADOOP-9372. Fix bad timeout annotations on tests.
-    (Arpit Agarwal via suresh)
-
-Release 2.0.5-beta - UNRELEASED
 
   INCOMPATIBLE CHANGES
 
@@ -197,12 +169,6 @@ Release 2.0.5-beta - UNRELEASED
     MAPREDUCE-5033. mapred shell script should respect usage flags
     (--help -help -h). (Andrew Wang via atm)
 
-    MAPREDUCE-4892. Modify CombineFileInputFormat to not skew input slits'
-    allocation on small clusters. (Bikas Saha via vinodkv)
-
-    MAPREDUCE-4990. Construct debug strings conditionally in 
-    ShuffleHandler.Shuffle#sendMapOutput(). (kkambatl via tucu)
-
   OPTIMIZATIONS
 
   BUG FIXES
@@ -229,42 +195,6 @@ Release 2.0.5-beta - UNRELEASED
     MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER.
     (Sandy Ryza via tomwhite)
 
-    MAPREDUCE-4693. History server should include counters for failed tasks.
-    (Xuan Gong via sseth)
-
-    MAPREDUCE-4896. mapred queue -info spits out ugly exception when queue does 
-    not exist. (sandyr via tucu)
-
-    MAPREDUCE-3685. Fix bugs in MergeManager to ensure compression codec is
-    appropriately used and that on-disk segments are correctly sorted on
-    file-size. (Anty Rao and Ravi Prakash via acmurthy) 
-
-    MAPREDUCE-4571. TestHsWebServicesJobs fails on jdk7. (tgraves via tucu)
-
-    MAPREDUCE-4716. TestHsWebServicesJobsQuery.testJobsQueryStateInvalid 
-    fails with jdk7. (tgraves via tucu)
-
-    MAPREDUCE-5075. DistCp leaks input file handles since ThrottledInputStream
-    does not close the wrapped InputStream.  (Chris Nauroth via szetszwo)
-
-    MAPREDUCE-3872. Fix an event handling races in ContainerLauncherImpl.
-    (Robert Kanter via sseth)
-
-    MAPREDUCE-5083. MiniMRCluster should use a random component when creating an
-    actual cluster (Siddharth Seth via hitesh)
-
-Release 2.0.4-alpha - UNRELEASED
-
-  INCOMPATIBLE CHANGES
-
-  NEW FEATURES
-
-  IMPROVEMENTS
-
-  OPTIMIZATIONS
-
-  BUG FIXES
-
 Release 2.0.3-alpha - 2013-02-06 
 
   INCOMPATIBLE CHANGES
@@ -795,12 +725,6 @@ Release 0.23.7 - UNRELEASED
     MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi
     Prakash via jlowe)
 
-    MAPREDUCE-5027. Shuffle does not limit number of outstanding connections
-    (Robert Parker via jeagles)
-
-    MAPREDUCE-4972. Coverage fixing for org.apache.hadoop.mapreduce.jobhistory
-    (Aleksey Gorshkov via bobby)
-
   OPTIMIZATIONS
 
     MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the
@@ -820,27 +744,6 @@ Release 0.23.7 - UNRELEASED
     MAPREDUCE-5009. Killing the Task Attempt slated for commit does not clear
     the value from the Task commitAttempt member (Robert Parker via jeagles)
 
-    MAPREDUCE-4871. AM uses mapreduce.jobtracker.split.metainfo.maxsize but
-    mapred-default has mapreduce.job.split.metainfo.maxsize (Jason Lowe via
-    jeagles)
-
-    MAPREDUCE-4794. DefaultSpeculator generates error messages on normal
-    shutdown (Jason Lowe via jeagles)
-
-    MAPREDUCE-5043. Fetch failure processing can cause AM event queue to
-    backup and eventually OOM (Jason Lowe via bobby)
-
-    MAPREDUCE-5023. History Server Web Services missing Job Counters (Ravi
-    Prakash via tgraves)
-
-    MAPREDUCE-5060. Fetch failures that time out only count against the first
-    map task (Robert Joseph Evans via jlowe)
-
-    MAPREDUCE-5042. Reducer unable to fetch for a map task that was recovered
-    (Jason Lowe via bobby)
-
-    MAPREDUCE-5053. java.lang.InternalError from decompression codec cause
-    reducer to fail (Robert Parker via jeagles)
 
 Release 0.23.6 - UNRELEASED
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java

@@ -161,6 +161,7 @@ public class MRAppMaster extends CompositeService {
   private final int nmPort;
   private final int nmHttpPort;
   protected final MRAppMetrics metrics;
+  private final int maxAppAttempts;
   private Map<TaskId, TaskInfo> completedTasksFromPreviousRun;
   private List<AMInfo> amInfos;
   private AppContext context;
@@ -194,14 +195,14 @@ public class MRAppMaster extends CompositeService {
 
   public MRAppMaster(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
-      long appSubmitTime) {
+      long appSubmitTime, int maxAppAttempts) {
     this(applicationAttemptId, containerId, nmHost, nmPort, nmHttpPort,
-        new SystemClock(), appSubmitTime);
+        new SystemClock(), appSubmitTime, maxAppAttempts);
   }
 
   public MRAppMaster(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String nmHost, int nmPort, int nmHttpPort,
-      Clock clock, long appSubmitTime) {
+      Clock clock, long appSubmitTime, int maxAppAttempts) {
     super(MRAppMaster.class.getName());
     this.clock = clock;
     this.startTime = clock.getTime();
@@ -212,6 +213,7 @@ public class MRAppMaster extends CompositeService {
     this.nmPort = nmPort;
     this.nmHttpPort = nmHttpPort;
     this.metrics = MRAppMetrics.create();
+    this.maxAppAttempts = maxAppAttempts;
     LOG.info("Created MRAppMaster for application " + applicationAttemptId);
   }
 
@@ -220,18 +222,13 @@ public class MRAppMaster extends CompositeService {
     conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
 
     downloadTokensAndSetupUGI(conf);
-    
+
-    //TODO this is a hack, we really need the RM to inform us when we
+    isLastAMRetry = appAttemptID.getAttemptId() >= maxAppAttempts;
-    // are the last one.  This would allow us to configure retries on
+    LOG.info("The specific max attempts: " + maxAppAttempts +
-    // a per application basis.
+        " for application: " + appAttemptID.getApplicationId().getId() +
-    int numAMRetries = conf.getInt(YarnConfiguration.RM_AM_MAX_RETRIES, 
+        ". Attempt num: " + appAttemptID.getAttemptId() +
-        YarnConfiguration.DEFAULT_RM_AM_MAX_RETRIES);
-    isLastAMRetry = appAttemptID.getAttemptId() >= numAMRetries;
-    LOG.info("AM Retries: " + numAMRetries + 
-        " attempt num: " + appAttemptID.getAttemptId() +
         " is last retry: " + isLastAMRetry);
-    
+
-    
     context = new RunningAppContext(conf);
 
     // Job name is the same as the app name util we support DAG of jobs
@@ -266,6 +263,9 @@ public class MRAppMaster extends CompositeService {
       boolean commitFailure = fs.exists(endCommitFailureFile);
       if(!stagingExists) {
         isLastAMRetry = true;
+        LOG.info("Attempt num: " + appAttemptID.getAttemptId() +
+            " is last retry: " + isLastAMRetry +
+            " because the staging dir doesn't exist.");
         errorHappenedShutDown = true;
         forcedState = JobStateInternal.ERROR;
         shutDownMessage = "Staging dir does not exist " + stagingDir;
@@ -275,6 +275,9 @@ public class MRAppMaster extends CompositeService {
         // what result we will use to notify, and how we will unregister
         errorHappenedShutDown = true;
         isLastAMRetry = true;
+        LOG.info("Attempt num: " + appAttemptID.getAttemptId() +
+            " is last retry: " + isLastAMRetry +
+            " because a commit was started.");
         copyHistory = true;
         if (commitSuccess) {
           shutDownMessage = "We crashed after successfully committing. Recovering.";
@@ -777,6 +780,10 @@ public class MRAppMaster extends CompositeService {
     return taskAttemptListener;
   }
 
+  public Boolean isLastAMRetry() {
+    return isLastAMRetry;
+  }
+
   /**
    * By the time life-cycle of this router starts, job-init would have already
    * happened.
@@ -1206,6 +1213,8 @@ public class MRAppMaster extends CompositeService {
           System.getenv(ApplicationConstants.NM_HTTP_PORT_ENV);
       String appSubmitTimeStr =
           System.getenv(ApplicationConstants.APP_SUBMIT_TIME_ENV);
+      String maxAppAttempts =
+          System.getenv(ApplicationConstants.MAX_APP_ATTEMPTS_ENV);
       
       validateInputParam(containerIdStr,
           ApplicationConstants.AM_CONTAINER_ID_ENV);
@@ -1215,6 +1224,8 @@ public class MRAppMaster extends CompositeService {
           ApplicationConstants.NM_HTTP_PORT_ENV);
       validateInputParam(appSubmitTimeStr,
           ApplicationConstants.APP_SUBMIT_TIME_ENV);
+      validateInputParam(maxAppAttempts,
+          ApplicationConstants.MAX_APP_ATTEMPTS_ENV);
 
       ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
       ApplicationAttemptId applicationAttemptId =
@@ -1224,7 +1235,8 @@ public class MRAppMaster extends CompositeService {
       MRAppMaster appMaster =
           new MRAppMaster(applicationAttemptId, containerId, nodeHostString,
               Integer.parseInt(nodePortString),
-              Integer.parseInt(nodeHttpPortString), appSubmitTime);
+              Integer.parseInt(nodeHttpPortString), appSubmitTime,
+              Integer.parseInt(maxAppAttempts));
       ShutdownHookManager.get().addShutdownHook(
         new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
       YarnConfiguration conf = new YarnConfiguration(new JobConf());

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java

@@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.util.BuilderUtils;
 public abstract class RMContainerRequestor extends RMCommunicator {
   
   private static final Log LOG = LogFactory.getLog(RMContainerRequestor.class);
-  static final String ANY = "*";
 
   private int lastResponseID;
   private Resource availableResources;
@@ -278,7 +277,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
     }
 
     // Off-switch
-    addResourceRequest(req.priority, ANY, req.capability); 
+    addResourceRequest(req.priority, ResourceRequest.ANY, req.capability);
   }
 
   protected void decContainerReq(ContainerRequest req) {
@@ -291,7 +290,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
       decResourceRequest(req.priority, rack, req.capability);
     }
    
-    decResourceRequest(req.priority, ANY, req.capability);
+    decResourceRequest(req.priority, ResourceRequest.ANY, req.capability);
   }
 
   private void addResourceRequest(Priority priority, String resourceName,

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java

@@ -32,6 +32,8 @@ import static org.mockito.Mockito.when;
 import java.io.IOException;
 import java.util.Arrays;
 
+import junit.framework.Assert;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.TypeConverter;
@@ -78,7 +80,7 @@ public class TestTaskAttemptListenerImpl {
     }
   }
   
-  @Test
+  @Test  (timeout=5000)
   public void testGetTask() throws IOException {
     AppContext appCtx = mock(AppContext.class);
     JobTokenSecretManager secret = mock(JobTokenSecretManager.class); 
@@ -136,9 +138,30 @@ public class TestTaskAttemptListenerImpl {
     assertTrue(result.shouldDie);
 
     listener.stop();
+
+    // test JVMID
+    JVMId jvmid = JVMId.forName("jvm_001_002_m_004");
+    assertNotNull(jvmid);
+    try {
+      JVMId.forName("jvm_001_002_m_004_006");
+      Assert.fail();
+    } catch (IllegalArgumentException e) {
+      assertEquals(e.getMessage(),
+          "TaskId string : jvm_001_002_m_004_006 is not properly formed");
+    }
+
   }
 
-  @Test
+  @Test (timeout=5000)
+  public void testJVMId() {
+
+    JVMId jvmid = new JVMId("test", 1, true, 2);
+    JVMId jvmid1 = JVMId.forName("jvm_test_0001_m_000002");
+    // test compare methot should be the same
+    assertEquals(0, jvmid.compareTo(jvmid1));
+  }
+
+  @Test (timeout=10000)
   public void testGetMapCompletionEvents() throws IOException {
     TaskAttemptCompletionEvent[] empty = {};
     TaskAttemptCompletionEvent[] taskEvents = {
@@ -205,7 +228,7 @@ public class TestTaskAttemptListenerImpl {
     return tce;
   }
 
-  @Test
+  @Test (timeout=1000)
   public void testCommitWindow() throws IOException {
     SystemClock clock = new SystemClock();
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java

@@ -192,7 +192,7 @@ public class MRApp extends MRAppMaster {
       int maps, int reduces, boolean autoComplete, String testName,
       boolean cleanOnStart, int startCount, Clock clock) {
     super(appAttemptId, amContainerId, NM_HOST, NM_PORT, NM_HTTP_PORT, clock, System
-        .currentTimeMillis());
+        .currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
     this.testWorkDir = new File("target", testName);
     testAbsPath = new Path(testWorkDir.getAbsolutePath());
     LOG.info("PathUsed: " + testAbsPath);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java

@@ -230,7 +230,7 @@ public class MRAppBenchmark {
                 List<ResourceRequest> askList = request.getAskList();
                 List<Container> containers = new ArrayList<Container>();
                 for (ResourceRequest req : askList) {
-                  if (req.getHostName() != "*") {
+                  if (!ResourceRequest.isAnyLocation(req.getHostName())) {
                     continue;
                   }
                   int numContainers = req.getNumContainers();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java

@@ -30,11 +30,15 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.TypeConverter;
 import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
+import org.apache.hadoop.mapreduce.v2.app.commit.CommitterEvent;
+import org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler;
 import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
 import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
+import org.apache.hadoop.mapreduce.v2.app.rm.RMHeartbeatHandler;
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -42,6 +46,7 @@ import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -80,7 +85,7 @@ public class TestMRAppMaster {
     ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
     MRAppMasterTest appMaster =
         new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
-            System.currentTimeMillis());
+            System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
     YarnConfiguration conf = new YarnConfiguration();
     conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
     MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
@@ -109,7 +114,8 @@ public class TestMRAppMaster {
     ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
     MRAppMaster appMaster =
         new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
-            System.currentTimeMillis(), false);
+            System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS,
+            false, false);
     boolean caught = false;
     try {
       MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
@@ -144,7 +150,8 @@ public class TestMRAppMaster {
     ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
     MRAppMaster appMaster =
         new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
-            System.currentTimeMillis(), false);
+            System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS,
+            false, false);
     boolean caught = false;
     try {
       MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
@@ -179,7 +186,8 @@ public class TestMRAppMaster {
     ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
     MRAppMaster appMaster =
         new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
-            System.currentTimeMillis(), false);
+            System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS,
+            false, false);
     boolean caught = false;
     try {
       MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
@@ -214,7 +222,8 @@ public class TestMRAppMaster {
     ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
     MRAppMaster appMaster =
         new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
-            System.currentTimeMillis(), false);
+            System.currentTimeMillis(), MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS,
+            false, false);
     boolean caught = false;
     try {
       MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
@@ -230,36 +239,73 @@ public class TestMRAppMaster {
     assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
     appMaster.stop();
   }
+
+  @Test (timeout = 30000)
+  public void testMRAppMasterMaxAppAttempts() throws IOException,
+      InterruptedException {
+    int[] maxAppAttemtps = new int[] { 1, 2, 3 };
+    Boolean[] expectedBools = new Boolean[]{ true, true, false };
+
+    String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
+    String containerIdStr = "container_1317529182569_0004_000002_1";
+
+    String userName = "TestAppMasterUser";
+    ApplicationAttemptId applicationAttemptId = ConverterUtils
+        .toApplicationAttemptId(applicationAttemptIdStr);
+    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
+    YarnConfiguration conf = new YarnConfiguration();
+    conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
+
+    File stagingDir =
+        new File(MRApps.getStagingAreaDir(conf, userName).toString());
+    stagingDir.mkdirs();
+    for (int i = 0; i < maxAppAttemtps.length; ++i) {
+      MRAppMasterTest appMaster =
+          new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
+              System.currentTimeMillis(), maxAppAttemtps[i], false, true);
+      MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
+      assertEquals("isLastAMRetry is correctly computed.", expectedBools[i],
+          appMaster.isLastAMRetry());
+    }
+  }
+
 }
 
 class MRAppMasterTest extends MRAppMaster {
 
   Path stagingDirPath;
   private Configuration conf;
-  private boolean overrideInitAndStart;
+  private boolean overrideInit;
+  private boolean overrideStart;
   ContainerAllocator mockContainerAllocator;
+  CommitterEventHandler mockCommitterEventHandler;
+  RMHeartbeatHandler mockRMHeartbeatHandler;
 
   public MRAppMasterTest(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String host, int port, int httpPort,
-      long submitTime) {
+      long submitTime, int maxAppAttempts) {
-    this(applicationAttemptId, containerId, host, port, httpPort, submitTime, 
+    this(applicationAttemptId, containerId, host, port, httpPort,
-        true);
+        submitTime, maxAppAttempts, true, true);
   }
   public MRAppMasterTest(ApplicationAttemptId applicationAttemptId,
       ContainerId containerId, String host, int port, int httpPort,
-      long submitTime, boolean overrideInitAndStart) {
+      long submitTime, int maxAppAttempts, boolean overrideInit,
-    super(applicationAttemptId, containerId, host, port, httpPort, submitTime);
+      boolean overrideStart) {
-    this.overrideInitAndStart = overrideInitAndStart;
+    super(applicationAttemptId, containerId, host, port, httpPort, submitTime,
+        maxAppAttempts);
+    this.overrideInit = overrideInit;
+    this.overrideStart = overrideStart;
     mockContainerAllocator = mock(ContainerAllocator.class);
+    mockCommitterEventHandler = mock(CommitterEventHandler.class);
+    mockRMHeartbeatHandler = mock(RMHeartbeatHandler.class);
   }
 
   @Override
   public void init(Configuration conf) {
-    if (overrideInitAndStart) {
+    if (!overrideInit) {
-      this.conf = conf; 
-    } else {
       super.init(conf);
     }
+    this.conf = conf;
   }
   
   @Override 
@@ -277,9 +323,20 @@ class MRAppMasterTest extends MRAppMaster {
     return mockContainerAllocator;
   }
 
+  @Override
+  protected EventHandler<CommitterEvent> createCommitterEventHandler(
+      AppContext context, OutputCommitter committer) {
+    return mockCommitterEventHandler;
+  }
+
+  @Override
+  protected RMHeartbeatHandler getRMHeartbeatHandler() {
+    return mockRMHeartbeatHandler;
+  }
+
   @Override
   public void start() {
-    if (overrideInitAndStart) {
+    if (overrideStart) {
       try {
         String user = UserGroupInformation.getCurrentUser().getShortUserName();
         stagingDirPath = MRApps.getStagingAreaDir(conf, user);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java

@@ -49,7 +49,6 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.service.AbstractService;
@@ -93,10 +92,9 @@ import org.junit.Test;
      verify(fs).delete(stagingJobPath, true);
    }
    
-   @Test
+   @Test (timeout = 30000)
    public void testDeletionofStagingOnKill() throws IOException {
      conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
-     conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 4);
      fs = mock(FileSystem.class);
      when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
      //Staging Dir exists
@@ -113,7 +111,7 @@ import org.junit.Test;
      JobId jobid = recordFactory.newRecordInstance(JobId.class);
      jobid.setAppId(appId);
      ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
-     MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc);
+     MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc, 4);
      appMaster.init(conf);
      //simulate the process being killed
      MRAppMaster.MRAppMasterShutdownHook hook = 
@@ -122,10 +120,9 @@ import org.junit.Test;
      verify(fs, times(0)).delete(stagingJobPath, true);
    }
    
-   @Test
+   @Test (timeout = 30000)
    public void testDeletionofStagingOnKillLastTry() throws IOException {
      conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
-     conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 1);
      fs = mock(FileSystem.class);
      when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
      //Staging Dir exists
@@ -142,7 +139,8 @@ import org.junit.Test;
      JobId jobid = recordFactory.newRecordInstance(JobId.class);
      jobid.setAppId(appId);
      ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
-     MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc);
+     MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
+         MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
      appMaster.init(conf);
      //simulate the process being killed
      MRAppMaster.MRAppMasterShutdownHook hook = 
@@ -155,15 +153,16 @@ import org.junit.Test;
      ContainerAllocator allocator;
 
      public TestMRApp(ApplicationAttemptId applicationAttemptId, 
-         ContainerAllocator allocator) {
+         ContainerAllocator allocator, int maxAppAttempts) {
        super(applicationAttemptId, BuilderUtils.newContainerId(
-           applicationAttemptId, 1), "testhost", 2222, 3333, System
+           applicationAttemptId, 1), "testhost", 2222, 3333,
-           .currentTimeMillis());
+           System.currentTimeMillis(), maxAppAttempts);
        this.allocator = allocator;
      }
 
      public TestMRApp(ApplicationAttemptId applicationAttemptId) {
-       this(applicationAttemptId, null);
+       this(applicationAttemptId, null,
+           MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
      }
 
      @Override

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java

@@ -37,9 +37,7 @@ public class LocalClientProtocolProvider extends ClientProtocolProvider {
     if (!MRConfig.LOCAL_FRAMEWORK_NAME.equals(framework)) {
       return null;
     }
-    if (conf.get("mapreduce.job.maps") == null) {
+    conf.setInt(JobContext.NUM_MAPS, 1);
-      conf.setInt("mapreduce.job.maps", 1);
-    }
 
     return new LocalJobRunner(conf);
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java

@@ -19,8 +19,6 @@
 package org.apache.hadoop.mapred;
 
 import java.io.IOException;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.DelayQueue;
 import java.util.concurrent.Delayed;
 import java.util.concurrent.TimeUnit;
 
@@ -39,62 +37,7 @@ public class JobEndNotifier {
   private static final Log LOG =
     LogFactory.getLog(JobEndNotifier.class.getName());
 
-  private static Thread thread;
+ 
-  private static volatile boolean running;
-  private static BlockingQueue<JobEndStatusInfo> queue =
-    new DelayQueue<JobEndStatusInfo>();
-
-  public static void startNotifier() {
-    running = true;
-    thread = new Thread(
-                        new Runnable() {
-                          public void run() {
-                            try {
-                              while (running) {
-                                sendNotification(queue.take());
-                              }
-                            }
-                            catch (InterruptedException irex) {
-                              if (running) {
-                                LOG.error("Thread has ended unexpectedly", irex);
-                              }
-                            }
-                          }
-
-                          private void sendNotification(JobEndStatusInfo notification) {
-                            try {
-                              int code = httpNotification(notification.getUri());
-                              if (code != 200) {
-                                throw new IOException("Invalid response status code: " + code);
-                              }
-                            }
-                            catch (IOException ioex) {
-                              LOG.error("Notification failure [" + notification + "]", ioex);
-                              if (notification.configureForRetry()) {
-                                try {
-                                  queue.put(notification);
-                                }
-                                catch (InterruptedException iex) {
-                                  LOG.error("Notification queuing error [" + notification + "]",
-                                            iex);
-                                }
-                              }
-                            }
-                            catch (Exception ex) {
-                              LOG.error("Notification failure [" + notification + "]", ex);
-                            }
-                          }
-
-                        }
-
-                        );
-    thread.start();
-  }
-
-  public static void stopNotifier() {
-    running = false;
-    thread.interrupt();
-  }
 
   private static JobEndStatusInfo createNotification(JobConf conf,
                                                      JobStatus status) {
@@ -118,18 +61,6 @@ public class JobEndNotifier {
     return notification;
   }
 
-  public static void registerNotification(JobConf jobConf, JobStatus status) {
-    JobEndStatusInfo notification = createNotification(jobConf, status);
-    if (notification != null) {
-      try {
-        queue.put(notification);
-      }
-      catch (InterruptedException iex) {
-        LOG.error("Notification queuing failure [" + notification + "]", iex);
-      }
-    }
-  }
-
   private static int httpNotification(String uri) throws IOException {
     URI url = new URI(uri, false);
     HttpClient m_client = new HttpClient();
@@ -194,10 +125,6 @@ public class JobEndNotifier {
       return retryInterval;
     }
 
-    public long getDelayTime() {
-      return delayTime;
-    }
-
     public boolean configureForRetry() {
       boolean retry = false;
       if (getRetryAttempts() > 0) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/QueueManager.java

@@ -40,7 +40,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
-import java.util.List;
 import java.net.URL;
 
 
@@ -487,73 +486,8 @@ public class QueueManager {
       new QueueAclsInfo[queueAclsInfolist.size()]);
   }
 
-  /**
+ 
-   * ONLY FOR TESTING - Do not use in production code.
+ 
-   * This method is used for setting up of leafQueues only.
-   * We are not setting the hierarchy here.
-   *
-   * @param queues
-   */
-  synchronized void setQueues(Queue[] queues) {
-    root.getChildren().clear();
-    leafQueues.clear();
-    allQueues.clear();
-
-    for (Queue queue : queues) {
-      root.addChild(queue);
-    }
-    //At this point we have root populated
-    //update data structures leafNodes.
-    leafQueues = getRoot().getLeafQueues();
-    allQueues.putAll(getRoot().getInnerQueues());
-    allQueues.putAll(leafQueues);
-  }
-
-  /**
-   * Return an array of {@link JobQueueInfo} objects for the root
-   * queues configured in the system.
-   * <p/>
-   * Root queues are queues that are at the top-most level in the
-   * hierarchy of queues in mapred-queues.xml, or they are the queues
-   * configured in the mapred.queue.names key in mapred-site.xml.
-   *
-   * @return array of JobQueueInfo objects for root level queues.
-   */
-
-  JobQueueInfo[] getRootQueues() {
-    List<JobQueueInfo> list = getRoot().getJobQueueInfo().getChildren();
-    return list.toArray(new JobQueueInfo[list.size()]);
-  }
-
-  /**
-   * Get the complete hierarchy of children for queue
-   * queueName
-   *
-   * @param queueName
-   * @return
-   */
-  JobQueueInfo[] getChildQueues(String queueName) {
-    List<JobQueueInfo> list =
-      allQueues.get(queueName).getJobQueueInfo().getChildren();
-    if (list != null) {
-      return list.toArray(new JobQueueInfo[list.size()]);
-    } else {
-      return new JobQueueInfo[0];
-    }
-  }
-
-  /**
-   * Used only for testing purposes .
-   * This method is unstable as refreshQueues would leave this
-   * data structure in unstable state.
-   *
-   * @param queueName
-   * @return
-   */
-  Queue getQueue(String queueName) {
-    return this.allQueues.get(queueName);
-  }
-
 
   /**
    * Return if ACLs are enabled for the Map/Reduce system
@@ -573,29 +507,7 @@ public class QueueManager {
     return root;
   }
 
-  /**
+  
-   * Returns the specific queue ACL for the given queue.
-   * Returns null if the given queue does not exist or the acl is not
-   * configured for that queue.
-   * If acls are disabled(mapreduce.cluster.acls.enabled set to false), returns
-   * ACL with all users.
-   */
-  synchronized AccessControlList getQueueACL(String queueName,
-      QueueACL qACL) {
-    if (areAclsEnabled) {
-      Queue q = leafQueues.get(queueName);
-      if (q != null) {
-        return q.getAcls().get(toFullPropertyName(
-          queueName, qACL.getAclName()));
-      }
-      else {
-        LOG.warn("Queue " + queueName + " is not present.");
-        return null;
-      }
-    }
-    return new AccessControlList("*");
-  }
-
   /**
    * Dumps the configuration of hierarchy of queues
    * @param out the writer object to which dump is written

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java

@@ -386,73 +386,6 @@ public class TaskLog {
     return conf.getLong(JobContext.TASK_USERLOG_LIMIT, 0) * 1024;
   }
 
-  /**
-   * Wrap a command in a shell to capture stdout and stderr to files.
-   * If the tailLength is 0, the entire output will be saved.
-   * @param cmd The command and the arguments that should be run
-   * @param stdoutFilename The filename that stdout should be saved to
-   * @param stderrFilename The filename that stderr should be saved to
-   * @param tailLength The length of the tail to be saved.
-   * @return the modified command that should be run
-   */
-  public static List<String> captureOutAndError(List<String> cmd, 
-                                                File stdoutFilename,
-                                                File stderrFilename,
-                                                long tailLength
-                                               ) throws IOException {
-    return captureOutAndError(null, cmd, stdoutFilename,
-                              stderrFilename, tailLength, false);
-  }
-
-  /**
-   * Wrap a command in a shell to capture stdout and stderr to files.
-   * Setup commands such as setting memory limit can be passed which 
-   * will be executed before exec.
-   * If the tailLength is 0, the entire output will be saved.
-   * @param setup The setup commands for the execed process.
-   * @param cmd The command and the arguments that should be run
-   * @param stdoutFilename The filename that stdout should be saved to
-   * @param stderrFilename The filename that stderr should be saved to
-   * @param tailLength The length of the tail to be saved.
-   * @return the modified command that should be run
-   */
-  public static List<String> captureOutAndError(List<String> setup,
-                                                List<String> cmd, 
-                                                File stdoutFilename,
-                                                File stderrFilename,
-                                                long tailLength
-                                               ) throws IOException {
-    return captureOutAndError(setup, cmd, stdoutFilename, stderrFilename,
-                              tailLength, false);
-  }
-
-  /**
-   * Wrap a command in a shell to capture stdout and stderr to files.
-   * Setup commands such as setting memory limit can be passed which 
-   * will be executed before exec.
-   * If the tailLength is 0, the entire output will be saved.
-   * @param setup The setup commands for the execed process.
-   * @param cmd The command and the arguments that should be run
-   * @param stdoutFilename The filename that stdout should be saved to
-   * @param stderrFilename The filename that stderr should be saved to
-   * @param tailLength The length of the tail to be saved.
-   * @param pidFileName The name of the pid-file. pid-file's usage is deprecated
-   * @return the modified command that should be run
-   * 
-   * @deprecated     pidFiles are no more used. Instead pid is exported to
-   *                 env variable JVM_PID.
-   */
-  @Deprecated
-  public static List<String> captureOutAndError(List<String> setup,
-                                                List<String> cmd, 
-                                                File stdoutFilename,
-                                                File stderrFilename,
-                                                long tailLength,
-                                                String pidFileName
-                                               ) throws IOException {
-    return captureOutAndError(setup, cmd, stdoutFilename, stderrFilename,
-        tailLength, false);
-  }
   
   /**
    * Wrap a command in a shell to capture stdout and stderr to files.
@@ -607,25 +540,6 @@ public class TaskLog {
     return command.toString();
   }
   
-  /**
-   * Wrap a command in a shell to capture debug script's 
-   * stdout and stderr to debugout.
-   * @param cmd The command and the arguments that should be run
-   * @param debugoutFilename The filename that stdout and stderr
-   *  should be saved to.
-   * @return the modified command that should be run
-   * @throws IOException
-   */
-  public static List<String> captureDebugOut(List<String> cmd, 
-                                             File debugoutFilename
-                                            ) throws IOException {
-    String debugout = FileUtil.makeShellPath(debugoutFilename);
-    List<String> result = new ArrayList<String>(3);
-    result.add(bashCommand);
-    result.add("-c");
-    result.add(buildDebugScriptCommandLine(cmd, debugout));
-    return result;
-  }
   
   /**
    * Method to return the location of user log directory.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskStatus.java

@@ -523,17 +523,5 @@ public abstract class TaskStatus implements Writable, Cloneable {
     return (isMap) ? new MapTaskStatus() : new ReduceTaskStatus();
   }
 
-  static TaskStatus readTaskStatus(DataInput in) throws IOException {
-    boolean isMap = in.readBoolean();
-    TaskStatus taskStatus = createTaskStatus(isMap);
-    taskStatus.readFields(in);
-    return taskStatus;
-  }
-  
-  static void writeTaskStatus(DataOutput out, TaskStatus taskStatus) 
-  throws IOException {
-    out.writeBoolean(taskStatus.getIsMap());
-    taskStatus.write(out);
-  }
 }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -663,5 +663,13 @@ public interface MRJobConfig {
   
   public static final String WORKFLOW_ADJACENCY_PREFIX_PATTERN =
       "^mapreduce\\.workflow\\.adjacency\\..+";
+
+  /**
+   * The maximum number of application attempts.
+   * It is a application-specific setting.
+   */
+  public static final String MR_AM_MAX_ATTEMPTS = "mapreduce.am.max-attempts";
+
+  public static final int DEFAULT_MR_AM_MAX_ATTEMPTS = 1;
   
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java

@@ -1,418 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import com.google.common.base.Charsets;
-
-/**
- * Plugin to calculate resource information on Linux systems.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class LinuxResourceCalculatorPlugin extends ResourceCalculatorPlugin {
-  private static final Log LOG =
-      LogFactory.getLog(LinuxResourceCalculatorPlugin.class);
-
-  public static final int UNAVAILABLE = -1;
-
-  /**
-   * proc's meminfo virtual file has keys-values in the format
-   * "key:[ \t]*value[ \t]kB".
-   */
-  private static final String PROCFS_MEMFILE = "/proc/meminfo";
-  private static final Pattern PROCFS_MEMFILE_FORMAT =
-      Pattern.compile("^([a-zA-Z]*):[ \t]*([0-9]*)[ \t]kB");
-
-  // We need the values for the following keys in meminfo
-  private static final String MEMTOTAL_STRING = "MemTotal";
-  private static final String SWAPTOTAL_STRING = "SwapTotal";
-  private static final String MEMFREE_STRING = "MemFree";
-  private static final String SWAPFREE_STRING = "SwapFree";
-  private static final String INACTIVE_STRING = "Inactive";
-
-  /**
-   * Patterns for parsing /proc/cpuinfo
-   */
-  private static final String PROCFS_CPUINFO = "/proc/cpuinfo";
-  private static final Pattern PROCESSOR_FORMAT =
-      Pattern.compile("^processor[ \t]:[ \t]*([0-9]*)");
-  private static final Pattern FREQUENCY_FORMAT =
-      Pattern.compile("^cpu MHz[ \t]*:[ \t]*([0-9.]*)");
-
-  /**
-   * Pattern for parsing /proc/stat
-   */
-  private static final String PROCFS_STAT = "/proc/stat";
-  private static final Pattern CPU_TIME_FORMAT =
-    Pattern.compile("^cpu[ \t]*([0-9]*)" +
-    		            "[ \t]*([0-9]*)[ \t]*([0-9]*)[ \t].*");
-  
-  private String procfsMemFile;
-  private String procfsCpuFile;
-  private String procfsStatFile;
-  long jiffyLengthInMillis;
-
-  private long ramSize = 0;
-  private long swapSize = 0;
-  private long ramSizeFree = 0;  // free ram space on the machine (kB)
-  private long swapSizeFree = 0; // free swap space on the machine (kB)
-  private long inactiveSize = 0; // inactive cache memory (kB)
-  private int numProcessors = 0; // number of processors on the system
-  private long cpuFrequency = 0L; // CPU frequency on the system (kHz)
-  private long cumulativeCpuTime = 0L; // CPU used time since system is on (ms)
-  private long lastCumulativeCpuTime = 0L; // CPU used time read last time (ms)
-  // Unix timestamp while reading the CPU time (ms)
-  private float cpuUsage = UNAVAILABLE;
-  private long sampleTime = UNAVAILABLE;
-  private long lastSampleTime = UNAVAILABLE;
-  private ProcfsBasedProcessTree pTree = null;
-
-  boolean readMemInfoFile = false;
-  boolean readCpuInfoFile = false;
-  
-  /**
-   * Get current time
-   * @return Unix time stamp in millisecond
-   */
-  long getCurrentTime() {
-    return System.currentTimeMillis();
-  }
-  
-  public LinuxResourceCalculatorPlugin() {
-    procfsMemFile = PROCFS_MEMFILE;
-    procfsCpuFile = PROCFS_CPUINFO;
-    procfsStatFile = PROCFS_STAT;
-    jiffyLengthInMillis = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS;
-    String pid = System.getenv().get("JVM_PID");
-    pTree = new ProcfsBasedProcessTree(pid);
-  }
-  
-  /**
-   * Constructor which allows assigning the /proc/ directories. This will be
-   * used only in unit tests
-   * @param procfsMemFile fake file for /proc/meminfo
-   * @param procfsCpuFile fake file for /proc/cpuinfo
-   * @param procfsStatFile fake file for /proc/stat
-   * @param jiffyLengthInMillis fake jiffy length value
-   */
-  public LinuxResourceCalculatorPlugin(String procfsMemFile,
-                                       String procfsCpuFile,
-                                       String procfsStatFile,
-                                       long jiffyLengthInMillis) {
-    this.procfsMemFile = procfsMemFile;
-    this.procfsCpuFile = procfsCpuFile;
-    this.procfsStatFile = procfsStatFile;
-    this.jiffyLengthInMillis = jiffyLengthInMillis;
-    String pid = System.getenv().get("JVM_PID");
-    pTree = new ProcfsBasedProcessTree(pid);
-  }
-
-  /**
-   * Read /proc/meminfo, parse and compute memory information only once
-   */
-  private void readProcMemInfoFile() {
-    readProcMemInfoFile(false);
-  }
-
-  /**
-   * Read /proc/meminfo, parse and compute memory information
-   * @param readAgain if false, read only on the first time
-   */
-  private void readProcMemInfoFile(boolean readAgain) {
-
-    if (readMemInfoFile && !readAgain) {
-      return;
-    }
-
-    // Read "/proc/memInfo" file
-    BufferedReader in = null;
-    InputStreamReader fReader = null;
-    try {
-      fReader = new InputStreamReader(new FileInputStream(procfsMemFile),
-          Charsets.UTF_8);
-      in = new BufferedReader(fReader);
-    } catch (FileNotFoundException f) {
-      // shouldn't happen....
-      return;
-    }
-
-    Matcher mat = null;
-
-    try {
-      String str = in.readLine();
-      while (str != null) {
-        mat = PROCFS_MEMFILE_FORMAT.matcher(str);
-        if (mat.find()) {
-          if (mat.group(1).equals(MEMTOTAL_STRING)) {
-            ramSize = Long.parseLong(mat.group(2));
-          } else if (mat.group(1).equals(SWAPTOTAL_STRING)) {
-            swapSize = Long.parseLong(mat.group(2));
-          } else if (mat.group(1).equals(MEMFREE_STRING)) {
-            ramSizeFree = Long.parseLong(mat.group(2));
-          } else if (mat.group(1).equals(SWAPFREE_STRING)) {
-            swapSizeFree = Long.parseLong(mat.group(2));
-          } else if (mat.group(1).equals(INACTIVE_STRING)) {
-            inactiveSize = Long.parseLong(mat.group(2));
-          }
-        }
-        str = in.readLine();
-      }
-    } catch (IOException io) {
-      LOG.warn("Error reading the stream " + io);
-    } finally {
-      // Close the streams
-      try {
-        fReader.close();
-        try {
-          in.close();
-        } catch (IOException i) {
-          LOG.warn("Error closing the stream " + in);
-        }
-      } catch (IOException i) {
-        LOG.warn("Error closing the stream " + fReader);
-      }
-    }
-
-    readMemInfoFile = true;
-  }
-
-  /**
-   * Read /proc/cpuinfo, parse and calculate CPU information
-   */
-  private void readProcCpuInfoFile() {
-    // This directory needs to be read only once
-    if (readCpuInfoFile) {
-      return;
-    }
-    // Read "/proc/cpuinfo" file
-    BufferedReader in = null;
-    InputStreamReader fReader = null;
-    try {
-      fReader = new InputStreamReader(new FileInputStream(procfsCpuFile), 
-          Charsets.UTF_8);
-      in = new BufferedReader(fReader);
-    } catch (FileNotFoundException f) {
-      // shouldn't happen....
-      return;
-    }
-    Matcher mat = null;
-    try {
-      numProcessors = 0;
-      String str = in.readLine();
-      while (str != null) {
-        mat = PROCESSOR_FORMAT.matcher(str);
-        if (mat.find()) {
-          numProcessors++;
-        }
-        mat = FREQUENCY_FORMAT.matcher(str);
-        if (mat.find()) {
-          cpuFrequency = (long)(Double.parseDouble(mat.group(1)) * 1000); // kHz
-        }
-        str = in.readLine();
-      }
-    } catch (IOException io) {
-      LOG.warn("Error reading the stream " + io);
-    } finally {
-      // Close the streams
-      try {
-        fReader.close();
-        try {
-          in.close();
-        } catch (IOException i) {
-          LOG.warn("Error closing the stream " + in);
-        }
-      } catch (IOException i) {
-        LOG.warn("Error closing the stream " + fReader);
-      }
-    }
-    readCpuInfoFile = true;
-  }
-
-  /**
-   * Read /proc/stat file, parse and calculate cumulative CPU
-   */
-  private void readProcStatFile() {
-    // Read "/proc/stat" file
-    BufferedReader in = null;
-    InputStreamReader fReader = null;
-    try {
-      fReader = new InputStreamReader(new FileInputStream(procfsStatFile),
-          Charsets.UTF_8);
-      in = new BufferedReader(fReader);
-    } catch (FileNotFoundException f) {
-      // shouldn't happen....
-      return;
-    }
-
-    Matcher mat = null;
-    try {
-      String str = in.readLine();
-      while (str != null) {
-        mat = CPU_TIME_FORMAT.matcher(str);
-        if (mat.find()) {
-          long uTime = Long.parseLong(mat.group(1));
-          long nTime = Long.parseLong(mat.group(2));
-          long sTime = Long.parseLong(mat.group(3));
-          cumulativeCpuTime = uTime + nTime + sTime; // milliseconds
-          break;
-        }
-        str = in.readLine();
-      }
-      cumulativeCpuTime *= jiffyLengthInMillis;
-    } catch (IOException io) {
-      LOG.warn("Error reading the stream " + io);
-    } finally {
-      // Close the streams
-      try {
-        fReader.close();
-        try {
-          in.close();
-        } catch (IOException i) {
-          LOG.warn("Error closing the stream " + in);
-        }
-      } catch (IOException i) {
-        LOG.warn("Error closing the stream " + fReader);
-      }
-    }
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public long getPhysicalMemorySize() {
-    readProcMemInfoFile();
-    return ramSize * 1024;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public long getVirtualMemorySize() {
-    readProcMemInfoFile();
-    return (ramSize + swapSize) * 1024;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public long getAvailablePhysicalMemorySize() {
-    readProcMemInfoFile(true);
-    return (ramSizeFree + inactiveSize) * 1024;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public long getAvailableVirtualMemorySize() {
-    readProcMemInfoFile(true);
-    return (ramSizeFree + swapSizeFree + inactiveSize) * 1024;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public int getNumProcessors() {
-    readProcCpuInfoFile();
-    return numProcessors;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public long getCpuFrequency() {
-    readProcCpuInfoFile();
-    return cpuFrequency;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public long getCumulativeCpuTime() {
-    readProcStatFile();
-    return cumulativeCpuTime;
-  }
-
-  /** {@inheritDoc} */
-  @Override
-  public float getCpuUsage() {
-    readProcStatFile();
-    sampleTime = getCurrentTime();
-    if (lastSampleTime == UNAVAILABLE ||
-        lastSampleTime > sampleTime) {
-      // lastSampleTime > sampleTime may happen when the system time is changed
-      lastSampleTime = sampleTime;
-      lastCumulativeCpuTime = cumulativeCpuTime;
-      return cpuUsage;
-    }
-    // When lastSampleTime is sufficiently old, update cpuUsage.
-    // Also take a sample of the current time and cumulative CPU time for the
-    // use of the next calculation.
-    final long MINIMUM_UPDATE_INTERVAL = 10 * jiffyLengthInMillis;
-    if (sampleTime > lastSampleTime + MINIMUM_UPDATE_INTERVAL) {
-	    cpuUsage = (float)(cumulativeCpuTime - lastCumulativeCpuTime) * 100F /
-	               ((float)(sampleTime - lastSampleTime) * getNumProcessors());
-	    lastSampleTime = sampleTime;
-      lastCumulativeCpuTime = cumulativeCpuTime;
-    }
-    return cpuUsage;
-  }
-
-  /**
-   * Test the {@link LinuxResourceCalculatorPlugin}
-   *
-   * @param args
-   */
-  public static void main(String[] args) {
-    LinuxResourceCalculatorPlugin plugin = new LinuxResourceCalculatorPlugin();
-    System.out.println("Physical memory Size (bytes) : "
-        + plugin.getPhysicalMemorySize());
-    System.out.println("Total Virtual memory Size (bytes) : "
-        + plugin.getVirtualMemorySize());
-    System.out.println("Available Physical memory Size (bytes) : "
-        + plugin.getAvailablePhysicalMemorySize());
-    System.out.println("Total Available Virtual memory Size (bytes) : "
-        + plugin.getAvailableVirtualMemorySize());
-    System.out.println("Number of Processors : " + plugin.getNumProcessors());
-    System.out.println("CPU frequency (kHz) : " + plugin.getCpuFrequency());
-    System.out.println("Cumulative CPU time (ms) : " +
-            plugin.getCumulativeCpuTime());
-    try {
-      // Sleep so we can compute the CPU usage
-      Thread.sleep(500L);
-    } catch (InterruptedException e) {
-      // do nothing
-    }
-    System.out.println("CPU usage % : " + plugin.getCpuUsage());
-  }
-
-  @Override
-  public ProcResourceValues getProcResourceValues() {
-    pTree.updateProcessTree();
-    long cpuTime = pTree.getCumulativeCpuTime();
-    long pMem = pTree.getCumulativeRssmem();
-    long vMem = pTree.getCumulativeVmem();
-    return new ProcResourceValues(cpuTime, pMem, vMem);
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java

@@ -1,743 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.LinkedList;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.StringUtils;
-
-import com.google.common.base.Charsets;
-
-/**
- * A Proc file-system based ProcessTree. Works only on Linux.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class ProcfsBasedProcessTree extends ProcessTree {
-
-  static final Log LOG = LogFactory
-      .getLog(ProcfsBasedProcessTree.class);
-
-  private static final String PROCFS = "/proc/";
-
-  private static final Pattern PROCFS_STAT_FILE_FORMAT = Pattern .compile(
-    "^([0-9-]+)\\s([^\\s]+)\\s[^\\s]\\s([0-9-]+)\\s([0-9-]+)\\s([0-9-]+)\\s" +
-    "([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)\\s([0-9-]+\\s){7}([0-9]+)\\s([0-9]+)" +
-    "(\\s[0-9-]+){15}");
-
-  static final String PROCFS_STAT_FILE = "stat";
-  static final String PROCFS_CMDLINE_FILE = "cmdline";
-  public static final long PAGE_SIZE;
-  static {
-    ShellCommandExecutor shellExecutor =
-            new ShellCommandExecutor(new String[]{"getconf",  "PAGESIZE"});
-    long pageSize = -1;
-    try {
-      shellExecutor.execute();
-      pageSize = Long.parseLong(shellExecutor.getOutput().replace("\n", ""));
-    } catch (IOException e) {
-      LOG.error(StringUtils.stringifyException(e));
-    } finally {
-      PAGE_SIZE = pageSize;
-    }
-  }
-  public static final long JIFFY_LENGTH_IN_MILLIS; // in millisecond
-  static {
-    ShellCommandExecutor shellExecutor =
-            new ShellCommandExecutor(new String[]{"getconf",  "CLK_TCK"});
-    long jiffiesPerSecond = -1;
-    try {
-      shellExecutor.execute();
-      jiffiesPerSecond = Long.parseLong(shellExecutor.getOutput().replace("\n", ""));
-    } catch (IOException e) {
-      LOG.error(StringUtils.stringifyException(e));
-    } finally {
-      JIFFY_LENGTH_IN_MILLIS = jiffiesPerSecond != -1 ?
-                     Math.round(1000D / jiffiesPerSecond) : -1;
-    }
-  }
-
-  // to enable testing, using this variable which can be configured
-  // to a test directory.
-  private String procfsDir;
-  
-  static private String deadPid = "-1";
-  private String pid = deadPid;
-  static private Pattern numberPattern = Pattern.compile("[1-9][0-9]*");
-  private Long cpuTime = 0L;
-  private boolean setsidUsed = false;
-  private long sleeptimeBeforeSigkill = DEFAULT_SLEEPTIME_BEFORE_SIGKILL;
-
-  private Map<String, ProcessInfo> processTree = new HashMap<String, ProcessInfo>();
-
-  public ProcfsBasedProcessTree(String pid) {
-    this(pid, false, DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
-  }
-
-  public ProcfsBasedProcessTree(String pid, boolean setsidUsed,
-                                long sigkillInterval) {
-    this(pid, setsidUsed, sigkillInterval, PROCFS);
-  }
-
-  /**
-   * Build a new process tree rooted at the pid.
-   * 
-   * This method is provided mainly for testing purposes, where
-   * the root of the proc file system can be adjusted.
-   * 
-   * @param pid root of the process tree
-   * @param setsidUsed true, if setsid was used for the root pid
-   * @param sigkillInterval how long to wait between a SIGTERM and SIGKILL 
-   *                        when killing a process tree
-   * @param procfsDir the root of a proc file system - only used for testing. 
-   */
-  public ProcfsBasedProcessTree(String pid, boolean setsidUsed,
-                                long sigkillInterval, String procfsDir) {
-    this.pid = getValidPID(pid);
-    this.setsidUsed = setsidUsed;
-    sleeptimeBeforeSigkill = sigkillInterval;
-    this.procfsDir = procfsDir;
-  }
-  
-  /**
-   * Sets SIGKILL interval
-   * @deprecated Use {@link ProcfsBasedProcessTree#ProcfsBasedProcessTree(
-   *                  String, boolean, long)} instead
-   * @param interval The time to wait before sending SIGKILL
-   *                 after sending SIGTERM
-   */
-  @Deprecated
-  public void setSigKillInterval(long interval) {
-    sleeptimeBeforeSigkill = interval;
-  }
-
-  /**
-   * Checks if the ProcfsBasedProcessTree is available on this system.
-   * 
-   * @return true if ProcfsBasedProcessTree is available. False otherwise.
-   */
-  public static boolean isAvailable() {
-    try {
-      String osName = System.getProperty("os.name");
-      if (!osName.startsWith("Linux")) {
-        LOG.info("ProcfsBasedProcessTree currently is supported only on "
-            + "Linux.");
-        return false;
-      }
-    } catch (SecurityException se) {
-      LOG.warn("Failed to get Operating System name. " + se);
-      return false;
-    }
-    return true;
-  }
-
-  /**
-   * Update the process-tree with latest state. If the root-process is not alive,
-   * tree will become empty.
-   */
-  public void updateProcessTree() {
-    if (!pid.equals(deadPid)) {
-      // Get the list of processes
-      List<String> processList = getProcessList();
-
-      Map<String, ProcessInfo> allProcessInfo = new HashMap<String, ProcessInfo>();
-      
-      // cache the processTree to get the age for processes
-      Map<String, ProcessInfo> oldProcs = 
-              new HashMap<String, ProcessInfo>(processTree);
-      processTree.clear();
-
-      ProcessInfo me = null;
-      for (String proc : processList) {
-        // Get information for each process
-        ProcessInfo pInfo = new ProcessInfo(proc);
-        if (constructProcessInfo(pInfo, procfsDir) != null) {
-          allProcessInfo.put(proc, pInfo);
-          if (proc.equals(this.pid)) {
-            me = pInfo; // cache 'me'
-            processTree.put(proc, pInfo);
-          }
-        }
-      }
-
-      if (me == null) {
-        return;
-      }
-
-      // Add each process to its parent.
-      for (Map.Entry<String, ProcessInfo> entry : allProcessInfo.entrySet()) {
-        String pID = entry.getKey();
-        if (!pID.equals("1")) {
-          ProcessInfo pInfo = entry.getValue();
-          ProcessInfo parentPInfo = allProcessInfo.get(pInfo.getPpid());
-          if (parentPInfo != null) {
-            parentPInfo.addChild(pInfo);
-          }
-        }
-      }
-
-      // now start constructing the process-tree
-      LinkedList<ProcessInfo> pInfoQueue = new LinkedList<ProcessInfo>();
-      pInfoQueue.addAll(me.getChildren());
-      while (!pInfoQueue.isEmpty()) {
-        ProcessInfo pInfo = pInfoQueue.remove();
-        if (!processTree.containsKey(pInfo.getPid())) {
-          processTree.put(pInfo.getPid(), pInfo);
-        }
-        pInfoQueue.addAll(pInfo.getChildren());
-      }
-
-      // update age values and compute the number of jiffies since last update
-      for (Map.Entry<String, ProcessInfo> procs : processTree.entrySet()) {
-        ProcessInfo oldInfo = oldProcs.get(procs.getKey());
-        if (procs.getValue() != null) {
-          procs.getValue().updateJiffy(oldInfo);
-          if (oldInfo != null) {
-            procs.getValue().updateAge(oldInfo);  
-          }
-        }
-      }
-
-      if (LOG.isDebugEnabled()) {
-        // Log.debug the ProcfsBasedProcessTree
-        LOG.debug(this.toString());
-      }
-    }
-  }
-
-  /**
-   * Is the root-process alive?
-   * 
-   * @return true if the root-process is alive, false otherwise.
-   */
-  public boolean isAlive() {
-    if (pid.equals(deadPid)) {
-      return false;
-    } else {
-      return isAlive(pid);
-    }
-  }
-
-  /**
-   * Is any of the subprocesses in the process-tree alive?
-   * 
-   * @return true if any of the processes in the process-tree is
-   *           alive, false otherwise.
-   */
-  public boolean isAnyProcessInTreeAlive() {
-    for (String pId : processTree.keySet()) {
-      if (isAlive(pId)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /** Verify that the given process id is same as its process group id.
-   * @param pidStr Process id of the to-be-verified-process
-   * @param procfsDir  Procfs root dir
-   */
-  static boolean checkPidPgrpidForMatch(String pidStr, String procfsDir) {
-    // Get information for this process
-    ProcessInfo pInfo = new ProcessInfo(pidStr);
-    pInfo = constructProcessInfo(pInfo, procfsDir);
-    if (pInfo == null) {
-      // process group leader may have finished execution, but we still need to
-      // kill the subProcesses in the process group.
-      return true;
-    }
-
-    String pgrpId = pInfo.getPgrpId().toString();
-    //make sure that pId and its pgrpId match
-    if (!pgrpId.equals(pidStr)) {
-      LOG.warn("Unexpected: Process with PID " + pidStr +
-               " is not a process group leader. pgrpId is: " + pInfo.getPgrpId());
-      return false;
-    }
-    if (LOG.isDebugEnabled()) {
-      LOG.debug(pidStr + " is a process group leader, as expected.");
-    }
-    return true;
-  }
-
-  /** Make sure that the given pid is a process group leader and then
-   * destroy the process group.
-   * @param pgrpId   Process group id of to-be-killed-processes
-   * @param interval The time to wait before sending SIGKILL
-   *                 after sending SIGTERM
-   * @param inBackground Process is to be killed in the back ground with
-   *                     a separate thread
-   */
-  public static void assertAndDestroyProcessGroup(String pgrpId, long interval,
-                       boolean inBackground)
-         throws IOException {
-    // Make sure that the pid given is a process group leader
-    if (!checkPidPgrpidForMatch(pgrpId, PROCFS)) {
-      throw new IOException("Process with PID " + pgrpId  +
-                          " is not a process group leader.");
-    }
-    destroyProcessGroup(pgrpId, interval, inBackground);
-  }
-
-  /**
-   * Destroy the process-tree.
-   */
-  public void destroy() {
-    destroy(true);
-  }
-  
-  /**
-   * Destroy the process-tree.
-   * @param inBackground Process is to be killed in the back ground with
-   *                     a separate thread
-   */
-  public void destroy(boolean inBackground) {
-    LOG.debug("Killing ProcfsBasedProcessTree of " + pid);
-    if (pid.equals(deadPid)) {
-      return;
-    }
-    if (isAlive(pid.toString())) {
-      if (isSetsidAvailable && setsidUsed) {
-        // In this case, we know that pid got created using setsid. So kill the
-        // whole processGroup.
-        try {
-          assertAndDestroyProcessGroup(pid.toString(), sleeptimeBeforeSigkill,
-                              inBackground);
-        } catch (IOException e) {
-          LOG.warn(StringUtils.stringifyException(e));
-        }
-      }
-      else {
-        //TODO: Destroy all the processes in the subtree in this case also.
-        // For the time being, killing only the root process.
-        destroyProcess(pid.toString(), sleeptimeBeforeSigkill, inBackground);
-      }
-    }
-  }
-
-  private static final String PROCESSTREE_DUMP_FORMAT =
-      "\t|- %s %s %d %d %s %d %d %d %d %s%n";
-
-  /**
-   * Get a dump of the process-tree.
-   * 
-   * @return a string concatenating the dump of information of all the processes
-   *         in the process-tree
-   */
-  public String getProcessTreeDump() {
-    StringBuilder ret = new StringBuilder();
-    // The header.
-    ret.append(String.format("\t|- PID PPID PGRPID SESSID CMD_NAME "
-        + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
-        + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE%n"));
-    for (ProcessInfo p : processTree.values()) {
-      if (p != null) {
-        ret.append(String.format(PROCESSTREE_DUMP_FORMAT, p.getPid(), p
-            .getPpid(), p.getPgrpId(), p.getSessionId(), p.getName(), p
-            .getUtime(), p.getStime(), p.getVmem(), p.getRssmemPage(), p
-            .getCmdLine(procfsDir)));
-      }
-    }
-    return ret.toString();
-  }
-
-  /**
-   * Get the cumulative virtual memory used by all the processes in the
-   * process-tree.
-   * 
-   * @return cumulative virtual memory used by the process-tree in bytes.
-   */
-  public long getCumulativeVmem() {
-    // include all processes.. all processes will be older than 0.
-    return getCumulativeVmem(0);
-  }
-
-  /**
-   * Get the cumulative resident set size (rss) memory used by all the processes
-   * in the process-tree.
-   *
-   * @return cumulative rss memory used by the process-tree in bytes. return 0
-   *         if it cannot be calculated
-   */
-  public long getCumulativeRssmem() {
-    // include all processes.. all processes will be older than 0.
-    return getCumulativeRssmem(0);
-  }
-
-  /**
-   * Get the cumulative virtual memory used by all the processes in the
-   * process-tree that are older than the passed in age.
-   * 
-   * @param olderThanAge processes above this age are included in the
-   *                      memory addition
-   * @return cumulative virtual memory used by the process-tree in bytes,
-   *          for processes older than this age.
-   */
-  public long getCumulativeVmem(int olderThanAge) {
-    long total = 0;
-    for (ProcessInfo p : processTree.values()) {
-      if ((p != null) && (p.getAge() > olderThanAge)) {
-        total += p.getVmem();
-      }
-    }
-    return total;
-  }
-  
-  /**
-   * Get the cumulative resident set size (rss) memory used by all the processes
-   * in the process-tree that are older than the passed in age.
-   *
-   * @param olderThanAge processes above this age are included in the
-   *                      memory addition
-   * @return cumulative rss memory used by the process-tree in bytes,
-   *          for processes older than this age. return 0 if it cannot be
-   *          calculated
-   */
-  public long getCumulativeRssmem(int olderThanAge) {
-    if (PAGE_SIZE < 0) {
-      return 0;
-    }
-    long totalPages = 0;
-    for (ProcessInfo p : processTree.values()) {
-      if ((p != null) && (p.getAge() > olderThanAge)) {
-        totalPages += p.getRssmemPage();
-      }
-    }
-    return totalPages * PAGE_SIZE; // convert # pages to byte
-  }
-
-  /**
-   * Get the CPU time in millisecond used by all the processes in the
-   * process-tree since the process-tree created
-   *
-   * @return cumulative CPU time in millisecond since the process-tree created
-   *         return 0 if it cannot be calculated
-   */
-  public long getCumulativeCpuTime() {
-    if (JIFFY_LENGTH_IN_MILLIS < 0) {
-      return 0;
-    }
-    long incJiffies = 0;
-    for (ProcessInfo p : processTree.values()) {
-      if (p != null) {
-        incJiffies += p.dtime;
-      }
-    }
-    cpuTime += incJiffies * JIFFY_LENGTH_IN_MILLIS;
-    return cpuTime;
-  }
-
-  private static String getValidPID(String pid) {
-    if (pid == null) return deadPid;
-    Matcher m = numberPattern.matcher(pid);
-    if (m.matches()) return pid;
-    return deadPid;
-  }
-
-  /**
-   * Get the list of all processes in the system.
-   */
-  private List<String> getProcessList() {
-    String[] processDirs = (new File(procfsDir)).list();
-    List<String> processList = new ArrayList<String>();
-
-    for (String dir : processDirs) {
-      Matcher m = numberPattern.matcher(dir);
-      if (!m.matches()) continue;
-      try {
-        if ((new File(procfsDir, dir)).isDirectory()) {
-          processList.add(dir);
-        }
-      } catch (SecurityException s) {
-        // skip this process
-      }
-    }
-    return processList;
-  }
-
-  /**
-   * Construct the ProcessInfo using the process' PID and procfs rooted at the
-   * specified directory and return the same. It is provided mainly to assist
-   * testing purposes.
-   * 
-   * Returns null on failing to read from procfs,
-   *
-   * @param pinfo ProcessInfo that needs to be updated
-   * @param procfsDir root of the proc file system
-   * @return updated ProcessInfo, null on errors.
-   */
-  private static ProcessInfo constructProcessInfo(ProcessInfo pinfo, 
-                                                    String procfsDir) {
-    ProcessInfo ret = null;
-    // Read "procfsDir/<pid>/stat" file - typically /proc/<pid>/stat
-    BufferedReader in = null;
-    InputStreamReader fReader = null;
-    try {
-      File pidDir = new File(procfsDir, pinfo.getPid());
-      fReader = new InputStreamReader(new FileInputStream(
-          new File(pidDir, PROCFS_STAT_FILE)), Charsets.UTF_8);
-      in = new BufferedReader(fReader);
-    } catch (FileNotFoundException f) {
-      // The process vanished in the interim!
-      LOG.info("The process " + pinfo.getPid()
-          + " may have finished in the interim.");
-      return ret;
-    }
-
-    ret = pinfo;
-    try {
-      String str = in.readLine(); // only one line
-      Matcher m = PROCFS_STAT_FILE_FORMAT.matcher(str);
-      boolean mat = m.find();
-      if (mat) {
-        // Set (name) (ppid) (pgrpId) (session) (utime) (stime) (vsize) (rss)
-        pinfo.updateProcessInfo(m.group(2), m.group(3),
-                Integer.parseInt(m.group(4)), Integer.parseInt(m.group(5)),
-                Long.parseLong(m.group(7)), new BigInteger(m.group(8)),
-                Long.parseLong(m.group(10)), Long.parseLong(m.group(11)));
-      } else {
-        LOG.warn("Unexpected: procfs stat file is not in the expected format"
-            + " for process with pid " + pinfo.getPid());
-        ret = null;
-      }
-    } catch (IOException io) {
-      LOG.warn("Error reading the stream " + io);
-      ret = null;
-    } finally {
-      // Close the streams
-      try {
-        fReader.close();
-        try {
-          in.close();
-        } catch (IOException i) {
-          LOG.warn("Error closing the stream " + in);
-        }
-      } catch (IOException i) {
-        LOG.warn("Error closing the stream " + fReader);
-      }
-    }
-
-    return ret;
-  }
-  /**
-   * Returns a string printing PIDs of process present in the
-   * ProcfsBasedProcessTree. Output format : [pid pid ..]
-   */
-  public String toString() {
-    StringBuffer pTree = new StringBuffer("[ ");
-    for (String p : processTree.keySet()) {
-      pTree.append(p);
-      pTree.append(" ");
-    }
-    return pTree.substring(0, pTree.length()) + "]";
-  }
-
-  /**
-   * 
-   * Class containing information of a process.
-   * 
-   */
-  private static class ProcessInfo {
-    private String pid; // process-id
-    private String name; // command name
-    private Integer pgrpId; // process group-id
-    private String ppid; // parent process-id
-    private Integer sessionId; // session-id
-    private Long vmem; // virtual memory usage
-    private Long rssmemPage; // rss memory usage in # of pages
-    private Long utime = 0L; // # of jiffies in user mode
-    private final BigInteger MAX_LONG = BigInteger.valueOf(Long.MAX_VALUE);
-    private BigInteger stime = new BigInteger("0"); // # of jiffies in kernel mode
-    // how many times has this process been seen alive
-    private int age; 
-
-    // # of jiffies used since last update:
-    private Long dtime = 0L;
-    // dtime = (utime + stime) - (utimeOld + stimeOld)
-    // We need this to compute the cumulative CPU time
-    // because the subprocess may finish earlier than root process
-
-    private List<ProcessInfo> children = new ArrayList<ProcessInfo>(); // list of children
-
-    public ProcessInfo(String pid) {
-      this.pid = pid;
-      // seeing this the first time.
-      this.age = 1;
-    }
-
-    public String getPid() {
-      return pid;
-    }
-
-    public String getName() {
-      return name;
-    }
-
-    public Integer getPgrpId() {
-      return pgrpId;
-    }
-
-    public String getPpid() {
-      return ppid;
-    }
-
-    public Integer getSessionId() {
-      return sessionId;
-    }
-
-    public Long getVmem() {
-      return vmem;
-    }
-
-    public Long getUtime() {
-      return utime;
-    }
-
-    public BigInteger getStime() {
-      return stime;
-    }
-
-    public Long getDtime() {
-      return dtime;
-    }
-
-    public Long getRssmemPage() { // get rss # of pages
-      return rssmemPage;
-    }
-
-    public int getAge() {
-      return age;
-    }
-    
-    public boolean isParent(ProcessInfo p) {
-      if (pid.equals(p.getPpid())) {
-        return true;
-      }
-      return false;
-    }
-
-    public void updateProcessInfo(String name, String ppid, Integer pgrpId,
-        Integer sessionId, Long utime, BigInteger stime, Long vmem, Long rssmem) {
-      this.name = name;
-      this.ppid = ppid;
-      this.pgrpId = pgrpId;
-      this.sessionId = sessionId;
-      this.utime = utime;
-      this.stime = stime;
-      this.vmem = vmem;
-      this.rssmemPage = rssmem;
-    }
-
-    public void updateJiffy(ProcessInfo oldInfo) {
-      if (oldInfo == null) {
-        BigInteger sum = this.stime.add(BigInteger.valueOf(this.utime));
-        if (sum.compareTo(MAX_LONG) > 0) {
-          this.dtime = 0L;
-          LOG.warn("Sum of stime (" + this.stime + ") and utime (" + this.utime
-              + ") is greater than " + Long.MAX_VALUE);
-        } else {
-          this.dtime = sum.longValue();
-        }
-        return;
-      }
-      this.dtime = (this.utime - oldInfo.utime +
-          this.stime.subtract(oldInfo.stime).longValue());
-    }
-
-    public void updateAge(ProcessInfo oldInfo) {
-      this.age = oldInfo.age + 1;
-    }
-    
-    public boolean addChild(ProcessInfo p) {
-      return children.add(p);
-    }
-
-    public List<ProcessInfo> getChildren() {
-      return children;
-    }
-
-    public String getCmdLine(String procfsDir) {
-      String ret = "N/A";
-      if (pid == null) {
-        return ret;
-      }
-      BufferedReader in = null;
-      InputStreamReader fReader = null;
-      try {
-        fReader = new InputStreamReader(new FileInputStream(
-            new File(new File(procfsDir, pid), PROCFS_CMDLINE_FILE)),
-            Charsets.UTF_8);
-      } catch (FileNotFoundException f) {
-        // The process vanished in the interim!
-        return ret;
-      }
-
-      in = new BufferedReader(fReader);
-
-      try {
-        ret = in.readLine(); // only one line
-        if (ret == null) {
-          ret = "N/A";
-        } else {
-          ret = ret.replace('\0', ' '); // Replace each null char with a space
-          if (ret.equals("")) {
-            // The cmdline might be empty because the process is swapped out or
-            // is a zombie.
-            ret = "N/A";
-          }
-        }
-      } catch (IOException io) {
-        LOG.warn("Error reading the stream " + io);
-        ret = "N/A";
-      } finally {
-        // Close the streams
-        try {
-          fReader.close();
-          try {
-            in.close();
-          } catch (IOException i) {
-            LOG.warn("Error closing the stream " + in);
-          }
-        } catch (IOException i) {
-          LOG.warn("Error closing the stream " + fReader);
-        }
-      }
-
-      return ret;
-    }
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceCalculatorPlugin.java

@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * Plugin to calculate resource information on the system.
- * 
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public abstract class ResourceCalculatorPlugin extends Configured {
-
-  /**
-   * Obtain the total size of the virtual memory present in the system.
-   * 
-   * @return virtual memory size in bytes.
-   */
-  public abstract long getVirtualMemorySize();
-
-  /**
-   * Obtain the total size of the physical memory present in the system.
-   * 
-   * @return physical memory size bytes.
-   */
-  public abstract long getPhysicalMemorySize();
-
-  /**
-   * Obtain the total size of the available virtual memory present
-   * in the system.
-   *
-   * @return available virtual memory size in bytes.
-   */
-  public abstract long getAvailableVirtualMemorySize();
-
-  /**
-   * Obtain the total size of the available physical memory present
-   * in the system.
-   *
-   * @return available physical memory size bytes.
-   */
-  public abstract long getAvailablePhysicalMemorySize();
-
-  /**
-   * Obtain the total number of processors present on the system.
-   *
-   * @return number of processors
-   */
-  public abstract int getNumProcessors();
-
-  /**
-   * Obtain the CPU frequency of on the system.
-   *
-   * @return CPU frequency in kHz
-   */
-  public abstract long getCpuFrequency();
-
-  /**
-   * Obtain the cumulative CPU time since the system is on.
-   *
-   * @return cumulative CPU time in milliseconds
-   */
-  public abstract long getCumulativeCpuTime();
-
-  /**
-   * Obtain the CPU usage % of the machine. Return -1 if it is unavailable
-   *
-   * @return CPU usage in %
-   */
-  public abstract float getCpuUsage();
-
-  /**
-   * Obtain resource status used by current process tree.
-   */
-  @InterfaceAudience.Private
-  @InterfaceStability.Unstable
-  public abstract ProcResourceValues getProcResourceValues();
-
-  public static class ProcResourceValues {
-    private final long cumulativeCpuTime;
-    private final long physicalMemorySize;
-    private final long virtualMemorySize;
-    public ProcResourceValues(long cumulativeCpuTime, long physicalMemorySize,
-                              long virtualMemorySize) {
-      this.cumulativeCpuTime = cumulativeCpuTime;
-      this.physicalMemorySize = physicalMemorySize;
-      this.virtualMemorySize = virtualMemorySize;
-    }
-    /**
-     * Obtain the physical memory size used by current process tree.
-     * @return physical memory size in bytes.
-     */
-    public long getPhysicalMemorySize() {
-      return physicalMemorySize;
-    }
-
-    /**
-     * Obtain the virtual memory size used by a current process tree.
-     * @return virtual memory size in bytes.
-     */
-    public long getVirtualMemorySize() {
-      return virtualMemorySize;
-    }
-
-    /**
-     * Obtain the cumulative CPU time used by a current process tree.
-     * @return cumulative CPU time in milliseconds
-     */
-    public long getCumulativeCpuTime() {
-      return cumulativeCpuTime;
-    }
-  }
-
-  /**
-   * Get the ResourceCalculatorPlugin from the class name and configure it. If
-   * class name is null, this method will try and return a memory calculator
-   * plugin available for this system.
-   * 
-   * @param clazz class-name
-   * @param conf configure the plugin with this.
-   * @return ResourceCalculatorPlugin
-   */
-  public static ResourceCalculatorPlugin getResourceCalculatorPlugin(
-      Class<? extends ResourceCalculatorPlugin> clazz, Configuration conf) {
-
-    if (clazz != null) {
-      return ReflectionUtils.newInstance(clazz, conf);
-    }
-
-    // No class given, try a os specific class
-    try {
-      String osName = System.getProperty("os.name");
-      if (osName.startsWith("Linux")) {
-        return new LinuxResourceCalculatorPlugin();
-      }
-    } catch (SecurityException se) {
-      // Failed to get Operating System name.
-      return null;
-    }
-
-    // Not supported on this system.
-    return null;
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

@@ -806,6 +806,14 @@
   </description>
 </property>
 
+<property>
+  <name>mapreduce.am.max-attempts</name>
+  <value>1</value>
+  <description>The maximum number of application attempts. It is a
+  application-specific setting. It should not be larger than the global number
+  set by resourcemanager. Otherwise, it will be override.</description>
+</property>
+
 <!-- Job Notification Configuration -->
 <property>
  <name>mapreduce.job.end-notification.url</name>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestClock.java

@@ -1,3 +1,4 @@
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -15,15 +16,23 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.hadoop.yarn.server.api.records;
+package org.apache.hadoop.mapred;
 
-public interface RegistrationResponse {
+import org.junit.Test;
 
-  MasterKey getMasterKey();
+import static org.junit.Assert.*;
-  
+/**
-  void setMasterKey(MasterKey secretKey);
+ *  test Clock class
-  
+ *
-  NodeAction getNodeAction();
+ */
-  
+public class TestClock {
-  void setNodeAction(NodeAction nodeAction);
+
+  @Test  (timeout=1000)
+  public void testClock(){
+    Clock clock= new Clock();
+    long templateTime=System.currentTimeMillis();
+    long time=clock.getTime();
+    assertEquals(templateTime, time,30);
+
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobConf.java

@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.util.regex.Pattern;
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+
+/**
+ * test JobConf
+ * 
+ */
+public class TestJobConf {
+
+  /**
+   * test getters and setters of JobConf
+   */
+  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
+  public void testJobConf() {
+    JobConf conf = new JobConf();
+    // test default value
+    Pattern pattern = conf.getJarUnpackPattern();
+    assertEquals(Pattern.compile("(?:classes/|lib/).*").toString(),
+        pattern.toString());
+    // default value
+    assertFalse(conf.getKeepFailedTaskFiles());
+    conf.setKeepFailedTaskFiles(true);
+    assertTrue(conf.getKeepFailedTaskFiles());
+
+    // default value
+    assertNull(conf.getKeepTaskFilesPattern());
+    conf.setKeepTaskFilesPattern("123454");
+    assertEquals("123454", conf.getKeepTaskFilesPattern());
+
+    // default value
+    assertNotNull(conf.getWorkingDirectory());
+    conf.setWorkingDirectory(new Path("test"));
+    assertTrue(conf.getWorkingDirectory().toString().endsWith("test"));
+
+    // default value
+    assertEquals(1, conf.getNumTasksToExecutePerJvm());
+
+    // default value
+    assertNull(conf.getKeyFieldComparatorOption());
+    conf.setKeyFieldComparatorOptions("keySpec");
+    assertEquals("keySpec", conf.getKeyFieldComparatorOption());
+
+    // default value
+    assertFalse(conf.getUseNewReducer());
+    conf.setUseNewReducer(true);
+    assertTrue(conf.getUseNewReducer());
+
+    // default
+    assertTrue(conf.getMapSpeculativeExecution());
+    assertTrue(conf.getReduceSpeculativeExecution());
+    assertTrue(conf.getSpeculativeExecution());
+    conf.setReduceSpeculativeExecution(false);
+    assertTrue(conf.getSpeculativeExecution());
+
+    conf.setMapSpeculativeExecution(false);
+    assertFalse(conf.getSpeculativeExecution());
+    assertFalse(conf.getMapSpeculativeExecution());
+    assertFalse(conf.getReduceSpeculativeExecution());
+
+    conf.setSessionId("ses");
+    assertEquals("ses", conf.getSessionId());
+
+    assertEquals(3, conf.getMaxTaskFailuresPerTracker());
+    conf.setMaxTaskFailuresPerTracker(2);
+    assertEquals(2, conf.getMaxTaskFailuresPerTracker());
+
+    assertEquals(0, conf.getMaxMapTaskFailuresPercent());
+    conf.setMaxMapTaskFailuresPercent(50);
+    assertEquals(50, conf.getMaxMapTaskFailuresPercent());
+
+    assertEquals(0, conf.getMaxReduceTaskFailuresPercent());
+    conf.setMaxReduceTaskFailuresPercent(70);
+    assertEquals(70, conf.getMaxReduceTaskFailuresPercent());
+
+    // by default
+    assertEquals(JobPriority.NORMAL.name(), conf.getJobPriority().name());
+    conf.setJobPriority(JobPriority.HIGH);
+    assertEquals(JobPriority.HIGH.name(), conf.getJobPriority().name());
+
+    assertNull(conf.getJobSubmitHostName());
+    conf.setJobSubmitHostName("hostname");
+    assertEquals("hostname", conf.getJobSubmitHostName());
+
+    // default
+    assertNull(conf.getJobSubmitHostAddress());
+    conf.setJobSubmitHostAddress("ww");
+    assertEquals("ww", conf.getJobSubmitHostAddress());
+
+    // default value
+    assertFalse(conf.getProfileEnabled());
+    conf.setProfileEnabled(true);
+    assertTrue(conf.getProfileEnabled());
+
+    // default value
+    assertEquals(conf.getProfileTaskRange(true).toString(), "0-2");
+    assertEquals(conf.getProfileTaskRange(false).toString(), "0-2");
+    conf.setProfileTaskRange(true, "0-3");
+    assertEquals(conf.getProfileTaskRange(false).toString(), "0-2");
+    assertEquals(conf.getProfileTaskRange(true).toString(), "0-3");
+
+    // default value
+    assertNull(conf.getMapDebugScript());
+    conf.setMapDebugScript("mDbgScript");
+    assertEquals("mDbgScript", conf.getMapDebugScript());
+
+    // default value
+    assertNull(conf.getReduceDebugScript());
+    conf.setReduceDebugScript("rDbgScript");
+    assertEquals("rDbgScript", conf.getReduceDebugScript());
+
+    // default value
+    assertNull(conf.getJobLocalDir());
+
+    assertEquals("default", conf.getQueueName());
+    conf.setQueueName("qname");
+    assertEquals("qname", conf.getQueueName());
+
+    assertEquals(1, conf.computeNumSlotsPerMap(100L));
+    assertEquals(1, conf.computeNumSlotsPerReduce(100L));
+
+    conf.setMemoryForMapTask(100 * 1000);
+    assertEquals(1000, conf.computeNumSlotsPerMap(100L));
+    conf.setMemoryForReduceTask(1000 * 1000);
+    assertEquals(1000, conf.computeNumSlotsPerReduce(1000L));
+
+    assertEquals(-1, conf.getMaxPhysicalMemoryForTask());
+    assertEquals("The variable key is no longer used.",
+        JobConf.deprecatedString("key"));
+
+  }
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestJobInfo.java

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * test class JobInfo
+ * 
+ * 
+ */
+public class TestJobInfo {
+  @Test (timeout=5000)
+  public void testJobInfo() throws IOException {
+    JobID jid = new JobID("001", 1);
+    Text user = new Text("User");
+    Path path = new Path("/tmp/test");
+    JobInfo info = new JobInfo(jid, user, path);
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    info.write(new DataOutputStream(out));
+
+    JobInfo copyinfo = new JobInfo();
+    copyinfo.readFields(new DataInputStream(new ByteArrayInputStream(out
+        .toByteArray())));
+    assertEquals(info.getJobID().toString(), copyinfo.getJobID().toString());
+    assertEquals(info.getJobSubmitDir().getName(), copyinfo.getJobSubmitDir()
+        .getName());
+    assertEquals(info.getUser().toString(), copyinfo.getUser().toString());
+
+  }
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestOldMethodsJobID.java

@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.mapred.TaskCompletionEvent.Status;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * Test deprecated methods
+ *
+ */
+public class TestOldMethodsJobID {
+
+  /**
+   * test deprecated methods of TaskID
+   * @throws IOException
+   */
+  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
+  public void testDepricatedMethods() throws IOException {
+    JobID jid = new JobID();
+    TaskID test = new TaskID(jid, true, 1);
+    assertEquals(test.getTaskType(), TaskType.MAP);
+    test = new TaskID(jid, false, 1);
+    assertEquals(test.getTaskType(), TaskType.REDUCE);
+
+    test = new TaskID("001", 1, false, 1);
+    assertEquals(test.getTaskType(), TaskType.REDUCE);
+    test = new TaskID("001", 1, true, 1);
+    assertEquals(test.getTaskType(), TaskType.MAP);
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+    test.write(new DataOutputStream(out));
+    TaskID ti = TaskID.read(new DataInputStream(new ByteArrayInputStream(out
+        .toByteArray())));
+    assertEquals(ti.toString(), test.toString());
+    assertEquals("task_001_0001_m_000002",
+        TaskID.getTaskIDsPattern("001", 1, true, 2));
+    assertEquals("task_003_0001_m_000004",
+        TaskID.getTaskIDsPattern("003", 1, TaskType.MAP, 4));
+    assertEquals("003_0001_m_000004",
+        TaskID.getTaskIDsPatternWOPrefix("003", 1, TaskType.MAP, 4).toString());
+   
+  }
+  
+  /**
+   * test JobID
+   * @throws IOException 
+   */
+  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
+  public void testJobID() throws IOException{
+    JobID jid = new JobID("001",2);
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    jid.write(new DataOutputStream(out));
+   assertEquals(jid,JobID.read(new DataInputStream(new ByteArrayInputStream(out.toByteArray()))));
+   assertEquals("job_001_0001",JobID.getJobIDsPattern("001",1));
+  }
+  /**
+   * test deprecated methods of TaskCompletionEvent
+   */
+  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
+  public void testTaskCompletionEvent() {
+    TaskAttemptID taid = new TaskAttemptID("001", 1, TaskType.REDUCE, 2, 3);
+    TaskCompletionEvent template = new TaskCompletionEvent(12, taid, 13, true,
+        Status.SUCCEEDED, "httptracker");
+    TaskCompletionEvent testEl = TaskCompletionEvent.downgrade(template);
+    testEl.setTaskAttemptId(taid);
+    testEl.setTaskTrackerHttp("httpTracker");
+
+    testEl.setTaskId("attempt_001_0001_m_000002_04");
+    assertEquals("attempt_001_0001_m_000002_4",testEl.getTaskId());
+
+    testEl.setTaskStatus(Status.OBSOLETE);
+    assertEquals(Status.OBSOLETE.toString(), testEl.getStatus().toString());
+
+    testEl.setTaskRunTime(20);
+    assertEquals(testEl.getTaskRunTime(), 20);
+    testEl.setEventId(16);
+    assertEquals(testEl.getEventId(), 16);
+
+  }
+
+  /**
+   * test depricated methods of JobProfile
+   * @throws IOException
+   */
+  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
+  public void testJobProfile() throws IOException {
+
+    JobProfile profile = new JobProfile("user", "job_001_03", "jobFile", "uri",
+        "name");
+    assertEquals("job_001_0003", profile.getJobId());
+    assertEquals("default", profile.getQueueName());
+   // serialization test
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    profile.write(new DataOutputStream(out));
+
+    JobProfile profile2 = new JobProfile();
+    profile2.readFields(new DataInputStream(new ByteArrayInputStream(out
+        .toByteArray())));
+
+    assertEquals(profile2.name, profile.name);
+    assertEquals(profile2.jobFile, profile.jobFile);
+    assertEquals(profile2.queueName, profile.queueName);
+    assertEquals(profile2.url, profile.url);
+    assertEquals(profile2.user, profile.user);
+  }
+  /**
+   * test TaskAttemptID 
+   */
+  @SuppressWarnings( "deprecation" )
+  @Test (timeout=5000)
+  public void testTaskAttemptID (){
+    TaskAttemptID task  = new TaskAttemptID("001",2,true,3,4);
+    assertEquals("attempt_001_0002_m_000003_4", TaskAttemptID.getTaskAttemptIDsPattern("001", 2, true, 3, 4));
+    assertEquals("task_001_0002_m_000003", task.getTaskID().toString());
+    assertEquals("attempt_001_0001_r_000002_3",TaskAttemptID.getTaskAttemptIDsPattern("001", 1, TaskType.REDUCE, 2, 3));
+    assertEquals("001_0001_m_000001_2", TaskAttemptID.getTaskAttemptIDsPatternWOPrefix("001",1, TaskType.MAP, 1, 2).toString());
+    
+  }
+  
+  /**
+   * test Reporter.NULL
+   * 
+   */
+  
+  @Test (timeout=5000)
+  public void testReporter(){
+    Reporter nullReporter=Reporter.NULL;
+    assertNull(nullReporter.getCounter(null));
+    assertNull(nullReporter.getCounter("group", "name"));
+    // getInputSplit method removed
+    try{
+      assertNull(nullReporter.getInputSplit());
+    }catch(UnsupportedOperationException e){
+      assertEquals( "NULL reporter has no input",e.getMessage());
+    }
+    assertEquals(0,nullReporter.getProgress(),0.01);
+
+  }
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestQueue.java

@@ -0,0 +1,238 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+
+import static junit.framework.Assert.*;
+import static org.mockito.Mockito.*;
+
+/**
+ * TestCounters checks the sanity and recoverability of Queue
+ */
+public class TestQueue {
+
+  /**
+   * test QueueManager
+   * configuration from file
+   * 
+   * @throws IOException
+   */
+  @Test (timeout=5000)
+  public void testQueue() throws IOException {
+    File f = null;
+    try {
+      f = writeFile();
+
+      QueueManager manager = new QueueManager(f.getCanonicalPath(), true);
+      manager.setSchedulerInfo("first", "queueInfo");
+      manager.setSchedulerInfo("second", "queueInfoqueueInfo");
+      Queue root = manager.getRoot();
+      assertTrue(root.getChildren().size() == 2);
+      Iterator<Queue> iterator = root.getChildren().iterator();
+      Queue firstSubQueue = iterator.next();
+      assertTrue(firstSubQueue.getName().equals("first"));
+      assertEquals(
+          firstSubQueue.getAcls().get("mapred.queue.first.acl-submit-job")
+              .toString(),
+          "Users [user1, user2] and members of the groups [group1, group2] are allowed");
+      Queue secondSubQueue = iterator.next();
+      assertTrue(secondSubQueue.getName().equals("second"));
+      assertEquals(secondSubQueue.getProperties().getProperty("key"), "value");
+      assertEquals(secondSubQueue.getProperties().getProperty("key1"), "value1");
+      // test status
+      assertEquals(firstSubQueue.getState().getStateName(), "running");
+      assertEquals(secondSubQueue.getState().getStateName(), "stopped");
+
+      Set<String> template = new HashSet<String>();
+      template.add("first");
+      template.add("second");
+      assertEquals(manager.getLeafQueueNames(), template);
+
+      // test user access
+
+      UserGroupInformation mockUGI = mock(UserGroupInformation.class);
+      when(mockUGI.getShortUserName()).thenReturn("user1");
+      String[] groups = { "group1" };
+      when(mockUGI.getGroupNames()).thenReturn(groups);
+      assertTrue(manager.hasAccess("first", QueueACL.SUBMIT_JOB, mockUGI));
+      assertFalse(manager.hasAccess("second", QueueACL.SUBMIT_JOB, mockUGI));
+      assertFalse(manager.hasAccess("first", QueueACL.ADMINISTER_JOBS, mockUGI));
+      when(mockUGI.getShortUserName()).thenReturn("user3");
+      assertTrue(manager.hasAccess("first", QueueACL.ADMINISTER_JOBS, mockUGI));
+
+      QueueAclsInfo[] qai = manager.getQueueAcls(mockUGI);
+      assertEquals(qai.length, 1);
+      // test refresh queue
+      manager.refreshQueues(getConfiguration(), null);
+
+      iterator = root.getChildren().iterator();
+      Queue firstSubQueue1 = iterator.next();
+      Queue secondSubQueue1 = iterator.next();
+      // tets equal method
+      assertTrue(firstSubQueue.equals(firstSubQueue1));
+      assertEquals(firstSubQueue1.getState().getStateName(), "running");
+      assertEquals(secondSubQueue1.getState().getStateName(), "stopped");
+
+      assertEquals(firstSubQueue1.getSchedulingInfo(), "queueInfo");
+      assertEquals(secondSubQueue1.getSchedulingInfo(), "queueInfoqueueInfo");
+
+      // test JobQueueInfo
+      assertEquals(firstSubQueue.getJobQueueInfo().getQueueName(), "first");
+      assertEquals(firstSubQueue.getJobQueueInfo().getQueueState(), "running");
+      assertEquals(firstSubQueue.getJobQueueInfo().getSchedulingInfo(),
+          "queueInfo");
+      assertEquals(secondSubQueue.getJobQueueInfo().getChildren().size(), 0);
+      // test
+      assertEquals(manager.getSchedulerInfo("first"), "queueInfo");
+      assertEquals(manager.getJobQueueInfos()[0].getQueueName(), secondSubQueue
+          .getJobQueueInfo().getQueueName());
+      assertEquals(manager.getJobQueueInfos()[1].getQueueName(), firstSubQueue
+          .getJobQueueInfo().getQueueName());
+      // test getJobQueueInfoMapping
+      assertEquals(
+          manager.getJobQueueInfoMapping().get("first").getQueueName(), "first");
+      // test dumpConfiguration
+      Writer writer = new StringWriter();
+
+      Configuration conf = getConfiguration();
+      conf.unset(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY);
+      QueueManager.dumpConfiguration(writer, f.getAbsolutePath(), conf);
+      String result = writer.toString();
+      assertTrue(result
+          .indexOf("\"name\":\"first\",\"state\":\"running\",\"acl_submit_job\":\"user1,user2 group1,group2\",\"acl_administer_jobs\":\"user3,user4 group3,group4\",\"properties\":[],\"children\":[]") > 0);
+
+      writer = new StringWriter();
+      QueueManager.dumpConfiguration(writer, conf);
+      result = writer.toString();
+      assertEquals(
+          "{\"queues\":[{\"name\":\"default\",\"state\":\"running\",\"acl_submit_job\":\"*\",\"acl_administer_jobs\":\"*\",\"properties\":[],\"children\":[]},{\"name\":\"q1\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[],\"children\":[{\"name\":\"q1:q2\",\"state\":\"running\",\"acl_submit_job\":\" \",\"acl_administer_jobs\":\" \",\"properties\":[{\"key\":\"capacity\",\"value\":\"20\"},{\"key\":\"user-limit\",\"value\":\"30\"}],\"children\":[]}]}]}",
+          result);
+      // test constructor QueueAclsInfo
+      QueueAclsInfo qi = new QueueAclsInfo();
+      assertNull(qi.getQueueName());
+
+    } finally {
+      if (f != null) {
+        f.delete();
+      }
+    }
+  }
+
+  private Configuration getConfiguration() {
+    Configuration conf = new Configuration();
+    conf.set(DeprecatedQueueConfigurationParser.MAPRED_QUEUE_NAMES_KEY,
+        "first,second");
+    conf.set(QueueManager.QUEUE_CONF_PROPERTY_NAME_PREFIX
+        + "first.acl-submit-job", "user1,user2 group1,group2");
+    conf.set(MRConfig.MR_ACLS_ENABLED, "true");
+    conf.set(QueueManager.QUEUE_CONF_PROPERTY_NAME_PREFIX + "first.state",
+        "running");
+    conf.set(QueueManager.QUEUE_CONF_PROPERTY_NAME_PREFIX + "second.state",
+        "stopped");
+    return conf;
+  }
+
+  @Test (timeout=5000)
+  public void testDefaultConfig() {
+    QueueManager manager = new QueueManager(true);
+    assertEquals(manager.getRoot().getChildren().size(), 2);
+  }
+
+  /**
+   * test for Qmanager with empty configuration
+   * 
+   * @throws IOException
+   */
+
+  @Test (timeout=5000)
+  public void test2Queue() throws IOException {
+    Configuration conf = getConfiguration();
+
+    QueueManager manager = new QueueManager(conf);
+    manager.setSchedulerInfo("first", "queueInfo");
+    manager.setSchedulerInfo("second", "queueInfoqueueInfo");
+
+    Queue root = manager.getRoot();
+    
+    // test children queues
+    assertTrue(root.getChildren().size() == 2);
+    Iterator<Queue> iterator = root.getChildren().iterator();
+    Queue firstSubQueue = iterator.next();
+    assertTrue(firstSubQueue.getName().equals("first"));
+    assertEquals(
+        firstSubQueue.getAcls().get("mapred.queue.first.acl-submit-job")
+            .toString(),
+        "Users [user1, user2] and members of the groups [group1, group2] are allowed");
+    Queue secondSubQueue = iterator.next();
+    assertTrue(secondSubQueue.getName().equals("second"));
+
+    assertEquals(firstSubQueue.getState().getStateName(), "running");
+    assertEquals(secondSubQueue.getState().getStateName(), "stopped");
+    assertTrue(manager.isRunning("first"));
+    assertFalse(manager.isRunning("second"));
+
+    assertEquals(firstSubQueue.getSchedulingInfo(), "queueInfo");
+    assertEquals(secondSubQueue.getSchedulingInfo(), "queueInfoqueueInfo");
+// test leaf queue
+    Set<String> template = new HashSet<String>();
+    template.add("first");
+    template.add("second");
+    assertEquals(manager.getLeafQueueNames(), template);
+
+    
+  }
+/**
+ * write cofiguration
+ * @return
+ * @throws IOException
+ */
+  private File writeFile() throws IOException {
+
+    File f = null;
+    f = File.createTempFile("tst", "xml");
+    BufferedWriter out = new BufferedWriter(new FileWriter(f));
+    String properties = "<properties><property key=\"key\" value=\"value\"/><property key=\"key1\" value=\"value1\"/></properties>";
+    out.write("<queues>");
+    out.newLine();
+    out.write("<queue><name>first</name><acl-submit-job>user1,user2 group1,group2</acl-submit-job><acl-administer-jobs>user3,user4 group3,group4</acl-administer-jobs><state>running</state></queue>");
+    out.newLine();
+    out.write("<queue><name>second</name><acl-submit-job>u1,u2 g1,g2</acl-submit-job>"
+        + properties + "<state>stopped</state></queue>");
+    out.newLine();
+    out.write("</queues>");
+    out.flush();
+    out.close();
+    return f;
+
+  }
+  
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestSkipBadRecords.java

@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+/**
+ * test SkipBadRecords
+ * 
+ * 
+ */
+public class TestSkipBadRecords {
+  @Test (timeout=5000)
+  public void testSkipBadRecords() {
+    // test default values
+    Configuration conf = new Configuration();
+    assertEquals(2, SkipBadRecords.getAttemptsToStartSkipping(conf));
+    assertTrue(SkipBadRecords.getAutoIncrMapperProcCount(conf));
+    assertTrue(SkipBadRecords.getAutoIncrReducerProcCount(conf));
+    assertEquals(0, SkipBadRecords.getMapperMaxSkipRecords(conf));
+    assertEquals(0, SkipBadRecords.getReducerMaxSkipGroups(conf), 0);
+    assertNull(SkipBadRecords.getSkipOutputPath(conf));
+
+    // test setters
+    SkipBadRecords.setAttemptsToStartSkipping(conf, 5);
+    SkipBadRecords.setAutoIncrMapperProcCount(conf, false);
+    SkipBadRecords.setAutoIncrReducerProcCount(conf, false);
+    SkipBadRecords.setMapperMaxSkipRecords(conf, 6L);
+    SkipBadRecords.setReducerMaxSkipGroups(conf, 7L);
+    JobConf jc= new JobConf();
+    SkipBadRecords.setSkipOutputPath(jc, new Path("test"));
+    
+    // test getters 
+    assertEquals(5, SkipBadRecords.getAttemptsToStartSkipping(conf));
+    assertFalse(SkipBadRecords.getAutoIncrMapperProcCount(conf));
+    assertFalse(SkipBadRecords.getAutoIncrReducerProcCount(conf));
+    assertEquals(6L, SkipBadRecords.getMapperMaxSkipRecords(conf));
+    assertEquals(7L, SkipBadRecords.getReducerMaxSkipGroups(conf), 0);
+    assertEquals("test",SkipBadRecords.getSkipOutputPath(jc).toString());
+    
+  }
+
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLog.java

@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.mapred.TaskLog.LogName;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.junit.Test;
+
+import static junit.framework.Assert.*;
+import static org.mockito.Mockito.*;
+
+/**
+ * TestCounters checks the sanity and recoverability of Queue
+ */
+public class TestTaskLog {
+
+  /**
+   * test TaskAttemptID
+   * 
+   * @throws IOException
+   */
+  @Test (timeout=50000)
+  public void testTaskLog() throws IOException {
+    // test TaskLog
+    System.setProperty(MRJobConfig.TASK_LOG_DIR, "testString");
+    assertEquals(TaskLog.getMRv2LogDir(), "testString");
+    TaskAttemptID taid = mock(TaskAttemptID.class);
+    JobID jid = new JobID("job", 1);
+
+    when(taid.getJobID()).thenReturn(jid);
+    when(taid.toString()).thenReturn("JobId");
+
+    File f = TaskLog.getTaskLogFile(taid, true, LogName.STDOUT);
+    assertTrue(f.getAbsolutePath().endsWith("testString/stdout"));
+
+    // test getRealTaskLogFileLocation
+
+    File indexFile = TaskLog.getIndexFile(taid, true);
+    if (!indexFile.getParentFile().exists()) {
+      indexFile.getParentFile().mkdirs();
+    }
+    indexFile.delete();
+    indexFile.createNewFile();
+
+    TaskLog.syncLogs("location", taid, true);
+
+    assertTrue(indexFile.getAbsolutePath().endsWith(
+        "userlogs/job_job_0001/JobId.cleanup/log.index"));
+
+    f = TaskLog.getRealTaskLogFileLocation(taid, true, LogName.DEBUGOUT);
+    if (f != null) {
+      assertTrue(f.getAbsolutePath().endsWith("location/debugout"));
+      FileUtils.copyFile(indexFile, f);
+    }
+    // test obtainLogDirOwner
+    assertTrue(TaskLog.obtainLogDirOwner(taid).length() > 0);
+    // test TaskLog.Reader
+    assertTrue(readTaskLog(TaskLog.LogName.DEBUGOUT, taid, true).length() > 0);
+
+  }
+
+  public String readTaskLog(TaskLog.LogName filter,
+      org.apache.hadoop.mapred.TaskAttemptID taskId, boolean isCleanup)
+      throws IOException {
+    // string buffer to store task log
+    StringBuffer result = new StringBuffer();
+    int res;
+
+    // reads the whole tasklog into inputstream
+    InputStream taskLogReader = new TaskLog.Reader(taskId, filter, 0, -1,
+        isCleanup);
+    // construct string log from inputstream.
+    byte[] b = new byte[65536];
+    while (true) {
+      res = taskLogReader.read(b);
+      if (res > 0) {
+        result.append(new String(b));
+      } else {
+        break;
+      }
+    }
+    taskLogReader.close();
+
+    // trim the string and return it
+    String str = result.toString();
+    str = str.trim();
+    return str;
+  }
+
+  /**
+   * test without TASK_LOG_DIR
+   * 
+   * @throws IOException
+   */
+  @Test (timeout=50000)
+  public void testTaskLogWithoutTaskLogDir() throws IOException {
+    // TaskLog tasklog= new TaskLog();
+    System.clearProperty(MRJobConfig.TASK_LOG_DIR);
+
+    // test TaskLog
+
+    assertEquals(TaskLog.getMRv2LogDir(), null);
+    TaskAttemptID taid = mock(TaskAttemptID.class);
+    JobID jid = new JobID("job", 1);
+
+    when(taid.getJobID()).thenReturn(jid);
+    when(taid.toString()).thenReturn("JobId");
+
+    File f = TaskLog.getTaskLogFile(taid, true, LogName.STDOUT);
+    assertTrue(f.getAbsolutePath().endsWith("stdout"));
+
+  }
+
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestTaskLogAppender.java

@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.hadoop.mapred;
+
+import java.io.StringWriter;
+import java.io.Writer;
+
+import org.apache.log4j.Category;
+import org.apache.log4j.Layout;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PatternLayout;
+import org.apache.log4j.Priority;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestTaskLogAppender {
+/**
+ * test TaskLogAppender 
+ */
+  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
+  public void testTaskLogAppender(){
+    TaskLogAppender appender= new TaskLogAppender();
+    
+    System.setProperty(TaskLogAppender.TASKID_PROPERTY,"attempt_01_02_m03_04_001");
+    System.setProperty(TaskLogAppender.LOGSIZE_PROPERTY, "1003");
+    appender.activateOptions();
+    assertEquals(appender.getTaskId(), "attempt_01_02_m03_04_001");
+    assertEquals(appender.getTotalLogFileSize(),1000);
+    assertEquals(appender.getIsCleanup(),false);
+    
+    // test writer   
+    Writer writer= new StringWriter();
+    appender.setWriter(writer);
+    Layout layout =  new PatternLayout("%-5p [%t]: %m%n");
+    appender.setLayout(layout);
+    Category logger= Logger.getLogger(getClass().getName());
+    LoggingEvent event = new LoggingEvent("fqnOfCategoryClass", logger, Priority.INFO, "message", new Throwable());
+    appender.append(event);
+    appender.flush() ;
+    appender.close();
+    assertTrue(writer.toString().length()>0);
+    
+    // test cleanup should not changed 
+    appender= new TaskLogAppender();
+    appender.setIsCleanup(true);
+    appender.activateOptions();
+    assertEquals(appender.getIsCleanup(),true);
+
+  
+  }
+  
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/util/TestLinuxResourceCalculatorPlugin.java

@@ -1,236 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Random;
-
-import junit.framework.TestCase;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin;
-import org.junit.Test;
-
-/**
- * A JUnit test to test {@link LinuxResourceCalculatorPlugin}
- * Create the fake /proc/ information and verify the parsing and calculation
- */
-public class TestLinuxResourceCalculatorPlugin extends TestCase {
-  /**
-   * LinuxResourceCalculatorPlugin with a fake timer
-   */
-  static class FakeLinuxResourceCalculatorPlugin extends
-      LinuxResourceCalculatorPlugin {
-    
-	  long currentTime = 0;
-	  public FakeLinuxResourceCalculatorPlugin(String procfsMemFile,
-			                                       String procfsCpuFile,
-			                                       String procfsStatFile,
-			                                       long jiffyLengthInMillis) {
-	    super(procfsMemFile, procfsCpuFile, procfsStatFile, jiffyLengthInMillis);
-	  }
-	  @Override
-	  long getCurrentTime() {
-	    return currentTime;
-	  }
-	  public void advanceTime(long adv) {
-	    currentTime += adv * jiffyLengthInMillis;
-	  }
-  }
-  private static final FakeLinuxResourceCalculatorPlugin plugin;
-  private static String TEST_ROOT_DIR = new Path(System.getProperty(
-         "test.build.data", "/tmp")).toString().replace(' ', '+');
-  private static final String FAKE_MEMFILE;
-  private static final String FAKE_CPUFILE;
-  private static final String FAKE_STATFILE;
-  private static final long FAKE_JIFFY_LENGTH = 10L;
-  static {
-    int randomNum = (new Random()).nextInt(1000000000);
-    FAKE_MEMFILE = TEST_ROOT_DIR + File.separator + "MEMINFO_" + randomNum;
-    FAKE_CPUFILE = TEST_ROOT_DIR + File.separator + "CPUINFO_" + randomNum;
-    FAKE_STATFILE = TEST_ROOT_DIR + File.separator + "STATINFO_" + randomNum;
-    plugin = new FakeLinuxResourceCalculatorPlugin(FAKE_MEMFILE, FAKE_CPUFILE,
-                                                   FAKE_STATFILE,
-                                                   FAKE_JIFFY_LENGTH);
-  }
-  static final String MEMINFO_FORMAT = 
-	  "MemTotal:      %d kB\n" +
-	  "MemFree:         %d kB\n" +
-	  "Buffers:        138244 kB\n" +
-	  "Cached:         947780 kB\n" +
-	  "SwapCached:     142880 kB\n" +
-	  "Active:        3229888 kB\n" +
-	  "Inactive:       %d kB\n" +
-	  "SwapTotal:     %d kB\n" +
-	  "SwapFree:      %d kB\n" +
-	  "Dirty:          122012 kB\n" +
-	  "Writeback:           0 kB\n" +
-	  "AnonPages:     2710792 kB\n" +
-	  "Mapped:          24740 kB\n" +
-	  "Slab:           132528 kB\n" +
-	  "SReclaimable:   105096 kB\n" +
-	  "SUnreclaim:      27432 kB\n" +
-	  "PageTables:      11448 kB\n" +
-	  "NFS_Unstable:        0 kB\n" +
-	  "Bounce:              0 kB\n" +
-	  "CommitLimit:   4125904 kB\n" +
-	  "Committed_AS:  4143556 kB\n" +
-	  "VmallocTotal: 34359738367 kB\n" +
-	  "VmallocUsed:      1632 kB\n" +
-	  "VmallocChunk: 34359736375 kB\n" +
-	  "HugePages_Total:     0\n" +
-	  "HugePages_Free:      0\n" +
-	  "HugePages_Rsvd:      0\n" +
-	  "Hugepagesize:     2048 kB";
-  
-  static final String CPUINFO_FORMAT =
-    "processor : %s\n" +
-    "vendor_id : AuthenticAMD\n" +
-    "cpu family  : 15\n" +
-    "model   : 33\n" +
-    "model name  : Dual Core AMD Opteron(tm) Processor 280\n" +
-    "stepping  : 2\n" +
-    "cpu MHz   : %f\n" +
-    "cache size  : 1024 KB\n" +
-    "physical id : 0\n" +
-    "siblings  : 2\n" +
-    "core id   : 0\n" +
-    "cpu cores : 2\n" +
-    "fpu   : yes\n" +
-    "fpu_exception : yes\n" +
-    "cpuid level : 1\n" +
-    "wp    : yes\n" +
-    "flags   : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov " +
-    "pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt lm " +
-    "3dnowext 3dnow pni lahf_lm cmp_legacy\n" +
-    "bogomips  : 4792.41\n" +
-    "TLB size  : 1024 4K pages\n" +
-    "clflush size  : 64\n" +
-    "cache_alignment : 64\n" +
-    "address sizes : 40 bits physical, 48 bits virtual\n" +
-    "power management: ts fid vid ttp";
-  
-  static final String STAT_FILE_FORMAT = 
-    "cpu  %d %d %d 1646495089 831319 48713 164346 0\n" +
-    "cpu0 15096055 30805 3823005 411456015 206027 13 14269 0\n" +
-    "cpu1 14760561 89890 6432036 408707910 456857 48074 130857 0\n" +
-    "cpu2 12761169 20842 3758639 413976772 98028 411 10288 0\n" +
-    "cpu3 12355207 47322 5789691 412354390 70406 213 8931 0\n" +
-    "intr 114648668 20010764 2 0 945665 2 0 0 0 0 0 0 0 4 0 0 0 0 0 0\n" +
-    "ctxt 242017731764\n" +
-    "btime 1257808753\n" +
-    "processes 26414943\n" +
-    "procs_running 1\n" +
-    "procs_blocked 0\n";
-  
-  /**
-   * Test parsing /proc/stat and /proc/cpuinfo
-   * @throws IOException
-   */
-  @Test
-  public void testParsingProcStatAndCpuFile() throws IOException {
-    // Write fake /proc/cpuinfo file.
-    long numProcessors = 8;
-    long cpuFrequencyKHz = 2392781;
-    String fileContent = "";
-    for (int i = 0; i < numProcessors; i++) {
-      fileContent += String.format(CPUINFO_FORMAT, i, cpuFrequencyKHz / 1000D) +
-                     "\n";
-    }
-    File tempFile = new File(FAKE_CPUFILE);
-    tempFile.deleteOnExit();
-    FileWriter fWriter = new FileWriter(FAKE_CPUFILE);
-    fWriter.write(fileContent);
-    fWriter.close();
-    assertEquals(plugin.getNumProcessors(), numProcessors);
-    assertEquals(plugin.getCpuFrequency(), cpuFrequencyKHz);
-    
-    // Write fake /proc/stat file.
-    long uTime = 54972994;
-    long nTime = 188860;
-    long sTime = 19803373;
-    tempFile = new File(FAKE_STATFILE);
-    tempFile.deleteOnExit();
-    updateStatFile(uTime, nTime, sTime);
-    assertEquals(plugin.getCumulativeCpuTime(),
-                 FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
-    assertEquals(plugin.getCpuUsage(), (float)(LinuxResourceCalculatorPlugin.UNAVAILABLE));
-    
-    // Advance the time and sample again to test the CPU usage calculation
-    uTime += 100L;
-    plugin.advanceTime(200L);
-    updateStatFile(uTime, nTime, sTime);
-    assertEquals(plugin.getCumulativeCpuTime(),
-                 FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
-    assertEquals(plugin.getCpuUsage(), 6.25F);
-    
-    // Advance the time and sample again. This time, we call getCpuUsage() only.
-    uTime += 600L;
-    plugin.advanceTime(300L);
-    updateStatFile(uTime, nTime, sTime);
-    assertEquals(plugin.getCpuUsage(), 25F);
-    
-    // Advance very short period of time (one jiffy length).
-    // In this case, CPU usage should not be updated.
-    uTime += 1L;
-    plugin.advanceTime(1L);
-    updateStatFile(uTime, nTime, sTime);
-    assertEquals(plugin.getCumulativeCpuTime(),
-                 FAKE_JIFFY_LENGTH * (uTime + nTime + sTime));
-    assertEquals(plugin.getCpuUsage(), 25F); // CPU usage is not updated.
-  }
-  
-  /**
-   * Write information to fake /proc/stat file
-   */
-  private void updateStatFile(long uTime, long nTime, long sTime)
-    throws IOException {
-    FileWriter fWriter = new FileWriter(FAKE_STATFILE);
-    fWriter.write(String.format(STAT_FILE_FORMAT, uTime, nTime, sTime));
-    fWriter.close();
-  }
-  
-  /**
-   * Test parsing /proc/meminfo
-   * @throws IOException
-   */
-  @Test
-  public void testParsingProcMemFile() throws IOException {
-    long memTotal = 4058864L;
-    long memFree = 99632L;
-    long inactive = 567732L;
-    long swapTotal = 2096472L;
-    long swapFree = 1818480L;
-    File tempFile = new File(FAKE_MEMFILE);
-    tempFile.deleteOnExit();
-    FileWriter fWriter = new FileWriter(FAKE_MEMFILE);
-    fWriter.write(String.format(MEMINFO_FORMAT,
-      memTotal, memFree, inactive, swapTotal, swapFree));
-    
-    fWriter.close();
-    assertEquals(plugin.getAvailablePhysicalMemorySize(),
-                 1024L * (memFree + inactive));
-    assertEquals(plugin.getAvailableVirtualMemorySize(),
-                 1024L * (memFree + inactive + swapFree));
-    assertEquals(plugin.getPhysicalMemorySize(), 1024L * memTotal);
-    assertEquals(plugin.getVirtualMemorySize(), 1024L * (memTotal + swapTotal));
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/resources/mapred-queues.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<!-- This is the template for queue configuration. The format supports nesting of
+     queues within queues - a feature called hierarchical queues. All queues are
+     defined within the 'queues' tag which is the top level element for this
+     XML document.
+     The 'aclsEnabled' attribute should be set to true, if ACLs should be checked
+     on queue operations such as submitting jobs, killing jobs etc. -->
+<queues aclsEnabled="false">
+
+  <!-- Configuration for a queue is specified by defining a 'queue' element. -->
+  <queue>
+
+    <!-- Name of a queue. Queue name cannot contain a ':'  -->
+    <name>default</name>
+
+    <!-- properties for a queue, typically used by schedulers,
+    can be defined here -->
+    <properties>
+    </properties>
+
+	<!-- State of the queue. If running, the queue will accept new jobs.
+         If stopped, the queue will not accept new jobs. -->
+    <state>running</state>
+
+    <!-- Specifies the ACLs to check for submitting jobs to this queue.
+         If set to '*', it allows all users to submit jobs to the queue.
+         For specifying a list of users and groups the format to use is
+         user1,user2 group1,group2 -->
+    <acl-submit-job>*</acl-submit-job>
+
+    <!-- Specifies the ACLs to check for modifying jobs in this queue.
+         Modifications include killing jobs, tasks of jobs or changing
+         priorities.
+         If set to '*', it allows all users to submit jobs to the queue.
+         For specifying a list of users and groups the format to use is
+         user1,user2 group1,group2 -->
+    <acl-administer-jobs>*</acl-administer-jobs>
+  </queue>
+
+  <!-- Here is a sample of a hierarchical queue configuration
+       where q2 is a child of q1. In this example, q2 is a leaf level
+       queue as it has no queues configured within it. Currently, ACLs
+       and state are only supported for the leaf level queues.
+       Note also the usage of properties for the queue q2. -->
+  <queue>
+    <name>q1</name>
+    <queue>
+      <name>q2</name>
+      <properties>
+        <property key="capacity" value="20"/>
+        <property key="user-limit" value="30"/>
+      </properties>
+    </queue>
+  </queue>
+ 
+</queues>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java

@@ -481,6 +481,9 @@ public class YARNRunner implements ClientProtocol {
     appContext.setCancelTokensWhenComplete(
         conf.getBoolean(MRJobConfig.JOB_CANCEL_DELEGATION_TOKEN, true));
     appContext.setAMContainerSpec(amContainer);         // AM Container
+    appContext.setMaxAppAttempts(
+        conf.getInt(MRJobConfig.MR_AM_MAX_ATTEMPTS,
+            MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS));
 
     return appContext;
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFile.java

@@ -26,12 +26,12 @@ import org.apache.hadoop.io.compress.DefaultCodec;
 import org.apache.hadoop.io.compress.GzipCodec;
 
 import org.junit.Test;
-
+import static org.junit.Assert.*;
 public class TestIFile {
 
   @Test
   /**
-   * Create an IFile.Writer using GzipCodec since this codec does not
+   * Create an IFile.Writer using GzipCodec since this code does not
    * have a compressor when run via the tests (ie no native libraries).
    */
   public void testIFileWriterWithCodec() throws Exception {
@@ -63,5 +63,11 @@ public class TestIFile {
     IFile.Reader<Text, Text> reader =
       new IFile.Reader<Text, Text>(conf, rfs, path, codec, null);
     reader.close();
+    
+    // test check sum 
+    byte[] ab= new byte[100];
+    int readed= reader.checksumIn.readWithChecksum(ab, 0, ab.length);
+    assertEquals( readed,reader.checksumIn.getChecksum().length);
+    
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMultiFileSplit.java

@@ -21,13 +21,20 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
 import java.util.Arrays;
 
 import junit.framework.TestCase;
 
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
-
+/**
+ * 
+ * test MultiFileSplit class
+ */
 public class TestMultiFileSplit extends TestCase{
 
     public void testReadWrite() throws Exception {
@@ -58,4 +65,26 @@ public class TestMultiFileSplit extends TestCase{
       assertTrue(Arrays.equals(split.getLengths(), readSplit.getLengths()));
       System.out.println(split.toString());
     }
+    
+    /**
+     * test method getLocations
+     * @throws IOException
+     */
+    public void testgetLocations() throws IOException{
+        JobConf job= new JobConf();
+      
+      File tmpFile = File.createTempFile("test","txt");
+      tmpFile.createNewFile();
+      OutputStream out=new FileOutputStream(tmpFile);
+      out.write("tempfile".getBytes());
+      out.flush();
+      out.close();
+      Path[] path= {new Path(tmpFile.getAbsolutePath())};
+      long[] lengths = {100};
+      
+      MultiFileSplit  split = new MultiFileSplit(job,path,lengths);
+     String [] locations= split.getLocations();
+     assertTrue(locations.length==1);
+     assertEquals(locations[0], "localhost");
+    }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java

@@ -18,24 +18,37 @@
 
 package org.apache.hadoop.mapred;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
-import static org.junit.Assert.assertNull;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.PrintStream;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.ClusterStatus.BlackListInfo;
+import org.apache.hadoop.mapred.JobClient.NetworkedJob;
+import org.apache.hadoop.mapred.JobClient.TaskStatusFilter;
 import org.apache.hadoop.mapred.lib.IdentityMapper;
 import org.apache.hadoop.mapred.lib.IdentityReducer;
+import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.yarn.YarnException;
 import org.junit.Test;
-
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
 
 public class TestNetworkedJob {
   private static String TEST_ROOT_DIR = new File(System.getProperty(
@@ -44,8 +57,7 @@ public class TestNetworkedJob {
   private static Path inFile = new Path(testDir, "in");
   private static Path outDir = new Path(testDir, "out");
 
-  @SuppressWarnings("deprecation")
+  @Test (timeout=5000)
-  @Test
   public void testGetNullCounters() throws Exception {
     //mock creation
     Job mockJob = mock(Job.class);
@@ -57,7 +69,7 @@ public class TestNetworkedJob {
     verify(mockJob).getCounters();
   }
   
-  @Test
+  @Test (timeout=500000)
   public void testGetJobStatus() throws IOException, InterruptedException,
       ClassNotFoundException {
     MiniMRClientCluster mr = null;
@@ -105,4 +117,278 @@ public class TestNetworkedJob {
       }
     }
   }
+/**
+ * test JobConf 
+ * @throws Exception
+ */
+  @SuppressWarnings( "deprecation" )
+  @Test (timeout=500000)
+  public void testNetworkedJob() throws Exception {
+    // mock creation
+    MiniMRClientCluster mr = null;
+    FileSystem fileSys = null;
+
+    try {
+      Configuration conf = new Configuration();
+      mr = MiniMRClientClusterFactory.create(this.getClass(), 2, conf);
+
+      JobConf job = new JobConf(mr.getConfig());
+
+      fileSys = FileSystem.get(job);
+      fileSys.delete(testDir, true);
+      FSDataOutputStream out = fileSys.create(inFile, true);
+      out.writeBytes("This is a test file");
+      out.close();
+
+      FileInputFormat.setInputPaths(job, inFile);
+      FileOutputFormat.setOutputPath(job, outDir);
+
+      job.setInputFormat(TextInputFormat.class);
+      job.setOutputFormat(TextOutputFormat.class);
+
+      job.setMapperClass(IdentityMapper.class);
+      job.setReducerClass(IdentityReducer.class);
+      job.setNumReduceTasks(0);
+
+      JobClient client = new JobClient(mr.getConfig());
+
+      RunningJob rj = client.submitJob(job);
+      JobID jobId = rj.getID();
+      NetworkedJob runningJob = (NetworkedJob) client.getJob(jobId);
+      runningJob.setJobPriority(JobPriority.HIGH.name());
+      // test getters
+      assertTrue(runningJob.getConfiguration().toString()
+          .endsWith("0001/job.xml"));
+      assertEquals(runningJob.getID(), jobId);
+      assertEquals(runningJob.getJobID(), jobId.toString());
+      assertEquals(runningJob.getJobName(), "N/A");
+      assertTrue(runningJob.getJobFile().endsWith(
+          ".staging/" + runningJob.getJobID() + "/job.xml"));
+      assertTrue(runningJob.getTrackingURL().length() > 0);
+      assertTrue(runningJob.mapProgress() == 0.0f);
+      assertTrue(runningJob.reduceProgress() == 0.0f);
+      assertTrue(runningJob.cleanupProgress() == 0.0f);
+      assertTrue(runningJob.setupProgress() == 0.0f);
+
+      TaskCompletionEvent[] tce = runningJob.getTaskCompletionEvents(0);
+      assertEquals(tce.length, 0);
+
+      assertEquals(runningJob.getHistoryUrl(),"");
+      assertFalse(runningJob.isRetired());
+      assertEquals( runningJob.getFailureInfo(),"");
+      assertEquals(runningJob.getJobStatus().getJobName(), "N/A");
+      assertEquals(client.getMapTaskReports(jobId).length, 0);
+      
+      try {
+        client.getSetupTaskReports(jobId);
+      } catch (YarnException e) {
+        assertEquals(e.getMessage(), "Unrecognized task type: JOB_SETUP");
+      }
+      try {
+        client.getCleanupTaskReports(jobId);
+      } catch (YarnException e) {
+        assertEquals(e.getMessage(), "Unrecognized task type: JOB_CLEANUP");
+      }
+      assertEquals(client.getReduceTaskReports(jobId).length, 0);
+      // test ClusterStatus
+      ClusterStatus status = client.getClusterStatus(true);
+      assertEquals(status.getActiveTrackerNames().size(), 2);
+      // it method does not implemented and always return empty array or null;
+      assertEquals(status.getBlacklistedTrackers(), 0);
+      assertEquals(status.getBlacklistedTrackerNames().size(), 0);
+      assertEquals(status.getBlackListedTrackersInfo().size(), 0);
+      assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING);
+      assertEquals(status.getMapTasks(), 1);
+      assertEquals(status.getMaxMapTasks(), 20);
+      assertEquals(status.getMaxReduceTasks(), 4);
+      assertEquals(status.getNumExcludedNodes(), 0);
+      assertEquals(status.getReduceTasks(), 1);
+      assertEquals(status.getTaskTrackers(), 2);
+      assertEquals(status.getTTExpiryInterval(), 0);
+      assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING);
+
+      // test read and write
+      ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
+      status.write(new DataOutputStream(dataOut));
+      ClusterStatus status2 = new ClusterStatus();
+
+      status2.readFields(new DataInputStream(new ByteArrayInputStream(dataOut
+          .toByteArray())));
+      assertEquals(status.getActiveTrackerNames(),
+          status2.getActiveTrackerNames());
+      assertEquals(status.getBlackListedTrackersInfo(),
+          status2.getBlackListedTrackersInfo());
+      assertEquals(status.getMapTasks(), status2.getMapTasks());
+
+      try {
+      } catch (RuntimeException e) {
+        assertTrue(e.getMessage().endsWith("not found on CLASSPATH"));
+      }
+
+      // test taskStatusfilter
+      JobClient.setTaskOutputFilter(job, TaskStatusFilter.ALL);
+      assertEquals(JobClient.getTaskOutputFilter(job), TaskStatusFilter.ALL);
+
+      // runningJob.setJobPriority(JobPriority.HIGH.name());
+
+      // test default map
+      assertEquals(client.getDefaultMaps(), 20);
+      assertEquals(client.getDefaultReduces(), 4);
+      assertEquals(client.getSystemDir().getName(), "jobSubmitDir");
+      // test queue information
+      JobQueueInfo[] rootQueueInfo = client.getRootQueues();
+      assertEquals(rootQueueInfo.length, 1);
+      assertEquals(rootQueueInfo[0].getQueueName(), "default");
+      JobQueueInfo[] qinfo = client.getQueues();
+      assertEquals(qinfo.length, 1);
+      assertEquals(qinfo[0].getQueueName(), "default");
+      assertEquals(client.getChildQueues("default").length, 0);
+      assertEquals(client.getJobsFromQueue("default").length, 1);
+      assertTrue(client.getJobsFromQueue("default")[0].getJobFile().endsWith(
+          "/job.xml"));
+
+      JobQueueInfo qi = client.getQueueInfo("default");
+      assertEquals(qi.getQueueName(), "default");
+      assertEquals(qi.getQueueState(), "running");
+
+      QueueAclsInfo[] aai = client.getQueueAclsForCurrentUser();
+      assertEquals(aai.length, 2);
+      assertEquals(aai[0].getQueueName(), "root");
+      assertEquals(aai[1].getQueueName(), "default");
+      // test token
+      Token<DelegationTokenIdentifier> token = client
+          .getDelegationToken(new Text(UserGroupInformation.getCurrentUser()
+              .getShortUserName()));
+      assertEquals(token.getKind().toString(), "RM_DELEGATION_TOKEN");
+      
+      // test JobClient
+      
+   
+      // The following asserts read JobStatus twice and ensure the returned
+      // JobStatus objects correspond to the same Job.
+      assertEquals("Expected matching JobIDs", jobId, client.getJob(jobId)
+          .getJobStatus().getJobID());
+      assertEquals("Expected matching startTimes", rj.getJobStatus()
+          .getStartTime(), client.getJob(jobId).getJobStatus().getStartTime());
+    } finally {
+      if (fileSys != null) {
+        fileSys.delete(testDir, true);
+      }
+      if (mr != null) {
+        mr.stop();
+      }
+    }
+  }
+
+  /**
+   * test BlackListInfo class
+   * 
+   * @throws IOException
+   */
+  @Test (timeout=5000)
+  public void testBlackListInfo() throws IOException {
+    BlackListInfo info = new BlackListInfo();
+    info.setBlackListReport("blackListInfo");
+    info.setReasonForBlackListing("reasonForBlackListing");
+    info.setTrackerName("trackerName");
+    ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
+    DataOutput out = new DataOutputStream(byteOut);
+    info.write(out);
+    BlackListInfo info2 = new BlackListInfo();
+    info2.readFields(new DataInputStream(new ByteArrayInputStream(byteOut
+        .toByteArray())));
+    assertEquals(info, info);
+    assertEquals(info.toString(), info.toString());
+    assertEquals(info.getTrackerName(), "trackerName");
+    assertEquals(info.getReasonForBlackListing(), "reasonForBlackListing");
+    assertEquals(info.getBlackListReport(), "blackListInfo");
+
+  }
+/**
+ *  test run from command line JobQueueClient
+ * @throws Exception
+ */
+  @Test (timeout=500000)
+  public void testJobQueueClient() throws Exception {
+        MiniMRClientCluster mr = null;
+    FileSystem fileSys = null;
+    PrintStream oldOut = System.out;
+    try {
+      Configuration conf = new Configuration();
+      mr = MiniMRClientClusterFactory.create(this.getClass(), 2, conf);
+
+      JobConf job = new JobConf(mr.getConfig());
+
+      fileSys = FileSystem.get(job);
+      fileSys.delete(testDir, true);
+      FSDataOutputStream out = fileSys.create(inFile, true);
+      out.writeBytes("This is a test file");
+      out.close();
+
+      FileInputFormat.setInputPaths(job, inFile);
+      FileOutputFormat.setOutputPath(job, outDir);
+
+      job.setInputFormat(TextInputFormat.class);
+      job.setOutputFormat(TextOutputFormat.class);
+
+      job.setMapperClass(IdentityMapper.class);
+      job.setReducerClass(IdentityReducer.class);
+      job.setNumReduceTasks(0);
+
+      JobClient client = new JobClient(mr.getConfig());
+
+      client.submitJob(job);
+
+      JobQueueClient jobClient = new JobQueueClient(job);
+
+      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+      System.setOut(new PrintStream(bytes));
+      String[] arg = { "-list" };
+      jobClient.run(arg);
+      assertTrue(bytes.toString().contains("Queue Name : default"));
+      assertTrue(bytes.toString().contains("Queue State : running"));
+      bytes = new ByteArrayOutputStream();
+      System.setOut(new PrintStream(bytes));
+      String[] arg1 = { "-showacls" };
+      jobClient.run(arg1);
+      assertTrue(bytes.toString().contains("Queue acls for user :"));
+      assertTrue(bytes.toString().contains(
+          "root  ADMINISTER_QUEUE,SUBMIT_APPLICATIONS"));
+      assertTrue(bytes.toString().contains(
+          "default  ADMINISTER_QUEUE,SUBMIT_APPLICATIONS"));
+
+      // test for info and default queue
+
+      bytes = new ByteArrayOutputStream();
+      System.setOut(new PrintStream(bytes));
+      String[] arg2 = { "-info", "default" };
+      jobClient.run(arg2);
+      assertTrue(bytes.toString().contains("Queue Name : default"));
+      assertTrue(bytes.toString().contains("Queue State : running"));
+      assertTrue(bytes.toString().contains("Scheduling Info"));
+
+      // test for info , default queue and jobs
+      bytes = new ByteArrayOutputStream();
+      System.setOut(new PrintStream(bytes));
+      String[] arg3 = { "-info", "default", "-showJobs" };
+      jobClient.run(arg3);
+      assertTrue(bytes.toString().contains("Queue Name : default"));
+      assertTrue(bytes.toString().contains("Queue State : running"));
+      assertTrue(bytes.toString().contains("Scheduling Info"));
+      assertTrue(bytes.toString().contains("job_1"));
+
+      String[] arg4 = {};
+      jobClient.run(arg4);
+
+      
+    } finally {
+      System.setOut(oldOut);
+      if (fileSys != null) {
+        fileSys.delete(testDir, true);
+      }
+      if (mr != null) {
+        mr.stop();
+      }
+    }
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestQueueConfigurationParser.java

@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.StringWriter;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+
+public class TestQueueConfigurationParser {
+/**
+ * test xml generation 
+ * @throws ParserConfigurationException
+ * @throws Exception 
+ */
+  @Test (timeout=5000)
+  public void testQueueConfigurationParser()
+      throws ParserConfigurationException, Exception {
+    JobQueueInfo info = new JobQueueInfo("root", "rootInfo");
+    JobQueueInfo infoChild1 = new JobQueueInfo("child1", "child1Info");
+    JobQueueInfo infoChild2 = new JobQueueInfo("child2", "child1Info");
+
+    info.addChild(infoChild1);
+    info.addChild(infoChild2);
+    DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
+        .newInstance();
+    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
+    
+    
+    Document document = builder.newDocument();
+    
+
+    // test QueueConfigurationParser.getQueueElement 
+    Element e = QueueConfigurationParser.getQueueElement(document, info);
+    // transform result to string for check
+    DOMSource domSource = new DOMSource(e);
+    StringWriter writer = new StringWriter();
+    StreamResult result = new StreamResult(writer);
+    TransformerFactory tf = TransformerFactory.newInstance();
+    Transformer transformer = tf.newTransformer();
+    transformer.transform(domSource, result);
+    String str= writer.toString();
+    assertTrue(str
+        .endsWith("<queue><name>root</name><properties/><state>running</state><queue><name>child1</name><properties/><state>running</state></queue><queue><name>child2</name><properties/><state>running</state></queue></queue>"));
+  }
+}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestStatisticsCollector.java

@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.mapred;
 
+import java.util.Map;
+
 import junit.framework.TestCase;
 
 import org.apache.hadoop.mapred.StatisticsCollector.TimeWindow;
@@ -24,6 +26,7 @@ import org.apache.hadoop.mapred.StatisticsCollector.Stat;
 
 public class TestStatisticsCollector extends TestCase{
 
+  @SuppressWarnings("rawtypes")
   public void testMovingWindow() throws Exception {
     StatisticsCollector collector = new StatisticsCollector(1);
     TimeWindow window = new TimeWindow("test", 6, 2);
@@ -78,6 +81,28 @@ public class TestStatisticsCollector extends TestCase{
     collector.update();
     assertEquals((10+10+10+12+13+14), stat.getValues().get(window).getValue());
     assertEquals(95, stat.getValues().get(sincStart).getValue());
+    
+    //  test Stat class 
+    Map updaters= collector.getUpdaters();
+    assertEquals(updaters.size(),2);
+    Map<String, Stat> ststistics=collector.getStatistics();
+    assertNotNull(ststistics.get("m1"));
+    
+   Stat newStat= collector.createStat("m2"); 
+    assertEquals(newStat.name, "m2");
+    Stat st=collector.removeStat("m1");
+    assertEquals(st.name, "m1");
+    assertEquals((10+10+10+12+13+14), stat.getValues().get(window).getValue());
+    assertEquals(95, stat.getValues().get(sincStart).getValue());
+     st=collector.removeStat("m1");
+     // try to remove stat again
+    assertNull(st);
+    collector.start();
+    // waiting 2,5 sec
+    Thread.sleep(2500);
+    assertEquals(69, stat.getValues().get(window).getValue());
+    assertEquals(95, stat.getValues().get(sincStart).getValue());
+  
   }
 
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextInputFormat.java

@@ -61,11 +61,12 @@ public class TestTextInputFormat {
       throw new RuntimeException("init failure", e);
     }
   }
+  @SuppressWarnings("deprecation")
   private static Path workDir =
     new Path(new Path(System.getProperty("test.build.data", "/tmp")),
              "TestTextInputFormat").makeQualified(localFs);
 
-  @Test
+  @Test (timeout=500000)
   public void testFormat() throws Exception {
     JobConf job = new JobConf(defaultConf);
     Path file = new Path(workDir, "test.txt");
@@ -145,7 +146,7 @@ public class TestTextInputFormat {
     }
   }
 
-  @Test
+  @Test (timeout=900000)
   public void testSplitableCodecs() throws IOException {
     JobConf conf = new JobConf(defaultConf);
     int seed = new Random().nextInt();
@@ -250,7 +251,7 @@ public class TestTextInputFormat {
                                            bufsz);
   }
 
-  @Test
+  @Test (timeout=5000)
   public void testUTF8() throws Exception {
     LineReader in = makeStream("abcd\u20acbdcd\u20ac");
     Text line = new Text();
@@ -269,7 +270,7 @@ public class TestTextInputFormat {
    *
    * @throws Exception
    */
-  @Test
+  @Test (timeout=5000)
   public void testNewLines() throws Exception {
     final String STR = "a\nbb\n\nccc\rdddd\r\r\r\n\r\neeeee";
     final int STRLENBYTES = STR.getBytes().length;
@@ -309,7 +310,7 @@ public class TestTextInputFormat {
    *
    * @throws Exception
    */
-  @Test
+  @Test (timeout=5000)
   public void testMaxLineLength() throws Exception {
     final String STR = "a\nbb\n\nccc\rdddd\r\neeeee";
     final int STRLENBYTES = STR.getBytes().length;
@@ -334,7 +335,7 @@ public class TestTextInputFormat {
     }
   }
 
-  @Test
+  @Test (timeout=5000)
   public void testMRMaxLine() throws Exception {
     final int MAXPOS = 1024 * 1024;
     final int MAXLINE = 10 * 1024;
@@ -354,6 +355,9 @@ public class TestTextInputFormat {
         position += b.length;
         return b.length;
       }
+      public void reset() {
+        position=0;
+      }
     };
     final LongWritable key = new LongWritable();
     final Text val = new Text();
@@ -362,8 +366,14 @@ public class TestTextInputFormat {
     conf.setInt(org.apache.hadoop.mapreduce.lib.input.
                 LineRecordReader.MAX_LINE_LENGTH, MAXLINE);
     conf.setInt("io.file.buffer.size", BUF); // used by LRR
-    final LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf);
+    // test another constructor 
+     LineRecordReader lrr = new LineRecordReader(infNull, 0, MAXPOS, conf);
     assertFalse("Read a line from null", lrr.next(key, val));
+    infNull.reset();
+     lrr = new LineRecordReader(infNull, 0L, MAXLINE, MAXPOS);
+    assertFalse("Read a line from null", lrr.next(key, val));
+    
+    
   }
 
   private static void writeFile(FileSystem fs, Path name, 
@@ -400,7 +410,7 @@ public class TestTextInputFormat {
   /**
    * Test using the gzip codec for reading
    */
-  @Test
+  @Test (timeout=5000)
   public void testGzip() throws IOException {
     JobConf job = new JobConf(defaultConf);
     CompressionCodec gzip = new GzipCodec();
@@ -434,7 +444,7 @@ public class TestTextInputFormat {
   /**
    * Test using the gzip codec and an empty input file
    */
-  @Test
+  @Test (timeout=5000)
   public void testGzipEmpty() throws IOException {
     JobConf job = new JobConf(defaultConf);
     CompressionCodec gzip = new GzipCodec();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestTextOutputFormat.java

@@ -44,7 +44,6 @@ public class TestTextOutputFormat extends TestCase {
                                "data"), 
                       FileOutputCommitter.TEMP_DIR_NAME), "_" + attempt);
 
-  @SuppressWarnings("unchecked")
   public void testFormat() throws Exception {
     JobConf job = new JobConf();
     job.set(JobContext.TASK_ATTEMPT_ID, attempt);
@@ -59,8 +58,8 @@ public class TestTextOutputFormat extends TestCase {
     // A reporter that does nothing
     Reporter reporter = Reporter.NULL;
 
-    TextOutputFormat theOutputFormat = new TextOutputFormat();
+    TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>();
-    RecordWriter theRecordWriter =
+    RecordWriter<Object,Object> theRecordWriter =
       theOutputFormat.getRecordWriter(localFs, job, file, reporter);
 
     Text key1 = new Text("key1");
@@ -95,7 +94,6 @@ public class TestTextOutputFormat extends TestCase {
 
   }
 
-  @SuppressWarnings("unchecked")
   public void testFormatWithCustomSeparator() throws Exception {
     JobConf job = new JobConf();
     String separator = "\u0001";
@@ -112,8 +110,8 @@ public class TestTextOutputFormat extends TestCase {
     // A reporter that does nothing
     Reporter reporter = Reporter.NULL;
 
-    TextOutputFormat theOutputFormat = new TextOutputFormat();
+    TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>();
-    RecordWriter theRecordWriter =
+    RecordWriter<Object,Object> theRecordWriter =
       theOutputFormat.getRecordWriter(localFs, job, file, reporter);
 
     Text key1 = new Text("key1");
@@ -147,7 +145,61 @@ public class TestTextOutputFormat extends TestCase {
     assertEquals(output, expectedOutput.toString());
 
   }
+  /**
+   * test compressed file
+   * @throws IOException
+   */
+ public void testCompress() throws IOException{
+   JobConf job = new JobConf();
+   String separator = "\u0001";
+   job.set("mapreduce.output.textoutputformat.separator", separator);
+   job.set(JobContext.TASK_ATTEMPT_ID, attempt);
+   job.set(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.COMPRESS,"true");
+   
+   FileOutputFormat.setOutputPath(job, workDir.getParent().getParent());
+   FileOutputFormat.setWorkOutputPath(job, workDir);
+   FileSystem fs = workDir.getFileSystem(job);
+   if (!fs.mkdirs(workDir)) {
+     fail("Failed to create output directory");
+   }
+   String file = "test.txt";
 
+   // A reporter that does nothing
+   Reporter reporter = Reporter.NULL;
+
+   TextOutputFormat<Object,Object> theOutputFormat = new TextOutputFormat<Object,Object>();
+   RecordWriter<Object,Object> theRecordWriter =
+     theOutputFormat.getRecordWriter(localFs, job, file, reporter);
+   Text key1 = new Text("key1");
+   Text key2 = new Text("key2");
+   Text val1 = new Text("val1");
+   Text val2 = new Text("val2");
+   NullWritable nullWritable = NullWritable.get();
+
+   try {
+     theRecordWriter.write(key1, val1);
+     theRecordWriter.write(null, nullWritable);
+     theRecordWriter.write(null, val1);
+     theRecordWriter.write(nullWritable, val2);
+     theRecordWriter.write(key2, nullWritable);
+     theRecordWriter.write(key1, null);
+     theRecordWriter.write(null, null);
+     theRecordWriter.write(key2, val2);
+
+   } finally {
+     theRecordWriter.close(reporter);
+   }
+   File expectedFile = new File(new Path(workDir, file).toString());
+   StringBuffer expectedOutput = new StringBuffer();
+   expectedOutput.append(key1).append(separator).append(val1).append("\n");
+   expectedOutput.append(val1).append("\n");
+   expectedOutput.append(val2).append("\n");
+   expectedOutput.append(key2).append("\n");
+   expectedOutput.append(key1).append("\n");
+   expectedOutput.append(key2).append(separator).append(val2).append("\n");
+   String output = UtilsForTests.slurp(expectedFile);
+   assertEquals(output, expectedOutput.toString());
+ }
   public static void main(String[] args) throws Exception {
     new TestTextOutputFormat().testFormat();
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/LinuxMemoryCalculatorPlugin.java

@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * Plugin to calculate virtual and physical memories on Linux systems.
- * @deprecated 
- * Use {@link org.apache.hadoop.mapreduce.util.LinuxResourceCalculatorPlugin}
- * instead
- */
-@Deprecated
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class LinuxMemoryCalculatorPlugin extends MemoryCalculatorPlugin {
-  private LinuxResourceCalculatorPlugin resourceCalculatorPlugin;
-  // Use everything from LinuxResourceCalculatorPlugin
-  public LinuxMemoryCalculatorPlugin() {
-    resourceCalculatorPlugin = new LinuxResourceCalculatorPlugin();
-  }
-  
-  /** {@inheritDoc} */
-  @Override
-  public long getPhysicalMemorySize() {
-    return resourceCalculatorPlugin.getPhysicalMemorySize();
-  }
-  
-  /** {@inheritDoc} */
-  @Override
-  public long getVirtualMemorySize() {
-    return resourceCalculatorPlugin.getVirtualMemorySize();
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/MemoryCalculatorPlugin.java

@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.util.ReflectionUtils;
-
-/**
- * Plugin to calculate virtual and physical memories on the system.
- * @deprecated Use
- *             {@link org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin}
- *             instead
- */
-@Deprecated
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public abstract class MemoryCalculatorPlugin extends Configured {
-
-  /**
-   * Obtain the total size of the virtual memory present in the system.
-   * 
-   * @return virtual memory size in bytes.
-   */
-  public abstract long getVirtualMemorySize();
-
-  /**
-   * Obtain the total size of the physical memory present in the system.
-   * 
-   * @return physical memory size bytes.
-   */
-  public abstract long getPhysicalMemorySize();
-
-  /**
-   * Get the MemoryCalculatorPlugin from the class name and configure it. If
-   * class name is null, this method will try and return a memory calculator
-   * plugin available for this system.
-   * 
-   * @param clazz class-name
-   * @param conf configure the plugin with this.
-   * @return MemoryCalculatorPlugin
-   */
-  public static MemoryCalculatorPlugin getMemoryCalculatorPlugin(
-      Class<? extends MemoryCalculatorPlugin> clazz, Configuration conf) {
-
-    if (clazz != null) {
-      return ReflectionUtils.newInstance(clazz, conf);
-    }
-
-    // No class given, try a os specific class
-    try {
-      String osName = System.getProperty("os.name");
-      if (osName.startsWith("Linux")) {
-        return new LinuxMemoryCalculatorPlugin();
-      }
-    } catch (SecurityException se) {
-      // Failed to get Operating System name.
-      return null;
-    }
-
-    // Not supported on this system.
-    return null;
-  }
-}

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/util/TestProcfsBasedProcessTree.java

@@ -1,677 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapreduce.util;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Random;
-import java.util.Vector;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Shell.ExitCodeException;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-
-import junit.framework.TestCase;
-
-/**
- * A JUnit test to test ProcfsBasedProcessTree.
- */
-public class TestProcfsBasedProcessTree extends TestCase {
-
-  private static final Log LOG = LogFactory
-      .getLog(TestProcfsBasedProcessTree.class);
-  private static String TEST_ROOT_DIR = new Path(System.getProperty(
-         "test.build.data", "/tmp")).toString().replace(' ', '+');
-
-  private ShellCommandExecutor shexec = null;
-  private String pidFile, lowestDescendant;
-  private String shellScript;
-  private static final int N = 6; // Controls the RogueTask
-
-  private class RogueTaskThread extends Thread {
-    public void run() {
-      try {
-        Vector<String> args = new Vector<String>();
-        if(ProcessTree.isSetsidAvailable) {
-          args.add("setsid");
-        }
-        args.add("bash");
-        args.add("-c");
-        args.add(" echo $$ > " + pidFile + "; sh " +
-                          shellScript + " " + N + ";") ;
-        shexec = new ShellCommandExecutor(args.toArray(new String[0]));
-        shexec.execute();
-      } catch (ExitCodeException ee) {
-        LOG.info("Shell Command exit with a non-zero exit code. This is" +
-                 " expected as we are killing the subprocesses of the" +
-                 " task intentionally. " + ee);
-      } catch (IOException ioe) {
-        LOG.info("Error executing shell command " + ioe);
-      } finally {
-        LOG.info("Exit code: " + shexec.getExitCode());
-      }
-    }
-  }
-
-  private String getRogueTaskPID() {
-    File f = new File(pidFile);
-    while (!f.exists()) {
-      try {
-        Thread.sleep(500);
-      } catch (InterruptedException ie) {
-        break;
-      }
-    }
-
-    // read from pidFile
-    return getPidFromPidFile(pidFile);
-  }
-
-  public void testProcessTree() {
-
-    try {
-      if (!ProcfsBasedProcessTree.isAvailable()) {
-        System.out
-            .println("ProcfsBasedProcessTree is not available on this system. Not testing");
-        return;
-      }
-    } catch (Exception e) {
-      LOG.info(StringUtils.stringifyException(e));
-      return;
-    }
-    // create shell script
-    Random rm = new Random();
-    File tempFile = new File(TEST_ROOT_DIR, this.getName() + "_shellScript_" +
-                             rm.nextInt() + ".sh");
-    tempFile.deleteOnExit();
-    shellScript = TEST_ROOT_DIR + File.separator + tempFile.getName();
-
-    // create pid file
-    tempFile = new File(TEST_ROOT_DIR,  this.getName() + "_pidFile_" +
-                        rm.nextInt() + ".pid");
-    tempFile.deleteOnExit();
-    pidFile = TEST_ROOT_DIR + File.separator + tempFile.getName();
-
-    lowestDescendant = TEST_ROOT_DIR + File.separator + "lowestDescendantPidFile";
-
-    // write to shell-script
-    try {
-      FileWriter fWriter = new FileWriter(shellScript);
-      fWriter.write(
-          "# rogue task\n" +
-          "sleep 1\n" +
-          "echo hello\n" +
-          "if [ $1 -ne 0 ]\n" +
-          "then\n" +
-          " sh " + shellScript + " $(($1-1))\n" +
-          "else\n" +
-          " echo $$ > " + lowestDescendant + "\n" +
-          " while true\n do\n" +
-          "  sleep 5\n" +
-          " done\n" +
-          "fi");
-      fWriter.close();
-    } catch (IOException ioe) {
-      LOG.info("Error: " + ioe);
-      return;
-    }
-
-    Thread t = new RogueTaskThread();
-    t.start();
-    String pid = getRogueTaskPID();
-    LOG.info("Root process pid: " + pid);
-    ProcfsBasedProcessTree p = new ProcfsBasedProcessTree(pid,
-                               ProcessTree.isSetsidAvailable,
-                               ProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL);
-    p.updateProcessTree(); // initialize
-    LOG.info("ProcessTree: " + p.toString());
-
-    File leaf = new File(lowestDescendant);
-    //wait till lowest descendant process of Rougue Task starts execution
-    while (!leaf.exists()) {
-      try {
-        Thread.sleep(500);
-      } catch (InterruptedException ie) {
-        break;
-      }
-    }
-
-    p.updateProcessTree(); // reconstruct
-    LOG.info("ProcessTree: " + p.toString());
-
-    // Get the process-tree dump
-    String processTreeDump = p.getProcessTreeDump();
-
-    // destroy the process and all its subprocesses
-    p.destroy(true/*in the background*/);
-
-    if(ProcessTree.isSetsidAvailable) {// whole processtree should be gone
-      assertEquals(false, p.isAnyProcessInTreeAlive());
-    }
-    else {// process should be gone
-      assertFalse("ProcessTree must have been gone", p.isAlive());
-    }
-
-    LOG.info("Process-tree dump follows: \n" + processTreeDump);
-    assertTrue("Process-tree dump doesn't start with a proper header",
-        processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " +
-        "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " +
-        "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
-    for (int i = N; i >= 0; i--) {
-      String cmdLineDump = "\\|- [0-9]+ [0-9]+ [0-9]+ [0-9]+ \\(sh\\)" +
-          " [0-9]+ [0-9]+ [0-9]+ [0-9]+ sh " + shellScript + " " + i;
-      Pattern pat = Pattern.compile(cmdLineDump);
-      Matcher mat = pat.matcher(processTreeDump);
-      assertTrue("Process-tree dump doesn't contain the cmdLineDump of " + i
-          + "th process!", mat.find());
-    }
-
-    // Not able to join thread sometimes when forking with large N.
-    try {
-      t.join(2000);
-      LOG.info("RogueTaskThread successfully joined.");
-    } catch (InterruptedException ie) {
-      LOG.info("Interrupted while joining RogueTaskThread.");
-    }
-
-    // ProcessTree is gone now. Any further calls should be sane.
-    p.updateProcessTree();
-    assertFalse("ProcessTree must have been gone", p.isAlive());
-    assertTrue("Cumulative vmem for the gone-process is "
-        + p.getCumulativeVmem() + " . It should be zero.", p
-        .getCumulativeVmem() == 0);
-    assertTrue(p.toString().equals("[ ]"));
-  }
-
-  /**
-   * Get PID from a pid-file.
-   * 
-   * @param pidFileName
-   *          Name of the pid-file.
-   * @return the PID string read from the pid-file. Returns null if the
-   *         pidFileName points to a non-existing file or if read fails from the
-   *         file.
-   */
-  public static String getPidFromPidFile(String pidFileName) {
-    BufferedReader pidFile = null;
-    FileReader fReader = null;
-    String pid = null;
-
-    try {
-      fReader = new FileReader(pidFileName);
-      pidFile = new BufferedReader(fReader);
-    } catch (FileNotFoundException f) {
-      LOG.debug("PidFile doesn't exist : " + pidFileName);
-      return pid;
-    }
-
-    try {
-      pid = pidFile.readLine();
-    } catch (IOException i) {
-      LOG.error("Failed to read from " + pidFileName);
-    } finally {
-      try {
-        if (fReader != null) {
-          fReader.close();
-        }
-        try {
-          if (pidFile != null) {
-            pidFile.close();
-          }
-        } catch (IOException i) {
-          LOG.warn("Error closing the stream " + pidFile);
-        }
-      } catch (IOException i) {
-        LOG.warn("Error closing the stream " + fReader);
-      }
-    }
-    return pid;
-  }
-  
-  public static class ProcessStatInfo {
-    // sample stat in a single line : 3910 (gpm) S 1 3910 3910 0 -1 4194624 
-    // 83 0 0 0 0 0 0 0 16 0 1 0 7852 2408448 88 4294967295 134512640 
-    // 134590050 3220521392 3220520036 10975138 0 0 4096 134234626 
-    // 4294967295 0 0 17 1 0 0
-    String pid;
-    String name;
-    String ppid;
-    String pgrpId;
-    String session;
-    String vmem = "0";
-    String rssmemPage = "0";
-    String utime = "0";
-    String stime = "0";
-    
-    public ProcessStatInfo(String[] statEntries) {
-      pid = statEntries[0];
-      name = statEntries[1];
-      ppid = statEntries[2];
-      pgrpId = statEntries[3];
-      session = statEntries[4];
-      vmem = statEntries[5];
-      if (statEntries.length > 6) {
-        rssmemPage = statEntries[6];
-      }
-      if (statEntries.length > 7) {
-        utime = statEntries[7];
-        stime = statEntries[8];
-      }
-    }
-    
-    // construct a line that mimics the procfs stat file.
-    // all unused numerical entries are set to 0.
-    public String getStatLine() {
-      return String.format("%s (%s) S %s %s %s 0 0 0" +
-                      " 0 0 0 0 %s %s 0 0 0 0 0 0 0 %s %s 0 0" +
-                      " 0 0 0 0 0 0 0 0" +
-                      " 0 0 0 0 0", 
-                      pid, name, ppid, pgrpId, session,
-                      utime, stime, vmem, rssmemPage);
-    }
-  }
-  
-  /**
-   * A basic test that creates a few process directories and writes
-   * stat files. Verifies that the cpu time and memory is correctly
-   * computed.
-   * @throws IOException if there was a problem setting up the
-   *                      fake procfs directories or files.
-   */
-  public void testCpuAndMemoryForProcessTree() throws IOException {
-
-    // test processes
-    String[] pids = { "100", "200", "300", "400" };
-    // create the fake procfs root directory. 
-    File procfsRootDir = new File(TEST_ROOT_DIR, "proc");
-
-    try {
-      setupProcfsRootDir(procfsRootDir);
-      setupPidDirs(procfsRootDir, pids);
-      
-      // create stat objects.
-      // assuming processes 100, 200, 300 are in tree and 400 is not.
-      ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
-      procInfos[0] = new ProcessStatInfo(new String[] 
-          {"100", "proc1", "1", "100", "100", "100000", "100", "1000", "200"});
-      procInfos[1] = new ProcessStatInfo(new String[] 
-          {"200", "proc2", "100", "100", "100", "200000", "200", "2000", "400"});
-      procInfos[2] = new ProcessStatInfo(new String[] 
-          {"300", "proc3", "200", "100", "100", "300000", "300", "3000", "600"});
-      procInfos[3] = new ProcessStatInfo(new String[] 
-          {"400", "proc4", "1", "400", "400", "400000", "400", "4000", "800"});
-      
-      writeStatFiles(procfsRootDir, pids, procInfos);
-      
-      // crank up the process tree class.
-      ProcfsBasedProcessTree processTree = 
-          new ProcfsBasedProcessTree("100", true, 100L, 
-                                  procfsRootDir.getAbsolutePath());
-      // build the process tree.
-      processTree.updateProcessTree();
-      
-      // verify cumulative memory
-      assertEquals("Cumulative virtual memory does not match", 600000L,
-                   processTree.getCumulativeVmem());
-
-      // verify rss memory
-      long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
-                        600L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
-      assertEquals("Cumulative rss memory does not match",
-                   cumuRssMem, processTree.getCumulativeRssmem());
-
-      // verify cumulative cpu time
-      long cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
-             7200L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L;
-      assertEquals("Cumulative cpu time does not match",
-                   cumuCpuTime, processTree.getCumulativeCpuTime());
-
-      // test the cpu time again to see if it cumulates
-      procInfos[0] = new ProcessStatInfo(new String[]
-          {"100", "proc1", "1", "100", "100", "100000", "100", "2000", "300"});
-      procInfos[1] = new ProcessStatInfo(new String[]
-          {"200", "proc2", "100", "100", "100", "200000", "200", "3000", "500"});
-      writeStatFiles(procfsRootDir, pids, procInfos);
-
-      // build the process tree.
-      processTree.updateProcessTree();
-
-      // verify cumulative cpu time again
-      cumuCpuTime = ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS > 0 ?
-             9400L * ProcfsBasedProcessTree.JIFFY_LENGTH_IN_MILLIS : 0L;
-      assertEquals("Cumulative cpu time does not match",
-                   cumuCpuTime, processTree.getCumulativeCpuTime());
-    } finally {
-      FileUtil.fullyDelete(procfsRootDir);
-    }
-  }
-  
-  /**
-   * Tests that cumulative memory is computed only for
-   * processes older than a given age.
-   * @throws IOException if there was a problem setting up the
-   *                      fake procfs directories or files.
-   */
-  public void testMemForOlderProcesses() throws IOException {
-    // initial list of processes
-    String[] pids = { "100", "200", "300", "400" };
-    // create the fake procfs root directory. 
-    File procfsRootDir = new File(TEST_ROOT_DIR, "proc");
-
-    try {
-      setupProcfsRootDir(procfsRootDir);
-      setupPidDirs(procfsRootDir, pids);
-      
-      // create stat objects.
-      // assuming 100, 200 and 400 are in tree, 300 is not.
-      ProcessStatInfo[] procInfos = new ProcessStatInfo[4];
-      procInfos[0] = new ProcessStatInfo(new String[] 
-                        {"100", "proc1", "1", "100", "100", "100000", "100"});
-      procInfos[1] = new ProcessStatInfo(new String[] 
-                        {"200", "proc2", "100", "100", "100", "200000", "200"});
-      procInfos[2] = new ProcessStatInfo(new String[] 
-                        {"300", "proc3", "1", "300", "300", "300000", "300"});
-      procInfos[3] = new ProcessStatInfo(new String[] 
-                        {"400", "proc4", "100", "100", "100", "400000", "400"});
-      
-      writeStatFiles(procfsRootDir, pids, procInfos);
-      
-      // crank up the process tree class.
-      ProcfsBasedProcessTree processTree = 
-          new ProcfsBasedProcessTree("100", true, 100L, 
-                                  procfsRootDir.getAbsolutePath());
-      // build the process tree.
-      processTree.updateProcessTree();
-      
-      // verify cumulative memory
-      assertEquals("Cumulative memory does not match",
-                   700000L, processTree.getCumulativeVmem());
-
-      // write one more process as child of 100.
-      String[] newPids = { "500" };
-      setupPidDirs(procfsRootDir, newPids);
-      
-      ProcessStatInfo[] newProcInfos = new ProcessStatInfo[1];
-      newProcInfos[0] = new ProcessStatInfo(new String[]
-                      {"500", "proc5", "100", "100", "100", "500000", "500"});
-      writeStatFiles(procfsRootDir, newPids, newProcInfos);
-      
-      // check memory includes the new process.
-      processTree.updateProcessTree();
-      assertEquals("Cumulative vmem does not include new process",
-                   1200000L, processTree.getCumulativeVmem());
-      long cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
-                        1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
-      assertEquals("Cumulative rssmem does not include new process",
-                   cumuRssMem, processTree.getCumulativeRssmem());
-      
-      // however processes older than 1 iteration will retain the older value
-      assertEquals("Cumulative vmem shouldn't have included new process",
-                   700000L, processTree.getCumulativeVmem(1));
-      cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
-                   700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
-      assertEquals("Cumulative rssmem shouldn't have included new process",
-                   cumuRssMem, processTree.getCumulativeRssmem(1));
-
-      // one more process
-      newPids = new String[]{ "600" };
-      setupPidDirs(procfsRootDir, newPids);
-      
-      newProcInfos = new ProcessStatInfo[1];
-      newProcInfos[0] = new ProcessStatInfo(new String[]
-                      {"600", "proc6", "100", "100", "100", "600000", "600"});
-      writeStatFiles(procfsRootDir, newPids, newProcInfos);
-
-      // refresh process tree
-      processTree.updateProcessTree();
-      
-      // processes older than 2 iterations should be same as before.
-      assertEquals("Cumulative vmem shouldn't have included new processes",
-                   700000L, processTree.getCumulativeVmem(2));
-      cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
-                   700L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
-      assertEquals("Cumulative rssmem shouldn't have included new processes",
-                   cumuRssMem, processTree.getCumulativeRssmem(2));
-
-      // processes older than 1 iteration should not include new process,
-      // but include process 500
-      assertEquals("Cumulative vmem shouldn't have included new processes",
-                   1200000L, processTree.getCumulativeVmem(1));
-      cumuRssMem = ProcfsBasedProcessTree.PAGE_SIZE > 0 ?
-                   1200L * ProcfsBasedProcessTree.PAGE_SIZE : 0L;
-      assertEquals("Cumulative rssmem shouldn't have included new processes",
-                   cumuRssMem, processTree.getCumulativeRssmem(1));
-
-      // no processes older than 3 iterations, this should be 0
-      assertEquals("Getting non-zero vmem for processes older than 3 iterations",
-                    0L, processTree.getCumulativeVmem(3));
-      assertEquals("Getting non-zero rssmem for processes older than 3 iterations",
-                    0L, processTree.getCumulativeRssmem(3));
-    } finally {
-      FileUtil.fullyDelete(procfsRootDir);
-    }
-  }
-
-  /**
-   * Verifies ProcfsBasedProcessTree.checkPidPgrpidForMatch() in case of
-   * 'constructProcessInfo() returning null' by not writing stat file for the
-   * mock process
-   * @throws IOException if there was a problem setting up the
-   *                      fake procfs directories or files.
-   */
-  public void testDestroyProcessTree() throws IOException {
-    // test process
-    String pid = "100";
-    // create the fake procfs root directory. 
-    File procfsRootDir = new File(TEST_ROOT_DIR, "proc");
-
-    try {
-      setupProcfsRootDir(procfsRootDir);
-      
-      // crank up the process tree class.
-      ProcfsBasedProcessTree processTree = new ProcfsBasedProcessTree(
-                        pid, true, 100L, procfsRootDir.getAbsolutePath());
-
-      // Let us not create stat file for pid 100.
-      assertTrue(ProcfsBasedProcessTree.checkPidPgrpidForMatch(
-                            pid, procfsRootDir.getAbsolutePath()));
-    } finally {
-      FileUtil.fullyDelete(procfsRootDir);
-    }
-  }
-  
-  /**
-   * Test the correctness of process-tree dump.
-   * 
-   * @throws IOException
-   */
-  public void testProcessTreeDump()
-      throws IOException {
-
-    String[] pids = { "100", "200", "300", "400", "500", "600" };
-
-    File procfsRootDir = new File(TEST_ROOT_DIR, "proc");
-
-    try {
-      setupProcfsRootDir(procfsRootDir);
-      setupPidDirs(procfsRootDir, pids);
-
-      int numProcesses = pids.length;
-      // Processes 200, 300, 400 and 500 are descendants of 100. 600 is not.
-      ProcessStatInfo[] procInfos = new ProcessStatInfo[numProcesses];
-      procInfos[0] = new ProcessStatInfo(new String[] {
-          "100", "proc1", "1", "100", "100", "100000", "100", "1000", "200"});
-      procInfos[1] = new ProcessStatInfo(new String[] {
-          "200", "proc2", "100", "100", "100", "200000", "200", "2000", "400"});
-      procInfos[2] = new ProcessStatInfo(new String[] {
-          "300", "proc3", "200", "100", "100", "300000", "300", "3000", "600"});
-      procInfos[3] = new ProcessStatInfo(new String[] {
-          "400", "proc4", "200", "100", "100", "400000", "400", "4000", "800"});
-      procInfos[4] = new ProcessStatInfo(new String[] {
-          "500", "proc5", "400", "100", "100", "400000", "400", "4000", "800"});
-      procInfos[5] = new ProcessStatInfo(new String[] {
-          "600", "proc6", "1", "1", "1", "400000", "400", "4000", "800"});
-
-      String[] cmdLines = new String[numProcesses];
-      cmdLines[0] = "proc1 arg1 arg2";
-      cmdLines[1] = "proc2 arg3 arg4";
-      cmdLines[2] = "proc3 arg5 arg6";
-      cmdLines[3] = "proc4 arg7 arg8";
-      cmdLines[4] = "proc5 arg9 arg10";
-      cmdLines[5] = "proc6 arg11 arg12";
-
-      writeStatFiles(procfsRootDir, pids, procInfos);
-      writeCmdLineFiles(procfsRootDir, pids, cmdLines);
-
-      ProcfsBasedProcessTree processTree =
-          new ProcfsBasedProcessTree("100", true, 100L, procfsRootDir
-              .getAbsolutePath());
-      // build the process tree.
-      processTree.updateProcessTree();
-
-      // Get the process-tree dump
-      String processTreeDump = processTree.getProcessTreeDump();
-
-      LOG.info("Process-tree dump follows: \n" + processTreeDump);
-      assertTrue("Process-tree dump doesn't start with a proper header",
-          processTreeDump.startsWith("\t|- PID PPID PGRPID SESSID CMD_NAME " +
-          "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) " +
-          "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
-      for (int i = 0; i < 5; i++) {
-        ProcessStatInfo p = procInfos[i];
-        assertTrue(
-            "Process-tree dump doesn't contain the cmdLineDump of process "
-                + p.pid, processTreeDump.contains("\t|- " + p.pid + " "
-                + p.ppid + " " + p.pgrpId + " " + p.session + " (" + p.name
-                + ") " + p.utime + " " + p.stime + " " + p.vmem + " "
-                + p.rssmemPage + " " + cmdLines[i]));
-      }
-
-      // 600 should not be in the dump
-      ProcessStatInfo p = procInfos[5];
-      assertFalse(
-          "Process-tree dump shouldn't contain the cmdLineDump of process "
-              + p.pid, processTreeDump.contains("\t|- " + p.pid + " " + p.ppid
-              + " " + p.pgrpId + " " + p.session + " (" + p.name + ") "
-              + p.utime + " " + p.stime + " " + p.vmem + " " + cmdLines[5]));
-    } finally {
-      FileUtil.fullyDelete(procfsRootDir);
-    }
-  }
-
-  /**
-   * Create a directory to mimic the procfs file system's root.
-   * @param procfsRootDir root directory to create.
-   * @throws IOException if could not delete the procfs root directory
-   */
-  public static void setupProcfsRootDir(File procfsRootDir) {
-    // cleanup any existing process root dir.
-    if (procfsRootDir.exists()) {
-      assertTrue(FileUtil.fullyDelete(procfsRootDir));  
-    }
-
-    // create afresh
-    assertTrue(procfsRootDir.mkdirs());
-  }
-
-  /**
-   * Create PID directories under the specified procfs root directory
-   * @param procfsRootDir root directory of procfs file system
-   * @param pids the PID directories to create.
-   * @throws IOException If PID dirs could not be created
-   */
-  public static void setupPidDirs(File procfsRootDir, String[] pids) 
-                      throws IOException {
-    for (String pid : pids) {
-      File pidDir = new File(procfsRootDir, pid);
-      pidDir.mkdir();
-      if (!pidDir.exists()) {
-        throw new IOException ("couldn't make process directory under " +
-            "fake procfs");
-      } else {
-        LOG.info("created pid dir");
-      }
-    }
-  }
-  
-  /**
-   * Write stat files under the specified pid directories with data
-   * setup in the corresponding ProcessStatInfo objects
-   * @param procfsRootDir root directory of procfs file system
-   * @param pids the PID directories under which to create the stat file
-   * @param procs corresponding ProcessStatInfo objects whose data should be
-   *              written to the stat files.
-   * @throws IOException if stat files could not be written
-   */
-  public static void writeStatFiles(File procfsRootDir, String[] pids, 
-                              ProcessStatInfo[] procs) throws IOException {
-    for (int i=0; i<pids.length; i++) {
-      File statFile =
-          new File(new File(procfsRootDir, pids[i]),
-              ProcfsBasedProcessTree.PROCFS_STAT_FILE);
-      BufferedWriter bw = null;
-      try {
-        FileWriter fw = new FileWriter(statFile);
-        bw = new BufferedWriter(fw);
-        bw.write(procs[i].getStatLine());
-        LOG.info("wrote stat file for " + pids[i] + 
-                  " with contents: " + procs[i].getStatLine());
-      } finally {
-        // not handling exception - will throw an error and fail the test.
-        if (bw != null) {
-          bw.close();
-        }
-      }
-    }
-  }
-
-  private static void writeCmdLineFiles(File procfsRootDir, String[] pids,
-      String[] cmdLines)
-      throws IOException {
-    for (int i = 0; i < pids.length; i++) {
-      File statFile =
-          new File(new File(procfsRootDir, pids[i]),
-              ProcfsBasedProcessTree.PROCFS_CMDLINE_FILE);
-      BufferedWriter bw = null;
-      try {
-        bw = new BufferedWriter(new FileWriter(statFile));
-        bw.write(cmdLines[i]);
-        LOG.info("wrote command-line file for " + pids[i] + " with contents: "
-            + cmdLines[i]);
-      } finally {
-        // not handling exception - will throw an error and fail the test.
-        if (bw != null) {
-          bw.close();
-        }
-      }
-    }
-  }
-}

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java

@@ -37,11 +37,11 @@ import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.tools.rumen.JobStory;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
 import org.apache.hadoop.tools.rumen.TaskInfo;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 import java.io.IOException;
 import java.security.PrivilegedExceptionAction;

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/CumulativeCpuUsageEmulatorPlugin.java

@@ -22,8 +22,8 @@ import java.util.Random;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 /**
  * <p>A {@link ResourceUsageEmulatorPlugin} that emulates the cumulative CPU 
@@ -166,7 +166,7 @@ implements ResourceUsageEmulatorPlugin {
      */
     public void calibrate(ResourceCalculatorPlugin monitor, 
                           long totalCpuUsage) {
-      long initTime = monitor.getProcResourceValues().getCumulativeCpuTime();
+      long initTime = monitor.getCumulativeCpuTime();
       
       long defaultLoopSize = 0;
       long finalTime = initTime;
@@ -175,7 +175,7 @@ implements ResourceUsageEmulatorPlugin {
       while (finalTime - initTime < 100) { // 100 ms
         ++defaultLoopSize;
         performUnitComputation(); //perform unit computation
-        finalTime = monitor.getProcResourceValues().getCumulativeCpuTime();
+        finalTime = monitor.getCumulativeCpuTime();
       }
       
       long referenceRuntime = finalTime - initTime;
@@ -230,7 +230,7 @@ implements ResourceUsageEmulatorPlugin {
   }
   
   private synchronized long getCurrentCPUUsage() {
-    return monitor.getProcResourceValues().getCumulativeCpuTime();
+    return monitor.getCumulativeCpuTime();
   }
   
   @Override

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageEmulatorPlugin.java

@@ -20,7 +20,7 @@ package org.apache.hadoop.mapred.gridmix.emulators.resourceusage;
 import java.io.IOException;
 
 import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
 import org.apache.hadoop.conf.Configuration;
 

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/ResourceUsageMatcher.java

@@ -23,9 +23,9 @@ import java.util.List;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 /**
  * <p>This is the driver class for managing all the resource usage emulators.

hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java

@@ -21,8 +21,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapred.gridmix.Progressive;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 /**
  * <p>A {@link ResourceUsageEmulatorPlugin} that emulates the total heap 

hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DummyResourceCalculatorPlugin.java

@@ -16,18 +16,17 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.mapred;
+package org.apache.hadoop.mapred.gridmix;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 /**
- * Plugin class to test resource information reported by TT. Use
+ * Plugin class to test resource information reported by NM. Use configuration
- * configuration items {@link #MAXVMEM_TESTING_PROPERTY} and
+ * items {@link #MAXVMEM_TESTING_PROPERTY} and {@link #MAXPMEM_TESTING_PROPERTY}
- * {@link #MAXPMEM_TESTING_PROPERTY} to tell TT the total vmem and the total
+ * to tell NM the total vmem and the total pmem. Use configuration items
- * pmem. Use configuration items {@link #NUM_PROCESSORS},
+ * {@link #NUM_PROCESSORS}, {@link #CPU_FREQUENCY}, {@link #CUMULATIVE_CPU_TIME}
- * {@link #CPU_FREQUENCY}, {@link #CUMULATIVE_CPU_TIME} and {@link #CPU_USAGE}
+ * and {@link #CPU_USAGE} to tell TT the CPU information.
- * to tell TT the CPU information.
  */
 @InterfaceAudience.Private
 public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
@@ -48,15 +47,14 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
   public static final String CUMULATIVE_CPU_TIME =
       "mapred.tasktracker.cumulativecputime.testing";
   /** CPU usage percentage for testing */
-  public static final String CPU_USAGE =
+  public static final String CPU_USAGE = "mapred.tasktracker.cpuusage.testing";
-      "mapred.tasktracker.cpuusage.testing";
   /** process cumulative CPU usage time for testing */
   public static final String PROC_CUMULATIVE_CPU_TIME =
       "mapred.tasktracker.proccumulativecputime.testing";
-  /** process pmem for testing*/
+  /** process pmem for testing */
   public static final String PROC_PMEM_TESTING_PROPERTY =
       "mapred.tasktracker.procpmem.testing";
-  /** process vmem for testing*/
+  /** process vmem for testing */
   public static final String PROC_VMEM_TESTING_PROPERTY =
       "mapred.tasktracker.procvmem.testing";
 
@@ -107,12 +105,4 @@ public class DummyResourceCalculatorPlugin extends ResourceCalculatorPlugin {
   public float getCpuUsage() {
     return getConf().getFloat(CPU_USAGE, -1);
   }
-
-  @Override
-  public ProcResourceValues getProcResourceValues() {
-    long cpuTime = getConf().getLong(PROC_CUMULATIVE_CPU_TIME, -1);
-    long pMem = getConf().getLong(PROC_PMEM_TESTING_PROPERTY, -1);
-    long vMem = getConf().getLong(PROC_VMEM_TESTING_PROPERTY, -1);
-    return new ProcResourceValues(cpuTime, pMem, vMem);
-  }
 }

hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java

@@ -23,7 +23,6 @@ import static org.junit.Assert.*;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob;
 import org.apache.hadoop.mapred.gridmix.TestHighRamJob.DummyGridmixJob;
@@ -32,8 +31,8 @@ import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEm
 import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.TotalHeapUsageEmulatorPlugin.DefaultHeapUsageEmulator;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 /**
  * Test Gridmix memory emulation.

hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestResourceUsageEmulators.java

@@ -31,14 +31,13 @@ import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
 import org.apache.hadoop.mapreduce.task.MapContextImpl;
-import org.apache.hadoop.mapreduce.util.ResourceCalculatorPlugin;
 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
-import org.apache.hadoop.mapred.DummyResourceCalculatorPlugin;
 import org.apache.hadoop.mapred.gridmix.LoadJob.ResourceUsageMatcherRunner;
 import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin;
 import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageEmulatorPlugin;
 import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.ResourceUsageMatcher;
 import org.apache.hadoop.mapred.gridmix.emulators.resourceusage.CumulativeCpuUsageEmulatorPlugin.DefaultCpuUsageEmulator;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
 
 /**
  * Test Gridmix's resource emulator framework and supported plugins.
@@ -242,16 +241,6 @@ public class TestResourceUsageEmulators {
     public long getCumulativeCpuTime() {
       return core.getCpuUsage();
     }
-
-    /**
-     * Returns a {@link ProcResourceValues} with cumulative cpu usage  
-     * computed using {@link #getCumulativeCpuTime()}.
-     */
-    @Override
-    public ProcResourceValues getProcResourceValues() {
-      long usageValue = getCumulativeCpuTime();
-      return new ProcResourceValues(usageValue, -1, -1);
-    }
   }
   
   /**

hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamReduceNone.java

@@ -68,6 +68,7 @@ public class TestStreamReduceNone
       "-reducer", "org.apache.hadoop.mapred.lib.IdentityReducer",
       "-numReduceTasks", "0",
       "-jobconf", "mapreduce.task.files.preserve.failedtasks=true",
+      "-jobconf", "mapreduce.job.maps=1",
       "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp")
     };
   }

hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamXmlRecordReader.java

@@ -54,6 +54,8 @@ public class TestStreamXmlRecordReader extends TestStreaming {
   protected String[] genArgs() {
     args.add("-inputreader");
     args.add("StreamXmlRecordReader,begin=<xmltag>,end=</xmltag>");
+    args.add("-jobconf");
+    args.add("mapreduce.job.maps=1");
     return super.genArgs();
   }
 }

hadoop-yarn-project/CHANGES.txt

@@ -62,6 +62,10 @@ Release 2.0.5-beta - UNRELEASED
     YARN-396. Rationalize AllocateResponse in RM Scheduler API. (Zhijie Shen
     via hitesh)
 
+    YARN-439. Flatten NodeHeartbeatResponse. (Xuan Gong via sseth)
+
+    YARN-440. Flatten RegisterNodeManagerResponse. (Xuan Gong via sseth)
+
   NEW FEATURES
 
   IMPROVEMENTS
@@ -91,6 +95,14 @@ Release 2.0.5-beta - UNRELEASED
     YARN-417. Create AMRMClient wrapper that provides asynchronous callbacks.
     (Sandy Ryza via bikas)
 
+    YARN-497. Yarn unmanaged-am launcher jar does not define a main class in
+    its manifest (Hitesh Shah via bikas)
+
+    YARN-469. Make scheduling mode in FS pluggable. (kkambatl via tucu)
+
+    YARN-450. Define value for * in the scheduling protocol (Zhijie Shen via
+    bikas)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -124,6 +136,30 @@ Release 2.0.5-beta - UNRELEASED
  
     YARN-470. Support a way to disable resource monitoring on the NodeManager.
     (Siddharth Seth via hitesh)
+    
+    YARN-71. Fix the NodeManager to clean up local-dirs on restart.
+    (Xuan Gong via sseth)
+
+    YARN-378. Fix RM to make the AM max attempts/retries to be configurable
+    per application by clients. (Zhijie Shen via vinodkv)
+
+    YARN-498. Unmanaged AM launcher does not set various constants in env for
+    an AM, also does not handle failed AMs properly. (Hitesh Shah via bikas)
+
+    YARN-474. Fix CapacityScheduler to trigger application-activation when
+    am-resource-percent configuration is refreshed. (Zhijie Shen via vinodkv)
+
+    YARN-496. Fair scheduler configs are refreshed inconsistently in
+    reinitialize. (Sandy Ryza via tomwhite)
+
+    YARN-209. Fix CapacityScheduler to trigger application-activation when
+    the cluster capacity changes. (Zhijie Shen via vinodkv)
+
+    YARN-24. Nodemanager fails to start if log aggregation enabled and 
+    namenode unavailable. (sandyr via tucu)
+
+    YARN-515. Node Manager not getting the master key. (Robert Joseph Evans
+    via jlowe)
 
 Release 2.0.4-alpha - UNRELEASED
 
@@ -463,6 +499,12 @@ Release 0.23.7 - UNRELEASED
     YARN-345. Many InvalidStateTransitonException errors for ApplicationImpl
     in Node Manager (Robert Parker via jlowe)
 
+    YARN-109. .tmp file is not deleted for localized archives (Mayank Bansal 
+    via bobby)
+
+    YARN-460. CS user left in list of active users for the queue even when 
+    application finished (tgraves)
+
 Release 0.23.6 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java

@@ -92,6 +92,12 @@ public interface ApplicationConstants {
 
   public static final String STDOUT = "stdout";
 
+  /**
+   * The environment variable for MAX_APP_ATTEMPTS. Set in AppMaster environment
+   * only
+   */
+  public static final String MAX_APP_ATTEMPTS_ENV = "MAX_APP_ATTEMPTS";
+
   /**
    * Environment for Applications.
    * 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java

@@ -189,4 +189,22 @@ public interface ApplicationSubmissionContext {
   @LimitedPrivate("mapreduce")
   @Unstable
   public void setCancelTokensWhenComplete(boolean cancel);
+
+  /**
+   * @return the number of max attempts of the application to be submitted
+   */
+  @Public
+  @Unstable
+  public int getMaxAppAttempts();
+
+  /**
+   * Set the number of max attempts of the application to be submitted. WARNING:
+   * it should be no larger than the global number of max attempts in the Yarn
+   * configuration.
+   * @param maxAppAttempts the number of max attempts of the application
+   * to be submitted.
+   */
+  @Public
+  @Unstable
+  public void setMaxAppAttempts(int maxAppAttempts);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java

@@ -49,6 +49,26 @@ import org.apache.hadoop.yarn.api.AMRMProtocol;
 @Public
 @Stable
 public abstract class ResourceRequest implements Comparable<ResourceRequest> {
+
+  /**
+   * The constant string representing no locality.
+   * It should be used by all references that want to pass an arbitrary host
+   * name in.
+   */
+  public static final String ANY = "*";
+
+  /**
+   * Check whether the given <em>host/rack</em> string represents an arbitrary
+   * host name.
+   *
+   * @param hostName <em>host/rack</em> on which the allocation is desired
+   * @return whether the given <em>host/rack</em> string represents an arbitrary
+   * host name
+   */
+  public static boolean isAnyLocation(String hostName) {
+    return ANY.equals(hostName);
+  }
+
   /**
    * Get the <code>Priority</code> of the request.
    * @return <code>Priority</code> of the request