MAPREDUCE-3954. Added new envs to separate heap size for different daemons started via bin scripts. Contributed by Robert Joseph Evans.

svn merge --ignore-ancestry -c 1297174 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1297176 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -77,6 +77,9 @@ Release 0.23.2 - UNRELEASED
 
     MAPREDUCE-3497. Added docs for YARN CLI. (tgraves via acmurthy) 
 
+    MAPREDUCE-3954. Added new envs to separate heap size for different daemons
+    started via bin scripts. (Robert Joseph Evans via vinodkv)
+
   OPTIMIZATIONS
 
     MAPREDUCE-3901. Modified JobHistory records in YARN to lazily load job and

hadoop-mapreduce-project/bin/mapred

@@ -51,8 +51,6 @@ fi
 COMMAND=$1
 shift
 
-HADOOP_JOB_HISTORYSERVER_OPTS="-Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console}"
-
 if [ "$COMMAND" = "job" ] ; then
   CLASS=org.apache.hadoop.mapred.JobClient
 elif [ "$COMMAND" = "queue" ] ; then
@@ -70,7 +68,10 @@ elif [ "$COMMAND" = "groups" ] ; then
   HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
 elif [ "$COMMAND" = "historyserver" ] ; then
   CLASS=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
-  HADOOP_OPTS="$HADOOP_OPTS $HADOOP_JOB_HISTORYSERVER_OPTS"
+  HADOOP_OPTS="$HADOOP_OPTS -Dmapred.jobsummary.logger=${HADOOP_JHS_LOGGER:-INFO,console} $HADOOP_JOB_HISTORYSERVER_OPTS"
+  if [ "$HADOOP_JOB_HISTORYSERVER_HEAPSIZE" != "" ]; then
+    JAVA_HEAP_MAX="-Xmx""$HADOOP_JOB_HISTORYSERVER_HEAPSIZE""m"
+  fi
 elif [ "$COMMAND" = "mradmin" ] \
     || [ "$COMMAND" = "jobtracker" ] \
     || [ "$COMMAND" = "tasktracker" ] ; then

hadoop-mapreduce-project/hadoop-yarn/bin/yarn

@@ -27,11 +27,16 @@
 #   YARN_HEAPSIZE  The maximum amount of heap to use, in MB. 
 #                    Default is 1000.
 #
+#   YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command
+#                           eg YARN_NODEMANAGER_HEAPSIZE sets the heap
+#                           size for the NodeManager.  If you set the
+#                           heap size in YARN_{COMMAND}_OPTS or YARN_OPTS
+#                           they take precedence.
+#
 #   YARN_OPTS      Extra Java runtime options.
 #   
-#   YARN_NAMENODE_OPTS       These options are added to YARN_OPTS 
 #   YARN_CLIENT_OPTS         when the respective command is run.
-#   YARN_{COMMAND}_OPTS etc  YARN_JT_OPTS applies to JobTracker 
+#   YARN_{COMMAND}_OPTS etc  YARN_NODEMANAGER_OPTS applies to NodeManager 
 #                              for e.g.  YARN_CLIENT_OPTS applies to 
 #                              more than one command (fs, dfs, fsck, 
 #                              dfsadmin etc)  
@@ -170,6 +175,9 @@ elif [ "$COMMAND" = "resourcemanager" ] ; then
   CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/rm-config/log4j.properties
   CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
   YARN_OPTS="$YARN_OPTS $YARN_RESOURCEMANAGER_OPTS"
+  if [ "$YARN_RESOURCEMANAGER_HEAPSIZE" != "" ]; then
+    JAVA_HEAP_MAX="-Xmx""$YARN_RESOURCEMANAGER_HEAPSIZE""m"
+  fi
 elif [ "$COMMAND" = "nodemanager" ] ; then
   CLASSPATH=${CLASSPATH}:$YARN_CONF_DIR/nm-config/log4j.properties
   CLASS='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
@@ -178,9 +186,15 @@ elif [ "$COMMAND" = "nodemanager" ] ; then
   else
     YARN_OPTS="$YARN_OPTS -server $YARN_NODEMANAGER_OPTS"
   fi
+  if [ "$YARN_NODEMANAGER_HEAPSIZE" != "" ]; then
+    JAVA_HEAP_MAX="-Xmx""$YARN_NODEMANAGER_HEAPSIZE""m"
+  fi
 elif [ "$COMMAND" = "proxyserver" ] ; then
   CLASS='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
   YARN_OPTS="$YARN_OPTS $YARN_PROXYSERVER_OPTS"
+  if [ "$YARN_PROXYSERVER_HEAPSIZE" != "" ]; then
+    JAVA_HEAP_MAX="-Xmx""$YARN_PROXYSERVER_HEAPSIZE""m"
+  fi
 elif [ "$COMMAND" = "version" ] ; then
   CLASS=org.apache.hadoop.util.VersionInfo
   YARN_OPTS="$YARN_OPTS $YARN_CLIENT_OPTS"

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm

@@ -103,6 +103,9 @@ Hadoop MapReduce Next Generation - Cluster Setup
 *--------------------------------------+--------------------------------------+
 | WebAppProxy                          | YARN_PROXYSERVER_OPTS                |
 *--------------------------------------+--------------------------------------+
+| Map Reduce Job History Server        | HADOOP_JOB_HISTORYSERVER_OPTS        |
+*--------------------------------------+--------------------------------------+
+
 
     For example, To configure Namenode to use parallelGC, the following 
     statement should be added in hadoop-env.sh :
@@ -118,9 +121,21 @@ Hadoop MapReduce Next Generation - Cluster Setup
         don't exist.
         
       * <<<HADOOP_HEAPSIZE>>> / <<<YARN_HEAPSIZE>>> - The maximum amount of 
-        heapsize to use, in MB e.g. 1000MB. This is used to configure the heap 
+        heapsize to use, in MB e.g. if the varibale is set to 1000 the heap
-        size for the daemon. By default, the value is 1000MB.
+        will be set to 1000MB.  This is used to configure the heap 
-
+        size for the daemon. By default, the value is 1000.  If you want to
+        configure the values separately for each deamon you can use.
+*--------------------------------------+--------------------------------------+
+|| Daemon                              || Environment Variable                |
+*--------------------------------------+--------------------------------------+
+| ResourceManager                      | YARN_RESOURCEMANAGER_HEAPSIZE        |
+*--------------------------------------+--------------------------------------+
+| NodeManager                          | YARN_NODEMANAGER_HEAPSIZE            |
+*--------------------------------------+--------------------------------------+
+| WebAppProxy                          | YARN_PROXYSERVER_HEAPSIZE            |
+*--------------------------------------+--------------------------------------+
+| Map Reduce Job History Server        | HADOOP_JOB_HISTORYSERVER_HEAPSIZE    |
+*--------------------------------------+--------------------------------------+
  
     * {Configuring the Hadoop Daemons in Non-Secure Mode}