Allow Datanode and Namenode heapsize to be configured seperately

Adds option for HDFS_NAMENODE_HEAPSIZE, HDFS_SECONDARYNAMENODE_HEAPSIZE, HDFS_JOURNALNODE_HEAPSIZE and HDFS_DATANODE_HEAPSIZE to hadoop-env.sh so that namenode and datanode JVM heapsizes can be separately configured. This matches the configuration of YARN daemon's heapsizes.

hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh

@@ -302,6 +302,9 @@ esac
 # this is the default:
 # export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
 
+# Use NameNode specific heap size
+# export HDFS_NAMENODE_HEAPSIZE=
+
 ###
 # SecondaryNameNode specific parameters
 ###
@@ -312,6 +315,9 @@ esac
 # This is the default:
 # export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
 
+# Use secondary NameNode specific heap size
+# export HDFS_SECONDARYNAMENODE_HEAPSIZE=
+
 ###
 # DataNode specific parameters
 ###
@@ -335,6 +341,9 @@ esac
 # server jvm.
 # export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
 
+# Use DataNode specific heap size
+# export HDFS_DATANODE_HEAPSIZE=
+
 ###
 # NFS3 Gateway specific parameters
 ###
@@ -377,6 +386,9 @@ esac
 #
 # export HDFS_JOURNALNODE_OPTS=""
 
+# Use JournalNode specific heap size
+# export HDFS_JOURNALNODE_HEAPSIZE=
+
 ###
 # HDFS Balancer specific parameters
 ###

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs

@@ -98,6 +98,9 @@ function hdfscmd_case
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
       hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR
       hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR
+      if [[ -n "${HDFS_DATANODE_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${HDFS_DATANODE_HEAPSIZE}"
+      fi
     ;;
     debug)
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin'
@@ -154,6 +157,9 @@ function hdfscmd_case
     journalnode)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
+      if [[ -n "${HDFS_JOURNALNODE_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${HDFS_JOURNALNODE_HEAPSIZE}"
+      fi
     ;;
     jmxget)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
@@ -169,6 +175,9 @@ function hdfscmd_case
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
       hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+      if [[ -n "${HDFS_NAMENODE_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${HDFS_NAMENODE_HEAPSIZE}"
+      fi
     ;;
     nfs3)
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -194,6 +203,12 @@ function hdfscmd_case
       HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
       HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
       hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
+      if [[ -n "${HDFS_NAMENODE_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${HDFS_NAMENODE_HEAPSIZE}"
+      fi
+      if [[ -n "${HDFS_SECONDARYNAMENODE_HEAPSIZE}" ]]; then
+        HADOOP_HEAPSIZE_MAX="${HDFS_SECONDARYNAMENODE_HEAPSIZE}"
+      fi
     ;;
     snapshotDiff)
       HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd

@@ -78,11 +78,17 @@ goto :eof
 :namenode
   set CLASS=org.apache.hadoop.hdfs.server.namenode.NameNode
   set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_NAMENODE_OPTS%
+  if defined HDFS_NAMENODE_HEAPSIZE (
+    set JAVA_HEAP_MAX=-Xmx%HDFS_NAMENODE_HEAPSIZE%m
+  )
   goto :eof
 
 :journalnode
   set CLASS=org.apache.hadoop.hdfs.qjournal.server.JournalNode
   set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_JOURNALNODE_OPTS%
+  if defined HDFS_JOURNALNODE_HEAPSIZE (
+    set JAVA_HEAP_MAX=-Xmx%HDFS_JOURNALNODE_HEAPSIZE%m
+  )
   goto :eof
 
 :zkfc
@@ -93,11 +99,20 @@ goto :eof
 :secondarynamenode
   set CLASS=org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
   set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_SECONDARYNAMENODE_OPTS%
+  if defined HDFS_NAMENODE_HEAPSIZE (
+    set JAVA_HEAP_MAX=-Xmx%HDFS_NAMENODE_HEAPSIZE%m
+  )
+  if defined HDFS_SECONDARYNAMENODE_HEAPSIZE (
+    set JAVA_HEAP_MAX=-Xmx%HDFS_SECONDARYNAMENODE_HEAPSIZE%m
+  )
   goto :eof
 
 :datanode
   set CLASS=org.apache.hadoop.hdfs.server.datanode.DataNode
   set HADOOP_OPTS=%HADOOP_OPTS% -server %HADOOP_DATANODE_OPTS%
+  if defined HDFS_DATANODE_HEAPSIZE (
+    set JAVA_HEAP_MAX=-Xmx%HDFS_DATANODE_HEAPSIZE%m
+  )
   goto :eof
 
 :dfs