Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allow Datanode and Namenode heapsize to be configured seperately 

Adds option for HDFS_NAMENODE_HEAPSIZE, HDFS_SECONDARYNAMENODE_HEAPSIZE, HDFS_JOURNALNODE_HEAPSIZE and HDFS_DATANODE_HEAPSIZE to hadoop-env.sh so that namenode and datanode JVM heapsizes can be separately configured. This matches the configuration of YARN daemon's heapsizes.
This commit is contained in:
Scheller 2018-07-10 16:06:48 -07:00
parent 556d9b36be
commit e724ef08ed
3 changed files with 42 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
View File

@ -302,6 +302,9 @@ esac
# this is the default:
# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
# Use NameNode specific heap size
# export HDFS_NAMENODE_HEAPSIZE=
###
# SecondaryNameNode specific parameters
###
@ -312,6 +315,9 @@ esac
# This is the default:
# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
# Use secondary NameNode specific heap size
# export HDFS_SECONDARYNAMENODE_HEAPSIZE=
###
# DataNode specific parameters
###
@ -335,6 +341,9 @@ esac
# server jvm.
# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
# Use DataNode specific heap size
# export HDFS_DATANODE_HEAPSIZE=
###
# NFS3 Gateway specific parameters
###
@ -377,6 +386,9 @@ esac
#
# export HDFS_JOURNALNODE_OPTS=""
# Use JournalNode specific heap size
# export HDFS_JOURNALNODE_HEAPSIZE=
###
# HDFS Balancer specific parameters
###

15
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
View File

@ -98,6 +98,9 @@ function hdfscmd_case
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
hadoop_deprecate_envvar HADOOP_SECURE_DN_PID_DIR HADOOP_SECURE_PID_DIR
hadoop_deprecate_envvar HADOOP_SECURE_DN_LOG_DIR HADOOP_SECURE_LOG_DIR
if [[ -n "${HDFS_DATANODE_HEAPSIZE}" ]]; then
HADOOP_HEAPSIZE_MAX="${HDFS_DATANODE_HEAPSIZE}"
fi
;;
debug)
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DebugAdmin'
@ -154,6 +157,9 @@ function hdfscmd_case
journalnode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
if [[ -n "${HDFS_JOURNALNODE_HEAPSIZE}" ]]; then
HADOOP_HEAPSIZE_MAX="${HDFS_JOURNALNODE_HEAPSIZE}"
fi
;;
jmxget)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
@ -169,6 +175,9 @@ function hdfscmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
if [[ -n "${HDFS_NAMENODE_HEAPSIZE}" ]]; then
HADOOP_HEAPSIZE_MAX="${HDFS_NAMENODE_HEAPSIZE}"
fi
;;
nfs3)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@ -194,6 +203,12 @@ function hdfscmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
if [[ -n "${HDFS_NAMENODE_HEAPSIZE}" ]]; then
HADOOP_HEAPSIZE_MAX="${HDFS_NAMENODE_HEAPSIZE}"
fi
if [[ -n "${HDFS_SECONDARYNAMENODE_HEAPSIZE}" ]]; then
HADOOP_HEAPSIZE_MAX="${HDFS_SECONDARYNAMENODE_HEAPSIZE}"
fi
;;
snapshotDiff)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.SnapshotDiff

15
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd
View File

@ -78,11 +78,17 @@ goto :eof
:namenode
set CLASS=org.apache.hadoop.hdfs.server.namenode.NameNode
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_NAMENODE_OPTS%
if defined HDFS_NAMENODE_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HDFS_NAMENODE_HEAPSIZE%m
)
goto :eof
:journalnode
set CLASS=org.apache.hadoop.hdfs.qjournal.server.JournalNode
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_JOURNALNODE_OPTS%
if defined HDFS_JOURNALNODE_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HDFS_JOURNALNODE_HEAPSIZE%m
)
goto :eof
:zkfc
@ -93,11 +99,20 @@ goto :eof
:secondarynamenode
set CLASS=org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode
set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_SECONDARYNAMENODE_OPTS%
if defined HDFS_NAMENODE_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HDFS_NAMENODE_HEAPSIZE%m
)
if defined HDFS_SECONDARYNAMENODE_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HDFS_SECONDARYNAMENODE_HEAPSIZE%m
)
goto :eof
:datanode
set CLASS=org.apache.hadoop.hdfs.server.datanode.DataNode
set HADOOP_OPTS=%HADOOP_OPTS% -server %HADOOP_DATANODE_OPTS%
if defined HDFS_DATANODE_HEAPSIZE (
set JAVA_HEAP_MAX=-Xmx%HDFS_DATANODE_HEAPSIZE%m
)
goto :eof
:dfs