From cefe4b7bb6322e7c89d7b129b41f5e1a2689ebcf Mon Sep 17 00:00:00 2001 From: Wei Yan Date: Thu, 25 Jan 2018 12:58:16 -0800 Subject: [PATCH] HDFS-13049. RBF: Inconsistent Router OPTS config in branch-2 and branch-3. Contributed by Wei Yan. --- .../hadoop-common/src/main/conf/hadoop-env.cmd | 9 +++++++++ .../hadoop-common/src/main/conf/hadoop-env.sh | 9 +++++++++ hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 9 ++++++++- hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd | 10 +++++++++- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd index 5aed07d21b6..5e0cd8605eb 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd @@ -74,6 +74,15 @@ set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER% @rem Where log files are stored in the secure data environment. set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER% +@rem +@rem Router-based HDFS Federation specific parameters +@rem Specify the JVM options to be used when starting the RBF Routers. +@rem These options will be appended to the options specified as HADOOP_OPTS +@rem and therefore may override any similar flags set in HADOOP_OPTS +@rem +@rem set HADOOP_DFSROUTER_OPTS="" +@rem + @rem The directory where pid files are stored. /tmp by default. @rem NOTE: this should be set to a directory that can only be written to by @rem the user that will run the hadoop daemons. Otherwise there is the diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index 8d15f980134..f1eb66b5507 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -93,6 +93,15 @@ export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER} # # export HADOOP_MOVER_OPTS="" +### +# Router-based HDFS Federation specific parameters +# Specify the JVM options to be used when starting the RBF Routers. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HADOOP_DFSROUTER_OPTS="" +### + ### # Advanced Users Only! ### diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 0b96ec2a181..b306698126c 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -134,6 +134,13 @@ if [ "$COMMAND" == "nfs3" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_PRIVILEGED_NFS fi fi +# Determine if we're using deprecated config HADOOP_ROUTER_OPTS +if [ "$COMMAND" == "dfsrouter" ] && [ "$HADOOP_ROUTER_OPTS" != "" ]; then + echo "WARNING: it looks like you're using a deprecated config HADOOP_ROUTER_OPTS. "\ + "Please switch to HADOOP_DFSROUTER_OPTS." + HADOOP_DFSROUTER_OPTS=$HADOOP_ROUTER_OPTS +fi + if [ "$COMMAND" = "namenode" ] ; then CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' HADOOP_OPTS="$HADOOP_OPTS $HADOOP_NAMENODE_OPTS" @@ -161,7 +168,7 @@ elif [ "$COMMAND" = "dfsadmin" ] ; then HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS" elif [ "$COMMAND" = "dfsrouter" ] ; then CLASS='org.apache.hadoop.hdfs.server.federation.router.DFSRouter' - HADOOP_OPTS="$HADOOP_OPTS $HADOOP_ROUTER_OPTS" + HADOOP_OPTS="$HADOOP_OPTS $HADOOP_DFSROUTER_OPTS" elif [ "$COMMAND" = "dfsrouteradmin" ] ; then CLASS='org.apache.hadoop.hdfs.tools.federation.RouterAdmin' elif [ "$COMMAND" = "haadmin" ] ; then diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd index a9a7852fa61..b91aec99d78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -59,6 +59,14 @@ if "%1" == "--loglevel" ( ) ) + if "%hdfs-command%" == "dfsrouter" ( + if defined HADOOP_ROUTER_OPTS ( + @echo WARNING: it looks like you're using a deprecated config HADOOP_ROUTER_OPTS. + @echo Please switch to HADOOP_DFSROUTER_OPTS. + set HADOOP_DFSROUTER_OPTS=%HADOOP_ROUTER_OPTS% + ) + ) + set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover storagepolicies classpath crypto dfsrouter dfsrouteradmin debug for %%i in ( %hdfscommands% ) do ( if %hdfs-command% == %%i set hdfscommand=true @@ -181,7 +189,7 @@ goto :eof :dfsrouter set CLASS=org.apache.hadoop.hdfs.server.federation.router.DFSRouter - set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ROUTER_OPTS% + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_DFSROUTER_OPTS% goto :eof :dfsrouteradmin