HADOOP-7984. Add hadoop --loglevel option to change log level. Contributed by Aikira AJISAKA.

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -8,6 +8,9 @@ Release 2.7.0 - UNRELEASED
 
     HADOOP-10987. Provide an iterator-based listing API for FileSystem (kihwal)
 
+    HADOOP-7984. Add hadoop --loglevel option to change log level.
+    (Akira AJISAKA via cnauroth)
+
   IMPROVEMENTS
 
     HADOOP-11156. DelegateToFileSystem should implement

hadoop-common-project/hadoop-common/src/main/bin/hadoop

@@ -26,7 +26,7 @@ HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
 . $HADOOP_LIBEXEC_DIR/hadoop-config.sh
 
 function print_usage(){
-  echo "Usage: hadoop [--config confdir] COMMAND"
+  echo "Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND"
   echo "       where COMMAND is one of:"
   echo "  fs                   run a generic filesystem user client"
   echo "  version              print the version"

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd

@@ -77,6 +77,16 @@ if "%1" == "--config" (
   shift
 )
 
+@rem
+@rem Set log level. Default to INFO.
+@rem
+
+if "%1" == "--loglevel" (
+  set HADOOP_LOGLEVEL=%2
+  shift
+  shift
+)
+
 @rem
 @rem check to see it is specified whether to use the slaves or the
 @rem masters file
@@ -157,8 +167,12 @@ if not defined HADOOP_LOGFILE (
   set HADOOP_LOGFILE=hadoop.log
 )
 
+if not defined HADOOP_LOGLEVEL (
+  set HADOOP_LOGLEVEL=INFO
+)
+
 if not defined HADOOP_ROOT_LOGGER (
-  set HADOOP_ROOT_LOGGER=INFO,console
+  set HADOOP_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console
 )
 
 @rem

hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh

@@ -86,6 +86,18 @@ then
     fi
 fi
 
+# Set log level. Default to INFO.
+if [ $# -gt 1 ]
+then
+  if [ "--loglevel" = "$1" ]
+  then
+    shift
+    HADOOP_LOGLEVEL=$1
+    shift
+  fi
+fi
+HADOOP_LOGLEVEL="${HADOOP_LOGLEVEL:-INFO}"
+ 
 # Allow alternate conf dir location.
 if [ -e "${HADOOP_PREFIX}/conf/hadoop-env.sh" ]; then
   DEFAULT_CONF_DIR="conf"
@@ -235,7 +247,7 @@ HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING"
-HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-INFO,console}"
+HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.root.logger=${HADOOP_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
 if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   HADOOP_OPTS="$HADOOP_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
   export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH

hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd

@@ -68,6 +68,10 @@ call :updatepath %HADOOP_BIN_PATH%
     shift
     shift
   )
+  if "%1" == "--loglevel" (
+    shift
+    shift
+  )
 
   set hadoop-command=%1
   if not defined hadoop-command (
@@ -218,6 +222,10 @@ call :updatepath %HADOOP_BIN_PATH%
     shift
     shift
   )
+  if "%1" == "--loglevel" (
+    shift
+    shift
+  )
   if [%2] == [] goto :eof
   shift
   set _arguments=
@@ -236,7 +244,7 @@ call :updatepath %HADOOP_BIN_PATH%
   goto :eof
 
 :print_usage
-  @echo Usage: hadoop [--config confdir] COMMAND
+  @echo Usage: hadoop [--config confdir] [--loglevel loglevel] COMMAND
   @echo where COMMAND is one of:
   @echo   fs                   run a generic filesystem user client
   @echo   version              print the version

hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm

@@ -27,7 +27,8 @@ Overview
    hadoop script without any arguments prints the description for all
    commands.
 
-   Usage: <<<hadoop [--config confdir] [COMMAND] [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
+   Usage: <<<hadoop [--config confdir] [--loglevel loglevel] [COMMAND]
+             [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
 
    Hadoop has an option parsing framework that employs parsing generic
    options as well as running classes.
@@ -37,6 +38,10 @@ Overview
 *-----------------------+---------------+
 | <<<--config confdir>>>| Overwrites the default Configuration directory.  Default is <<<${HADOOP_HOME}/conf>>>.
 *-----------------------+---------------+
+| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are
+|                       | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
+|                       | Default is INFO.
+*-----------------------+---------------+
 | GENERIC_OPTIONS       | The common set of options supported by multiple commands.
 | COMMAND_OPTIONS       | Various commands with their options are described in the following sections. The commands have been grouped into User Commands and Administration Commands.
 *-----------------------+---------------+

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs

@@ -34,7 +34,7 @@ HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
 . $HADOOP_LIBEXEC_DIR/hdfs-config.sh
 
 function print_usage(){
-  echo "Usage: hdfs [--config confdir] COMMAND"
+  echo "Usage: hdfs [--config confdir] [--loglevel loglevel] COMMAND"
   echo "       where COMMAND is one of:"
   echo "  dfs                  run a filesystem command on the file systems supported in Hadoop."
   echo "  namenode -format     format the DFS filesystem"

hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd

@@ -34,6 +34,10 @@ if "%1" == "--config" (
   shift
   shift
 )
+if "%1" == "--loglevel" (
+  shift
+  shift
+)
 
 :main
   if exist %HADOOP_CONF_DIR%\hadoop-env.cmd (
@@ -165,6 +169,10 @@ goto :eof
     shift
     shift
   )
+  if "%1" == "--loglevel" (
+    shift
+    shift
+  )
   if [%2] == [] goto :eof
   shift
   set _hdfsarguments=
@@ -183,7 +191,7 @@ goto :eof
   goto :eof
 
 :print_usage
-  @echo Usage: hdfs [--config confdir] COMMAND
+  @echo Usage: hdfs [--config confdir] [--loglevel loglevel] COMMAND
   @echo        where COMMAND is one of:
   @echo   dfs                  run a filesystem command on the file systems supported in Hadoop.
   @echo   namenode -format     format the DFS filesystem

hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm

@@ -26,8 +26,8 @@ HDFS Commands Guide
    hdfs script without any arguments prints the description for all
    commands.
 
-   Usage: <<<hdfs [--config confdir] [COMMAND] [GENERIC_OPTIONS]
+   Usage: <<<hdfs [--config confdir] [--loglevel loglevel] [COMMAND]
-          [COMMAND_OPTIONS]>>>
+          [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
 
    Hadoop has an option parsing framework that employs parsing generic options
    as well as running classes.
@@ -38,6 +38,10 @@ HDFS Commands Guide
 | <<<--config confdir>>>| Overwrites the default Configuration directory.
 |                       | Default is <<<${HADOOP_HOME}/conf>>>.
 *-----------------------+---------------+
+| <<<--loglevel loglevel>>>| Overwrites the log level. Valid log levels are
+|                       | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
+|                       | Default is INFO.
+*-----------------------+---------------+
 | GENERIC_OPTIONS       | The common set of options supported by multiple
 |                       | commands. Full list is
 |                       | {{{../hadoop-common/CommandsManual.html#Generic_Options}here}}.

hadoop-mapreduce-project/bin/mapred

@@ -28,7 +28,7 @@ else
 fi
 
 function print_usage(){
-  echo "Usage: mapred [--config confdir] COMMAND"
+  echo "Usage: mapred [--config confdir] [--loglevel loglevel] COMMAND"
   echo "       where COMMAND is one of:"
   echo "  pipes                run a Pipes job"
   echo "  job                  manipulate MapReduce jobs"

hadoop-mapreduce-project/bin/mapred-config.sh

@@ -43,7 +43,7 @@ fi
 # The following defaults are useful when somebody directly invokes bin/mapred.
 HADOOP_MAPRED_LOG_DIR=${HADOOP_MAPRED_LOG_DIR:-${HADOOP_MAPRED_HOME}/logs}
 HADOOP_MAPRED_LOGFILE=${HADOOP_MAPRED_LOGFILE:-hadoop.log}
-HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-INFO,console}
+HADOOP_MAPRED_ROOT_LOGGER=${HADOOP_MAPRED_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}
 
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_MAPRED_LOG_DIR"
 HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_MAPRED_LOGFILE"

hadoop-mapreduce-project/bin/mapred.cmd

@@ -36,6 +36,10 @@ if "%1" == "--config" (
   shift
   shift
 )
+if "%1" == "--loglevel" (
+  shift
+  shift
+)
 
 :main
   if exist %MAPRED_CONF_DIR%\mapred-env.cmd (
@@ -162,6 +166,10 @@ goto :eof
     shift
     shift
   )
+  if "%1" == "--loglevel" (
+    shift
+    shift
+  )
   shift
   set _mapredarguments=
   :MakeCmdArgsLoop 
@@ -184,7 +192,7 @@ goto :eof
   goto print_usage
 
 :print_usage
-  @echo Usage: mapred [--config confdir] COMMAND
+  @echo Usage: mapred [--config confdir] [--loglevel loglevel] COMMAND
   @echo        where COMMAND is one of:
   @echo   job                  manipulate MapReduce jobs
   @echo   queue                get information regarding JobQueues

hadoop-mapreduce-project/conf/mapred-env.cmd

@@ -16,5 +16,5 @@
 
 set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
 
-set HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+set HADOOP_MAPRED_ROOT_LOGGER=%HADOOP_LOGLEVEL%,RFA
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm

@@ -28,7 +28,7 @@ MapReduce Commands Guide
   MapReduce commands are invoked by the <<<bin/mapred>>> script. Running the
   script without any arguments prints the description for all commands.
 
-   Usage: <<<mapred [--config confdir] COMMAND>>>
+   Usage: <<<mapred [--config confdir] [--loglevel loglevel] COMMAND>>>
 
    MapReduce has an option parsing framework that employs parsing generic
    options as well as running classes.
@@ -39,6 +39,9 @@ MapReduce Commands Guide
 | --config confdir | Overwrites the default Configuration directory. Default
 |                  | is $\{HADOOP_PREFIX\}/conf.
 *-------------------------+---------------------------------------------------+
+| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL,
+|                     | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO.
+*-------------------------+---------------------------------------------------+
 | COMMAND COMMAND_OPTIONS | Various commands with their options are described
 |                         | in the following sections. The commands have been
 |                         | grouped into {{User Commands}} and

hadoop-yarn-project/hadoop-yarn/bin/yarn

@@ -59,7 +59,7 @@ HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
 . $HADOOP_LIBEXEC_DIR/yarn-config.sh
 
 function print_usage(){
-  echo "Usage: yarn [--config confdir] COMMAND"
+  echo "Usage: yarn [--config confdir] [--loglevel] COMMAND"
   echo "where COMMAND is one of:"
   echo "  resourcemanager -format-state-store   deletes the RMStateStore"
   echo "  resourcemanager                       run the ResourceManager"
@@ -276,8 +276,8 @@ YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE"
 YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE"
 YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$HADOOP_YARN_HOME"
 YARN_OPTS="$YARN_OPTS -Dhadoop.home.dir=$HADOOP_YARN_HOME"
-YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
-YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}"
+YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-${HADOOP_LOGLEVEL},console}"
 if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
 fi  

hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd

@@ -64,6 +64,10 @@ if "%1" == "--config" (
   shift
   shift
 )
+if "%1" == "--loglevel" (
+  shift
+  shift
+)
 
 :main
   if exist %YARN_CONF_DIR%\yarn-env.cmd (
@@ -273,6 +277,10 @@ goto :eof
     shift
     shift
   )
+  if "%1" == "--loglevel" (
+    shift
+    shift
+  )
   if [%2] == [] goto :eof
   shift
   set _yarnarguments=
@@ -291,7 +299,7 @@ goto :eof
   goto :eof
 
 :print_usage
-  @echo Usage: yarn [--config confdir] COMMAND
+  @echo Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
   @echo        where COMMAND is one of:
   @echo   resourcemanager      run the ResourceManager
   @echo   nodemanager          run a nodemanager on each slave

hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd

@@ -42,7 +42,7 @@ if not defined YARN_POLICYFILE (
 )
 
 if not defined YARN_ROOT_LOGGER (
-  set YARN_ROOT_LOGGER=INFO,console
+  set YARN_ROOT_LOGGER=%HADOOP_LOGLEVEL%,console
 )
 
 set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR%

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm

@@ -26,7 +26,7 @@ Yarn Commands
   without any arguments prints the description for all commands.
 
 ------
-Usage: yarn [--config confdir] COMMAND
+Usage: yarn [--config confdir] [--loglevel loglevel] COMMAND
 ------
 
   Yarn has an option parsing framework that employs parsing generic options as
@@ -38,6 +38,9 @@ Usage: yarn [--config confdir] COMMAND
 | --config confdir | Overwrites the default Configuration directory. Default
 |                  | is $\{HADOOP_PREFIX\}/conf.
 *---------------+--------------+
+| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL,
+|                     | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO.
+*---------------+--------------+
 | COMMAND COMMAND_OPTIONS | Various commands with their options are described
 |                         | in the following sections. The commands have been
 |                         | grouped into {{User Commands}} and