Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13209. replace slaves with workers (John Smith via aw)

This commit is contained in:
Allen Wittenauer 2016-06-28 05:53:03 -07:00
parent 2a0082c51d
commit 23c3ff85a9
40 changed files with 505 additions and 482 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-common-project/hadoop-common/src/main/bin/hadoop
View File

@ -28,7 +28,7 @@ function hadoop_usage
hadoop_add_option "hostnames list[,of,host,names]" "hosts to use in slave mode" hadoop_add_option "hostnames list[,of,host,names]" "hosts to use in slave mode"
hadoop_add_option "loglevel level" "set the log4j level for this command" hadoop_add_option "loglevel level" "set the log4j level for this command"
hadoop_add_option "hosts filename" "list of hosts to use in slave mode" hadoop_add_option "hosts filename" "list of hosts to use in slave mode"
hadoop_add_option "slaves" "turn on slave mode" hadoop_add_option "workers" "turn on worker mode"
hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability" hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability"
hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries" hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries"
@ -205,8 +205,8 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}" hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}" hadoop_common_worker_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}"
exit $? exit $?
fi fi

4
hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd
View File

@ -80,12 +80,12 @@ if "%1" == "--config" (
) )
@rem @rem
@rem check to see it is specified whether to use the slaves or the @rem check to see it is specified whether to use the workers or the
@rem masters file @rem masters file
@rem @rem
if "%1" == "--hosts" ( if "%1" == "--hosts" (
set HADOOP_SLAVES=%HADOOP_CONF_DIR%\%2 set HADOOP_WORKERS=%HADOOP_CONF_DIR%\%2
shift shift
shift shift
) )

4
hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh
View File

@ -113,6 +113,10 @@ hadoop_exec_userfuncs
hadoop_exec_user_hadoopenv hadoop_exec_user_hadoopenv
hadoop_verify_confdir hadoop_verify_confdir
hadoop_deprecate_envvar HADOOP_SLAVES HADOOP_WORKERS
hadoop_deprecate_envvar HADOOP_SLAVE_NAMES HADOOP_WORKER_NAMES
hadoop_deprecate_envvar HADOOP_SLAVE_SLEEP HADOOP_WORKER_SLEEP
# do all the OS-specific startup bits here # do all the OS-specific startup bits here
# this allows us to get a decent JAVA_HOME, # this allows us to get a decent JAVA_HOME,
# call crle for LD_LIBRARY_PATH, etc. # call crle for LD_LIBRARY_PATH, etc.

6
hadoop-common-project/hadoop-common/src/main/bin/hadoop-daemons.sh
View File

@ -57,13 +57,13 @@ else
fi fi
hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated." hadoop_error "WARNING: Use of this script to ${daemonmode} HDFS daemons is deprecated."
hadoop_error "WARNING: Attempting to execute replacement \"hdfs --slaves --daemon ${daemonmode}\" instead." hadoop_error "WARNING: Attempting to execute replacement \"hdfs --workers --daemon ${daemonmode}\" instead."
# #
# Original input was usually: # Original input was usually:
# hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options) # hadoop-daemons.sh (shell options) (start|stop) (datanode|...) (daemon options)
# we're going to turn this into # we're going to turn this into
# hdfs --slaves --daemon (start|stop) (rest of options) # hdfs --workers --daemon (start|stop) (rest of options)
# #
for (( i = 0; i < ${#HADOOP_USER_PARAMS[@]}; i++ )) for (( i = 0; i < ${#HADOOP_USER_PARAMS[@]}; i++ ))
do do
@ -74,4 +74,4 @@ do
fi fi
done done
${hdfsscript} --slaves --daemon "${daemonmode}" "${HADOOP_USER_PARAMS[@]}" ${hdfsscript} --workers --daemon "${daemonmode}" "${HADOOP_USER_PARAMS[@]}"

73
hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
View File

@ -602,25 +602,25 @@ function hadoop_basic_init
HADOOP_SSH_PARALLEL=${HADOOP_SSH_PARALLEL:-10} HADOOP_SSH_PARALLEL=${HADOOP_SSH_PARALLEL:-10}
} }
## @description Set the slave support information to the contents ## @description Set the worker support information to the contents
## @description of `filename` ## @description of `filename`
## @audience public ## @audience public
## @stability stable ## @stability stable
## @replaceable no ## @replaceable no
## @param filename ## @param filename
## @return will exit if file does not exist ## @return will exit if file does not exist
function hadoop_populate_slaves_file function hadoop_populate_workers_file
{ {
local slavesfile=$1 local workersfile=$1
shift shift
if [[ -f "${slavesfile}" ]]; then if [[ -f "${workersfile}" ]]; then
# shellcheck disable=2034 # shellcheck disable=2034
HADOOP_SLAVES="${slavesfile}" HADOOP_WORKERS="${workersfile}"
elif [[ -f "${HADOOP_CONF_DIR}/${slavesfile}" ]]; then elif [[ -f "${HADOOP_CONF_DIR}/${workersfile}" ]]; then
# shellcheck disable=2034 # shellcheck disable=2034
HADOOP_SLAVES="${HADOOP_CONF_DIR}/${slavesfile}" HADOOP_WORKERS="${HADOOP_CONF_DIR}/${workersfile}"
else else
hadoop_error "ERROR: Cannot find hosts file \"${slavesfile}\"" hadoop_error "ERROR: Cannot find hosts file \"${workersfile}\""
hadoop_exit_with_usage 1 hadoop_exit_with_usage 1
fi fi
} }
@ -669,14 +669,14 @@ function hadoop_actual_ssh
{ {
# we are passing this function to xargs # we are passing this function to xargs
# should get hostname followed by rest of command line # should get hostname followed by rest of command line
local slave=$1 local worker=$1
shift shift
# shellcheck disable=SC2086 # shellcheck disable=SC2086
ssh ${HADOOP_SSH_OPTS} ${slave} $"${@// /\\ }" 2>&1 | sed "s/^/$slave: /" ssh ${HADOOP_SSH_OPTS} ${worker} $"${@// /\\ }" 2>&1 | sed "s/^/$worker: /"
} }
## @description Connect to ${HADOOP_SLAVES} or ${HADOOP_SLAVE_NAMES} ## @description Connect to ${HADOOP_WORKERS} or ${HADOOP_WORKER_NAMES}
## @description and execute command. ## @description and execute command.
## @audience private ## @audience private
## @stability evolving ## @stability evolving
@ -687,45 +687,52 @@ function hadoop_connect_to_hosts
{ {
# shellcheck disable=SC2124 # shellcheck disable=SC2124
local params="$@" local params="$@"
local slave_file local worker_file
local tmpslvnames local tmpslvnames
# #
# ssh (or whatever) to a host # ssh (or whatever) to a host
# #
# User can specify hostnames or a file where the hostnames are (not both) # User can specify hostnames or a file where the hostnames are (not both)
if [[ -n "${HADOOP_SLAVES}" && -n "${HADOOP_SLAVE_NAMES}" ]] ; then if [[ -n "${HADOOP_WORKERS}" && -n "${HADOOP_WORKER_NAMES}" ]] ; then
hadoop_error "ERROR: Both HADOOP_SLAVES and HADOOP_SLAVE_NAME were defined. Aborting." hadoop_error "ERROR: Both HADOOP_WORKERS and HADOOP_WORKER_NAME were defined. Aborting."
exit 1 exit 1
elif [[ -z "${HADOOP_SLAVE_NAMES}" ]]; then elif [[ -z "${HADOOP_WORKER_NAMES}" ]]; then
slave_file=${HADOOP_SLAVES:-${HADOOP_CONF_DIR}/slaves} if [[ -n "${HADOOP_WORKERS}" ]]; then
worker_file=${HADOOP_WORKERS}
elif [[ -f "${HADOOP_CONF_DIR}/workers" ]]; then
worker_file=${HADOOP_CONF_DIR}/workers
elif [[ -f "${HADOOP_CONF_DIR}/slaves" ]]; then
hadoop_error "WARNING: 'slaves' file has been deprecated. Please use 'workers' file instead."
worker_file=${HADOOP_CONF_DIR}/slaves
fi
fi fi
# if pdsh is available, let's use it. otherwise default # if pdsh is available, let's use it. otherwise default
# to a loop around ssh. (ugh) # to a loop around ssh. (ugh)
if [[ -e '/usr/bin/pdsh' ]]; then if [[ -e '/usr/bin/pdsh' ]]; then
if [[ -z "${HADOOP_SLAVE_NAMES}" ]] ; then if [[ -z "${HADOOP_WORKER_NAMES}" ]] ; then
# if we were given a file, just let pdsh deal with it. # if we were given a file, just let pdsh deal with it.
# shellcheck disable=SC2086 # shellcheck disable=SC2086
PDSH_SSH_ARGS_APPEND="${HADOOP_SSH_OPTS}" pdsh \ PDSH_SSH_ARGS_APPEND="${HADOOP_SSH_OPTS}" pdsh \
-f "${HADOOP_SSH_PARALLEL}" -w ^"${slave_file}" $"${@// /\\ }" 2>&1 -f "${HADOOP_SSH_PARALLEL}" -w ^"${worker_file}" $"${@// /\\ }" 2>&1
else else
# no spaces allowed in the pdsh arg host list # no spaces allowed in the pdsh arg host list
# shellcheck disable=SC2086 # shellcheck disable=SC2086
tmpslvnames=$(echo ${SLAVE_NAMES} | tr -s ' ' ,) tmpslvnames=$(echo ${HADOOP_WORKER_NAMES} | tr -s ' ' ,)
PDSH_SSH_ARGS_APPEND="${HADOOP_SSH_OPTS}" pdsh \ PDSH_SSH_ARGS_APPEND="${HADOOP_SSH_OPTS}" pdsh \
-f "${HADOOP_SSH_PARALLEL}" \ -f "${HADOOP_SSH_PARALLEL}" \
-w "${tmpslvnames}" $"${@// /\\ }" 2>&1 -w "${tmpslvnames}" $"${@// /\\ }" 2>&1
fi fi
else else
if [[ -z "${HADOOP_SLAVE_NAMES}" ]]; then if [[ -z "${HADOOP_WORKER_NAMES}" ]]; then
HADOOP_SLAVE_NAMES=$(sed 's/#.*$//;/^$/d' "${slave_file}") HADOOP_WORKER_NAMES=$(sed 's/#.*$//;/^$/d' "${worker_file}")
fi fi
hadoop_connect_to_hosts_without_pdsh "${params}" hadoop_connect_to_hosts_without_pdsh "${params}"
fi fi
} }
## @description Connect to ${SLAVE_NAMES} and execute command ## @description Connect to ${HADOOP_WORKER_NAMES} and execute command
## @description under the environment which does not support pdsh. ## @description under the environment which does not support pdsh.
## @audience private ## @audience private
## @stability evolving ## @stability evolving
@ -736,24 +743,24 @@ function hadoop_connect_to_hosts_without_pdsh
{ {
# shellcheck disable=SC2124 # shellcheck disable=SC2124
local params="$@" local params="$@"
local slaves=(${HADOOP_SLAVE_NAMES}) local workers=(${HADOOP_WORKER_NAMES})
for (( i = 0; i < ${#slaves[@]}; i++ )) for (( i = 0; i < ${#workers[@]}; i++ ))
do do
if (( i != 0 && i % HADOOP_SSH_PARALLEL == 0 )); then if (( i != 0 && i % HADOOP_SSH_PARALLEL == 0 )); then
wait wait
fi fi
# shellcheck disable=SC2086 # shellcheck disable=SC2086
hadoop_actual_ssh "${slaves[$i]}" ${params} & hadoop_actual_ssh "${workers[$i]}" ${params} &
done done
wait wait
} }
## @description Utility routine to handle --slaves mode ## @description Utility routine to handle --workers mode
## @audience private ## @audience private
## @stability evolving ## @stability evolving
## @replaceable yes ## @replaceable yes
## @param commandarray ## @param commandarray
function hadoop_common_slave_mode_execute function hadoop_common_worker_mode_execute
{ {
# #
# input should be the command line as given by the user # input should be the command line as given by the user
@ -761,13 +768,13 @@ function hadoop_common_slave_mode_execute
# #
local argv=("$@") local argv=("$@")
# if --slaves is still on the command line, remove it # if --workers is still on the command line, remove it
# to prevent loops # to prevent loops
# Also remove --hostnames and --hosts along with arg values # Also remove --hostnames and --hosts along with arg values
local argsSize=${#argv[@]}; local argsSize=${#argv[@]};
for (( i = 0; i < argsSize; i++ )) for (( i = 0; i < argsSize; i++ ))
do do
if [[ "${argv[$i]}" =~ ^--slaves$ ]]; then if [[ "${argv[$i]}" =~ ^--workers$ ]]; then
unset argv[$i] unset argv[$i]
elif [[ "${argv[$i]}" =~ ^--hostnames$ ]] || elif [[ "${argv[$i]}" =~ ^--hostnames$ ]] ||
[[ "${argv[$i]}" =~ ^--hosts$ ]]; then [[ "${argv[$i]}" =~ ^--hosts$ ]]; then
@ -2051,13 +2058,13 @@ function hadoop_parse_args
--hostnames) --hostnames)
shift shift
# shellcheck disable=SC2034 # shellcheck disable=SC2034
HADOOP_SLAVE_NAMES="$1" HADOOP_WORKER_NAMES="$1"
shift shift
((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2)) ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
;; ;;
--hosts) --hosts)
shift shift
hadoop_populate_slaves_file "$1" hadoop_populate_workers_file "$1"
shift shift
((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2)) ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
;; ;;
@ -2068,10 +2075,10 @@ function hadoop_parse_args
shift shift
((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2)) ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
;; ;;
--slaves) --workers)
shift shift
# shellcheck disable=SC2034 # shellcheck disable=SC2034
HADOOP_SLAVE_MODE=true HADOOP_WORKER_MODE=true
((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1)) ((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1))
;; ;;
*) *)

10
hadoop-common-project/hadoop-common/src/main/bin/slaves.sh → hadoop-common-project/hadoop-common/src/main/bin/workers.sh
View File

@ -16,20 +16,20 @@
# limitations under the License. # limitations under the License.
# Run a shell command on all slave hosts. # Run a shell command on all worker hosts.
# #
# Environment Variables # Environment Variables
# #
# HADOOP_SLAVES File naming remote hosts. # HADOOP_WORKERS File naming remote hosts.
# Default is ${HADOOP_CONF_DIR}/slaves. # Default is ${HADOOP_CONF_DIR}/workers.
# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. # HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf.
# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. # HADOOP_WORKER_SLEEP Seconds to sleep between spawning remote commands.
# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. # HADOOP_SSH_OPTS Options passed to ssh when running remote commands.
## ##
function hadoop_usage function hadoop_usage
{ {
echo "Usage: slaves.sh [--config confdir] command..." echo "Usage: workers.sh [--config confdir] command..."
} }
# let's locate libexec... # let's locate libexec...

4
hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
View File

@ -169,8 +169,8 @@ esac
# export HADOOP_SSH_PARALLEL=10 # export HADOOP_SSH_PARALLEL=10
# Filename which contains all of the hosts for any remote execution # Filename which contains all of the hosts for any remote execution
# helper scripts # such as slaves.sh, start-dfs.sh, etc. # helper scripts # such as workers.sh, start-dfs.sh, etc.
# export HADOOP_SLAVES="${HADOOP_CONF_DIR}/slaves" # export HADOOP_WORKERS="${HADOOP_CONF_DIR}/workers"
### ###
# Options for all daemons # Options for all daemons

4
hadoop-common-project/hadoop-common/src/main/conf/hadoop-user-functions.sh.example
View File

@ -87,7 +87,7 @@
#} #}
# #
# Example: efficient command execution for the slaves # Example: efficient command execution for the workers
# #
# To improve performance, you can use xargs -P # To improve performance, you can use xargs -P
# instead of the for loop, if supported. # instead of the for loop, if supported.
@ -108,7 +108,7 @@
# # list with each hostname read from stdin/pipe. But it consider one # # list with each hostname read from stdin/pipe. But it consider one
# # line as one argument while reading from stdin/pipe. So place each # # line as one argument while reading from stdin/pipe. So place each
# # hostname in different lines while passing via pipe. # # hostname in different lines while passing via pipe.
# tmpslvnames=$(echo "${HADOOP_SLAVE_NAMES}" | tr ' ' '\n' ) # tmpslvnames=$(echo "${HADOOP_WORKER_NAMES}" | tr ' ' '\n' )
# echo "${tmpslvnames}" | \ # echo "${tmpslvnames}" | \
# xargs -n 1 -P"${HADOOP_SSH_PARALLEL}" \ # xargs -n 1 -P"${HADOOP_SSH_PARALLEL}" \
# -I {} bash -c -- "hadoop_actual_ssh {} ${params}" # -I {} bash -c -- "hadoop_actual_ssh {} ${params}"

0
hadoop-yarn-project/hadoop-yarn/conf/slaves → hadoop-common-project/hadoop-common/src/main/conf/workers
View File

14
hadoop-common-project/hadoop-common/src/main/java/overview.html
View File

@ -141,8 +141,8 @@ host and port. This is specified with the configuration property
href="../mapred-default.html#mapred.job.tracker">mapred.job.tracker</a></tt>. href="../mapred-default.html#mapred.job.tracker">mapred.job.tracker</a></tt>.
</li> </li>
<li>A <em>slaves</em> file that lists the names of all the hosts in <li>A <em>workers</em> file that lists the names of all the hosts in
the cluster. The default slaves file is <tt>conf/slaves</tt>. the cluster. The default workers file is <tt>conf/workers</tt>.
</ol> </ol>
@ -245,7 +245,7 @@ href="../hdfs-default.html#dfs.name.dir">dfs.name.dir</a></tt> and
<tt><a href="../hdfs-default.html#dfs.data.dir">dfs.data.dir</a> <tt><a href="../hdfs-default.html#dfs.data.dir">dfs.data.dir</a>
in <tt>conf/hdfs-site.xml</tt>. in <tt>conf/hdfs-site.xml</tt>.
</tt>These are local directories used to hold distributed filesystem </tt>These are local directories used to hold distributed filesystem
data on the master node and slave nodes respectively. Note data on the master node and worker nodes respectively. Note
that <tt>dfs.data.dir</tt> may contain a space- or comma-separated that <tt>dfs.data.dir</tt> may contain a space- or comma-separated
list of directory names, so that data may be stored on multiple local list of directory names, so that data may be stored on multiple local
devices.</li> devices.</li>
@ -260,13 +260,13 @@ and <tt><a
href="../mapred-default.html#mapred.reduce.tasks">mapred.reduce.tasks</a></tt> href="../mapred-default.html#mapred.reduce.tasks">mapred.reduce.tasks</a></tt>
in <tt>conf/mapred-site.xml</tt>. in <tt>conf/mapred-site.xml</tt>.
As a rule of thumb, use 10x the As a rule of thumb, use 10x the
number of slave processors for <tt>mapred.map.tasks</tt>, and 2x the number of worker processors for <tt>mapred.map.tasks</tt>, and 2x the
number of slave processors for <tt>mapred.reduce.tasks</tt>.</li> number of worker processors for <tt>mapred.reduce.tasks</tt>.</li>
</ol> </ol>
<p>Finally, list all slave hostnames or IP addresses in your <p>Finally, list all worker hostnames or IP addresses in your
<tt>conf/slaves</tt> file, one per line. Then format your filesystem <tt>conf/workers</tt> file, one per line. Then format your filesystem
and start your cluster on your master node, as above. and start your cluster on your master node, as above.
</body> </body>

16
hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
View File

@ -37,7 +37,7 @@ Installing a Hadoop cluster typically involves unpacking the software on all the
Typically one machine in the cluster is designated as the NameNode and another machine the as ResourceManager, exclusively. These are the masters. Other services (such as Web App Proxy Server and MapReduce Job History server) are usually run either on dedicated hardware or on shared infrastrucutre, depending upon the load. Typically one machine in the cluster is designated as the NameNode and another machine the as ResourceManager, exclusively. These are the masters. Other services (such as Web App Proxy Server and MapReduce Job History server) are usually run either on dedicated hardware or on shared infrastrucutre, depending upon the load.
The rest of the machines in the cluster act as both DataNode and NodeManager. These are the slaves. The rest of the machines in the cluster act as both DataNode and NodeManager. These are the workers.
Configuring Hadoop in Non-Secure Mode Configuring Hadoop in Non-Secure Mode
------------------------------------- -------------------------------------
@ -216,12 +216,12 @@ The health checker script is not supposed to give ERROR if only some of the loca
Slaves File Slaves File
----------- -----------
List all slave hostnames or IP addresses in your `etc/hadoop/slaves` file, one per line. Helper scripts (described below) will use the `etc/hadoop/slaves` file to run commands on many hosts at once. It is not used for any of the Java-based Hadoop configuration. In order to use this functionality, ssh trusts (via either passphraseless ssh or some other means, such as Kerberos) must be established for the accounts used to run Hadoop. List all worker hostnames or IP addresses in your `etc/hadoop/workers` file, one per line. Helper scripts (described below) will use the `etc/hadoop/workers` file to run commands on many hosts at once. It is not used for any of the Java-based Hadoop configuration. In order to use this functionality, ssh trusts (via either passphraseless ssh or some other means, such as Kerberos) must be established for the accounts used to run Hadoop.
Hadoop Rack Awareness Hadoop Rack Awareness
--------------------- ---------------------
Many Hadoop components are rack-aware and take advantage of the network topology for performance and safety. Hadoop daemons obtain the rack information of the slaves in the cluster by invoking an administrator configured module. See the [Rack Awareness](./RackAwareness.html) documentation for more specific information. Many Hadoop components are rack-aware and take advantage of the network topology for performance and safety. Hadoop daemons obtain the rack information of the workers in the cluster by invoking an administrator configured module. See the [Rack Awareness](./RackAwareness.html) documentation for more specific information.
It is highly recommended configuring rack awareness prior to starting HDFS. It is highly recommended configuring rack awareness prior to starting HDFS.
@ -253,7 +253,7 @@ Start a HDFS DataNode with the following command on each designated node as *hdf
[hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start datanode [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon start datanode
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes can be started with a utility script. As *hdfs*: If `etc/hadoop/workers` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes can be started with a utility script. As *hdfs*:
[hdfs]$ $HADOOP_HOME/sbin/start-dfs.sh [hdfs]$ $HADOOP_HOME/sbin/start-dfs.sh
@ -269,7 +269,7 @@ Start a standalone WebAppProxy server. Run on the WebAppProxy server as *yarn*.
[yarn]$ $HADOOP_HOME/bin/yarn --daemon start proxyserver [yarn]$ $HADOOP_HOME/bin/yarn --daemon start proxyserver
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be started with a utility script. As *yarn*: If `etc/hadoop/workers` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be started with a utility script. As *yarn*:
[yarn]$ $HADOOP_HOME/sbin/start-yarn.sh [yarn]$ $HADOOP_HOME/sbin/start-yarn.sh
@ -287,7 +287,7 @@ Run a script to stop a DataNode as *hdfs*:
[hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop datanode [hdfs]$ $HADOOP_HOME/bin/hdfs --daemon stop datanode
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes may be stopped with a utility script. As *hdfs*: If `etc/hadoop/workers` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes may be stopped with a utility script. As *hdfs*:
[hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh [hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh
@ -295,11 +295,11 @@ Stop the ResourceManager with the following command, run on the designated Resou
[yarn]$ $HADOOP_HOME/bin/yarn --daemon stop resourcemanager [yarn]$ $HADOOP_HOME/bin/yarn --daemon stop resourcemanager
Run a script to stop a NodeManager on a slave as *yarn*: Run a script to stop a NodeManager on a worker as *yarn*:
[yarn]$ $HADOOP_HOME/bin/yarn --daemon stop nodemanager [yarn]$ $HADOOP_HOME/bin/yarn --daemon stop nodemanager
If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be stopped with a utility script. As *yarn*: If `etc/hadoop/workers` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be stopped with a utility script. As *yarn*:
[yarn]$ $HADOOP_HOME/sbin/stop-yarn.sh [yarn]$ $HADOOP_HOME/sbin/stop-yarn.sh

6
hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
View File

@ -43,10 +43,10 @@ All of the shell commands will accept a common set of options. For some commands
| `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. For commands that do not support daemonization, this option is ignored. | | `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. For commands that do not support daemonization, this option is ignored. |
| `--debug` | Enables shell level configuration debugging information | | `--debug` | Enables shell level configuration debugging information |
| `--help` | Shell script usage information. | | `--help` | Shell script usage information. |
| `--hostnames` | When `--slaves` is used, override the slaves file with a space delimited list of hostnames where to execute a multi-host subcommand. If `--slaves` is not used, this option is ignored. | | `--hostnames` | When `--workers` is used, override the workers file with a space delimited list of hostnames where to execute a multi-host subcommand. If `--workers` is not used, this option is ignored. |
| `--hosts` | When `--slaves` is used, override the slaves file with another file that contains a list of hostnames where to execute a multi-host subcommand. If `--slaves` is not used, this option is ignored. | | `--hosts` | When `--workers` is used, override the workers file with another file that contains a list of hostnames where to execute a multi-host subcommand. If `--workers` is not used, this option is ignored. |
| `--loglevel loglevel` | Overrides the log level. Valid log levels are FATAL, ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO. | | `--loglevel loglevel` | Overrides the log level. Valid log levels are FATAL, ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO. |
| `--slaves` | If possible, execute this command on all hosts in the `slaves` file. | | `--workers` | If possible, execute this command on all hosts in the `workers` file. |
### Generic Options ### Generic Options

4
hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
View File

@ -22,7 +22,7 @@ will use rack awareness for fault tolerance by placing one block
replica on a different rack. This provides data availability in the replica on a different rack. This provides data availability in the
event of a network switch failure or partition within the cluster. event of a network switch failure or partition within the cluster.
Hadoop master daemons obtain the rack id of the cluster slaves by Hadoop master daemons obtain the rack id of the cluster workers by
invoking either an external script or java class as specified by invoking either an external script or java class as specified by
configuration files. Using either the java class or external script configuration files. Using either the java class or external script
for topology, output must adhere to the java for topology, output must adhere to the java
@ -40,7 +40,7 @@ in the configuration file. An example, NetworkTopology.java, is
included with the hadoop distribution and can be customized by the included with the hadoop distribution and can be customized by the
Hadoop administrator. Using a Java class instead of an external script Hadoop administrator. Using a Java class instead of an external script
has a performance benefit in that Hadoop doesn't need to fork an has a performance benefit in that Hadoop doesn't need to fork an
external process when a new slave node registers itself. external process when a new worker node registers itself.
If implementing an external script, it will be specified with the If implementing an external script, it will be specified with the
**net.topology.script.file.name** parameter in the configuration **net.topology.script.file.name** parameter in the configuration

16
hadoop-common-project/hadoop-common/src/test/scripts/hadoop_ssh.bats
View File

@ -30,22 +30,22 @@ load hadoop-functions_test_helper
hadoop_connect_to_hosts_without_pdsh hadoop_connect_to_hosts_without_pdsh
} }
@test "hadoop_common_slave_mode_execute (--slaves 1)" { @test "hadoop_common_worker_mode_execute (--workers 1)" {
run hadoop_common_slave_mode_execute --slaves command run hadoop_common_worker_mode_execute --workers command
[ "${output}" = "command" ] [ "${output}" = "command" ]
} }
@test "hadoop_common_slave_mode_execute (--slaves 2)" { @test "hadoop_common_worker_mode_execute (--workers 2)" {
run hadoop_common_slave_mode_execute --slaves command1 command2 run hadoop_common_worker_mode_execute --workers command1 command2
[ "${output}" = "command1 command2" ] [ "${output}" = "command1 command2" ]
} }
@test "hadoop_common_slave_mode_execute (--hosts)" { @test "hadoop_common_worker_mode_execute (--hosts)" {
run hadoop_common_slave_mode_execute --hosts filename command run hadoop_common_worker_mode_execute --hosts filename command
[ "${output}" = "command" ] [ "${output}" = "command" ]
} }
@test "hadoop_common_slave_mode_execute (--hostnames 2)" { @test "hadoop_common_worker_mode_execute (--hostnames 2)" {
run hadoop_common_slave_mode_execute --hostnames "host1,host2" command1 command2 run hadoop_common_worker_mode_execute --hostnames "host1,host2" command1 command2
[ "${output}" = "command1 command2" ] [ "${output}" = "command1 command2" ]
} }

18
hadoop-common-project/hadoop-common/src/test/scripts/hadoop_slaves.bats → hadoop-common-project/hadoop-common/src/test/scripts/hadoop_workers.bats
View File

@ -15,23 +15,23 @@
load hadoop-functions_test_helper load hadoop-functions_test_helper
@test "hadoop_populate_slaves_file (specific file)" { @test "hadoop_populate_workers_file (specific file)" {
touch "${TMP}/file" touch "${TMP}/file"
hadoop_populate_slaves_file "${TMP}/file" hadoop_populate_workers_file "${TMP}/file"
[ "${HADOOP_SLAVES}" = "${TMP}/file" ] [ "${HADOOP_WORKERS}" = "${TMP}/file" ]
} }
@test "hadoop_populate_slaves_file (specific conf dir file)" { @test "hadoop_populate_workers_file (specific conf dir file)" {
HADOOP_CONF_DIR=${TMP}/1 HADOOP_CONF_DIR=${TMP}/1
mkdir -p "${HADOOP_CONF_DIR}" mkdir -p "${HADOOP_CONF_DIR}"
touch "${HADOOP_CONF_DIR}/file" touch "${HADOOP_CONF_DIR}/file"
hadoop_populate_slaves_file "file" hadoop_populate_workers_file "file"
echo "${HADOOP_SLAVES}" echo "${HADOOP_WORKERS}"
[ "${HADOOP_SLAVES}" = "${HADOOP_CONF_DIR}/file" ] [ "${HADOOP_WORKERS}" = "${HADOOP_CONF_DIR}/file" ]
} }
@test "hadoop_populate_slaves_file (no file)" { @test "hadoop_populate_workers_file (no file)" {
HADOOP_CONF_DIR=${TMP} HADOOP_CONF_DIR=${TMP}
run hadoop_populate_slaves_file "foo" run hadoop_populate_workers_file "foo"
[ "${status}" -eq 1 ] [ "${status}" -eq 1 ]
} }

10
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
View File

@ -26,10 +26,10 @@ function hadoop_usage
{ {
hadoop_add_option "--buildpaths" "attempt to add class files from build tree" hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon" hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon"
hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in slave mode" hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode"
hadoop_add_option "--loglevel level" "set the log4j level for this command" hadoop_add_option "--loglevel level" "set the log4j level for this command"
hadoop_add_option "--hosts filename" "list of hosts to use in slave mode" hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--slaves" "turn on slave mode" hadoop_add_option "--workers" "turn on worker mode"
hadoop_add_subcommand "balancer" "run a cluster balancing utility" hadoop_add_subcommand "balancer" "run a cluster balancing utility"
hadoop_add_subcommand "cacheadmin" "configure the HDFS cache" hadoop_add_subcommand "cacheadmin" "configure the HDFS cache"
@ -293,8 +293,8 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}" hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}" hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
exit $? exit $?
fi fi

12
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
View File

@ -79,14 +79,14 @@ fi
echo "Starting namenodes on [${NAMENODES}]" echo "Starting namenodes on [${NAMENODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${NAMENODES}" \ --hostnames "${NAMENODES}" \
--daemon start \ --daemon start \
namenode ${nameStartOpt} namenode ${nameStartOpt}
#--------------------------------------------------------- #---------------------------------------------------------
# datanodes (using default slaves file) # datanodes (using default workers file)
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
@ -98,7 +98,7 @@ else
echo "Starting datanodes" echo "Starting datanodes"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--daemon start \ --daemon start \
datanode ${dataStartOpt} datanode ${dataStartOpt}
@ -125,7 +125,7 @@ if [[ -n "${SECONDARY_NAMENODES}" ]]; then
echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]" echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${SECONDARY_NAMENODES}" \ --hostnames "${SECONDARY_NAMENODES}" \
--daemon start \ --daemon start \
@ -144,7 +144,7 @@ case "${SHARED_EDITS_DIR}" in
echo "Starting journal nodes [${JOURNAL_NODES}]" echo "Starting journal nodes [${JOURNAL_NODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${JOURNAL_NODES}" \ --hostnames "${JOURNAL_NODES}" \
--daemon start \ --daemon start \
@ -159,7 +159,7 @@ if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]" echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${NAMENODES}" \ --hostnames "${NAMENODES}" \
--daemon start \ --daemon start \

2
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-secure-dns.sh
View File

@ -45,7 +45,7 @@ fi
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \ exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--slaves \ --workers \
--daemon start \ --daemon start \
datanode datanode
else else

12
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.sh
View File

@ -56,14 +56,14 @@ fi
echo "Stopping namenodes on [${NAMENODES}]" echo "Stopping namenodes on [${NAMENODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${NAMENODES}" \ --hostnames "${NAMENODES}" \
--daemon stop \ --daemon stop \
namenode namenode
#--------------------------------------------------------- #---------------------------------------------------------
# datanodes (using default slaves file) # datanodes (using default workers file)
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] && if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
@ -75,7 +75,7 @@ else
echo "Stopping datanodes" echo "Stopping datanodes"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--daemon stop \ --daemon stop \
datanode datanode
@ -94,7 +94,7 @@ if [[ -n "${SECONDARY_NAMENODES}" ]]; then
echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]" echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${SECONDARY_NAMENODES}" \ --hostnames "${SECONDARY_NAMENODES}" \
--daemon stop \ --daemon stop \
@ -112,7 +112,7 @@ case "${SHARED_EDITS_DIR}" in
echo "Stopping journal nodes [${JOURNAL_NODES}]" echo "Stopping journal nodes [${JOURNAL_NODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${JOURNAL_NODES}" \ --hostnames "${JOURNAL_NODES}" \
--daemon stop \ --daemon stop \
@ -127,7 +127,7 @@ if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]" echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]"
"${HADOOP_HDFS_HOME}/bin/hdfs" \ "${HADOOP_HDFS_HOME}/bin/hdfs" \
--slaves \ --workers \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--hostnames "${NAMENODES}" \ --hostnames "${NAMENODES}" \
--daemon stop \ --daemon stop \

2
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-secure-dns.sh
View File

@ -45,7 +45,7 @@ fi
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \ exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--slaves \ --workers \
--daemon stop \ --daemon stop \
datanode datanode
else else

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenSecretManager.java
View File

@ -43,11 +43,12 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions; import com.google.common.base.Preconditions;
/** /**
* BlockTokenSecretManager can be instantiated in 2 modes, master mode and slave * BlockTokenSecretManager can be instantiated in 2 modes, master mode
* mode. Master can generate new block keys and export block keys to slaves, * and worker mode. Master can generate new block keys and export block
* while slaves can only import and use block keys received from master. Both * keys to workers, while workers can only import and use block keys
* master and slave can generate and verify block tokens. Typically, master mode * received from master. Both master and worker can generate and verify
* is used by NN and slave mode is used by DN. * block tokens. Typically, master mode is used by NN and worker mode
* is used by DN.
*/ */
@InterfaceAudience.Private @InterfaceAudience.Private
public class BlockTokenSecretManager extends public class BlockTokenSecretManager extends
@ -78,7 +79,7 @@ public class BlockTokenSecretManager extends
private final SecureRandom nonceGenerator = new SecureRandom(); private final SecureRandom nonceGenerator = new SecureRandom();
/** /**
* Constructor for slaves. * Constructor for workers.
* *
* @param keyUpdateInterval how often a new key will be generated * @param keyUpdateInterval how often a new key will be generated
* @param tokenLifetime how long an individual token is valid * @param tokenLifetime how long an individual token is valid
@ -180,7 +181,7 @@ public class BlockTokenSecretManager extends
} }
/** /**
* Set block keys, only to be used in slave mode * Set block keys, only to be used in worker mode
*/ */
public synchronized void addKeys(ExportedBlockKeys exportedKeys) public synchronized void addKeys(ExportedBlockKeys exportedKeys)
throws IOException { throws IOException {

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/overview.html
View File

@ -141,8 +141,8 @@ host and port. This is specified with the configuration property
href="../mapred-default.html#mapred.job.tracker">mapred.job.tracker</a></tt>. href="../mapred-default.html#mapred.job.tracker">mapred.job.tracker</a></tt>.
</li> </li>
<li>A <em>slaves</em> file that lists the names of all the hosts in <li>A <em>workers</em> file that lists the names of all the hosts in
the cluster. The default slaves file is <tt>conf/slaves</tt>. the cluster. The default workers file is <tt>conf/workers</tt>.
</ol> </ol>
@ -245,7 +245,7 @@ href="../hdfs-default.html#dfs.name.dir">dfs.name.dir</a></tt> and
<tt><a href="../hdfs-default.html#dfs.data.dir">dfs.data.dir</a> <tt><a href="../hdfs-default.html#dfs.data.dir">dfs.data.dir</a>
in <tt>conf/hdfs-site.xml</tt>. in <tt>conf/hdfs-site.xml</tt>.
</tt>These are local directories used to hold distributed filesystem </tt>These are local directories used to hold distributed filesystem
data on the master node and slave nodes respectively. Note data on the master node and worker nodes respectively. Note
that <tt>dfs.data.dir</tt> may contain a space- or comma-separated that <tt>dfs.data.dir</tt> may contain a space- or comma-separated
list of directory names, so that data may be stored on multiple local list of directory names, so that data may be stored on multiple local
devices.</li> devices.</li>
@ -260,13 +260,13 @@ and <tt><a
href="../mapred-default.html#mapred.reduce.tasks">mapred.reduce.tasks</a></tt> href="../mapred-default.html#mapred.reduce.tasks">mapred.reduce.tasks</a></tt>
in <tt>conf/mapred-site.xml</tt>. in <tt>conf/mapred-site.xml</tt>.
As a rule of thumb, use 10x the As a rule of thumb, use 10x the
number of slave processors for <tt>mapred.map.tasks</tt>, and 2x the number of worker processors for <tt>mapred.map.tasks</tt>, and 2x the
number of slave processors for <tt>mapred.reduce.tasks</tt>.</li> number of worker processors for <tt>mapred.reduce.tasks</tt>.</li>
</ol> </ol>
<p>Finally, list all slave hostnames or IP addresses in your <p>Finally, list all worker hostnames or IP addresses in your
<tt>conf/slaves</tt> file, one per line. Then format your filesystem <tt>conf/workers</tt> file, one per line. Then format your filesystem
and start your cluster on your master node, as above. and start your cluster on your master node, as above.
</body> </body>

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md
View File

@ -202,7 +202,7 @@ To stop the cluster run the following command:
[hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh [hdfs]$ $HADOOP_HOME/sbin/stop-dfs.sh
These commands can be run from any node where the HDFS configuration is available. The command uses the configuration to determine the Namenodes in the cluster and then starts the Namenode process on those nodes. The Datanodes are started on the nodes specified in the `slaves` file. The script can be used as a reference for building your own scripts to start and stop the cluster. These commands can be run from any node where the HDFS configuration is available. The command uses the configuration to determine the Namenodes in the cluster and then starts the Namenode process on those nodes. The Datanodes are started on the nodes specified in the `workers` file. The script can be used as a reference for building your own scripts to start and stop the cluster.
### Balancer ### Balancer

2
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
View File

@ -70,7 +70,7 @@ The HDFS High Availability feature addresses the above problems by providing the
Architecture Architecture
------------ ------------
In a typical HA cluster, two or more separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the others are in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standbys are simply acting as slaves, maintaining enough state to provide a fast failover if necessary. In a typical HA cluster, two or more separate machines are configured as NameNodes. At any point in time, exactly one of the NameNodes is in an *Active* state, and the others are in a *Standby* state. The Active NameNode is responsible for all client operations in the cluster, while the Standbys are simply acting as workers, maintaining enough state to provide a fast failover if necessary.
In order for the Standby node to keep its state synchronized with the Active node, both nodes communicate with a group of separate daemons called "JournalNodes" (JNs). When any namespace modification is performed by the Active node, it durably logs a record of the modification to a majority of these JNs. The Standby node is capable of reading the edits from the JNs, and is constantly watching them for changes to the edit log. As the Standby Node sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the JounalNodes before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs. In order for the Standby node to keep its state synchronized with the Active node, both nodes communicate with a group of separate daemons called "JournalNodes" (JNs). When any namespace modification is performed by the Active node, it durably logs a record of the modification to a majority of these JNs. The Standby node is capable of reading the edits from the JNs, and is constantly watching them for changes to the edit log. As the Standby Node sees the edits, it applies them to its own namespace. In the event of a failover, the Standby will ensure that it has read all of the edits from the JounalNodes before promoting itself to the Active state. This ensures that the namespace state is fully synchronized before a failover occurs.

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java
View File

@ -43,14 +43,14 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
* already present on the {@link FileSystem} at the path specified by the url * already present on the {@link FileSystem} at the path specified by the url
* and are accessible by every machine in the cluster.</p> * and are accessible by every machine in the cluster.</p>
* *
* <p>The framework will copy the necessary files on to the slave node before * <p>The framework will copy the necessary files on to the worker node before
* any tasks for the job are executed on that node. Its efficiency stems from * any tasks for the job are executed on that node. Its efficiency stems from
* the fact that the files are only copied once per job and the ability to * the fact that the files are only copied once per job and the ability to
* cache archives which are un-archived on the slaves.</p> * cache archives which are un-archived on the workers.</p>
* *
* <p><code>DistributedCache</code> can be used to distribute simple, read-only * <p><code>DistributedCache</code> can be used to distribute simple, read-only
* data/text files and/or more complex types such as archives, jars etc. * data/text files and/or more complex types such as archives, jars etc.
* Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes. * Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes.
* Jars may be optionally added to the classpath of the tasks, a rudimentary * Jars may be optionally added to the classpath of the tasks, a rudimentary
* software distribution mechanism. Files have execution permissions. * software distribution mechanism. Files have execution permissions.
* In older version of Hadoop Map/Reduce users could optionally ask for symlinks * In older version of Hadoop Map/Reduce users could optionally ask for symlinks

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java
View File

@ -45,14 +45,14 @@ import java.net.URI;
* already present on the {@link FileSystem} at the path specified by the url * already present on the {@link FileSystem} at the path specified by the url
* and are accessible by every machine in the cluster.</p> * and are accessible by every machine in the cluster.</p>
* *
* <p>The framework will copy the necessary files on to the slave node before * <p>The framework will copy the necessary files on to the worker node before
* any tasks for the job are executed on that node. Its efficiency stems from * any tasks for the job are executed on that node. Its efficiency stems from
* the fact that the files are only copied once per job and the ability to * the fact that the files are only copied once per job and the ability to
* cache archives which are un-archived on the slaves.</p> * cache archives which are un-archived on the workers.</p>
* *
* <p><code>DistributedCache</code> can be used to distribute simple, read-only * <p><code>DistributedCache</code> can be used to distribute simple, read-only
* data/text files and/or more complex types such as archives, jars etc. * data/text files and/or more complex types such as archives, jars etc.
* Archives (zip, tar and tgz/tar.gz files) are un-archived at the slave nodes. * Archives (zip, tar and tgz/tar.gz files) are un-archived at the worker nodes.
* Jars may be optionally added to the classpath of the tasks, a rudimentary * Jars may be optionally added to the classpath of the tasks, a rudimentary
* software distribution mechanism. Files have execution permissions. * software distribution mechanism. Files have execution permissions.
* In older version of Hadoop Map/Reduce users could optionally ask for symlinks * In older version of Hadoop Map/Reduce users could optionally ask for symlinks

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
View File

@ -85,11 +85,11 @@ A MapReduce *job* usually splits the input data-set into independent chunks whic
Typically the compute nodes and the storage nodes are the same, that is, the MapReduce framework and the Hadoop Distributed File System (see [HDFS Architecture Guide](../../hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)) are running on the same set of nodes. This configuration allows the framework to effectively schedule tasks on the nodes where data is already present, resulting in very high aggregate bandwidth across the cluster. Typically the compute nodes and the storage nodes are the same, that is, the MapReduce framework and the Hadoop Distributed File System (see [HDFS Architecture Guide](../../hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)) are running on the same set of nodes. This configuration allows the framework to effectively schedule tasks on the nodes where data is already present, resulting in very high aggregate bandwidth across the cluster.
The MapReduce framework consists of a single master `ResourceManager`, one slave `NodeManager` per cluster-node, and `MRAppMaster` per application (see [YARN Architecture Guide](../../hadoop-yarn/hadoop-yarn-site/YARN.html)). The MapReduce framework consists of a single master `ResourceManager`, one worker `NodeManager` per cluster-node, and `MRAppMaster` per application (see [YARN Architecture Guide](../../hadoop-yarn/hadoop-yarn-site/YARN.html)).
Minimally, applications specify the input/output locations and supply *map* and *reduce* functions via implementations of appropriate interfaces and/or abstract-classes. These, and other job parameters, comprise the *job configuration*. Minimally, applications specify the input/output locations and supply *map* and *reduce* functions via implementations of appropriate interfaces and/or abstract-classes. These, and other job parameters, comprise the *job configuration*.
The Hadoop *job client* then submits the job (jar/executable etc.) and configuration to the `ResourceManager` which then assumes the responsibility of distributing the software/configuration to the slaves, scheduling tasks and monitoring them, providing status and diagnostic information to the job-client. The Hadoop *job client* then submits the job (jar/executable etc.) and configuration to the `ResourceManager` which then assumes the responsibility of distributing the software/configuration to the workers, scheduling tasks and monitoring them, providing status and diagnostic information to the job-client.
Although the Hadoop framework is implemented in Java™, MapReduce applications need not be written in Java. Although the Hadoop framework is implemented in Java™, MapReduce applications need not be written in Java.
@ -787,11 +787,11 @@ or Counters.incrCounter(String, String, long) in the `map` and/or `reduce` metho
Applications specify the files to be cached via urls (hdfs://) in the `Job`. The `DistributedCache` assumes that the files specified via hdfs:// urls are already present on the `FileSystem`. Applications specify the files to be cached via urls (hdfs://) in the `Job`. The `DistributedCache` assumes that the files specified via hdfs:// urls are already present on the `FileSystem`.
The framework will copy the necessary files to the slave node before any tasks for the job are executed on that node. Its efficiency stems from the fact that the files are only copied once per job and the ability to cache archives which are un-archived on the slaves. The framework will copy the necessary files to the worker node before any tasks for the job are executed on that node. Its efficiency stems from the fact that the files are only copied once per job and the ability to cache archives which are un-archived on the workers.
`DistributedCache` tracks the modification timestamps of the cached files. Clearly the cache files should not be modified by the application or externally while the job is executing. `DistributedCache` tracks the modification timestamps of the cached files. Clearly the cache files should not be modified by the application or externally while the job is executing.
`DistributedCache` can be used to distribute simple, read-only data/text files and more complex types such as archives and jars. Archives (zip, tar, tgz and tar.gz files) are *un-archived* at the slave nodes. Files have *execution permissions* set. `DistributedCache` can be used to distribute simple, read-only data/text files and more complex types such as archives and jars. Archives (zip, tar, tgz and tar.gz files) are *un-archived* at the worker nodes. Files have *execution permissions* set.
The files/archives can be distributed by setting the property `mapreduce.job.cache.{files |archives}`. If more than one file/archive has to be distributed, they can be added as comma separated paths. The properties can also be set by APIs The files/archives can be distributed by setting the property `mapreduce.job.cache.{files |archives}`. If more than one file/archive has to be distributed, they can be added as comma separated paths. The properties can also be set by APIs
[Job.addCacheFile(URI)](../../api/org/apache/hadoop/mapreduce/Job.html)/ [Job.addCacheFile(URI)](../../api/org/apache/hadoop/mapreduce/Job.html)/
@ -808,12 +808,12 @@ api can be used to cache files/jars and also add them to the *classpath* of chil
##### Private and Public DistributedCache Files ##### Private and Public DistributedCache Files
DistributedCache files can be private or public, that determines how they can be shared on the slave nodes. DistributedCache files can be private or public, that determines how they can be shared on the worker nodes.
* "Private" DistributedCache files are cached in a localdirectory private to * "Private" DistributedCache files are cached in a localdirectory private to
the user whose jobs need these files. These files are shared by all tasks the user whose jobs need these files. These files are shared by all tasks
and jobs of the specific user only and cannot be accessed by jobs of and jobs of the specific user only and cannot be accessed by jobs of
other users on the slaves. A DistributedCache file becomes private by other users on the workers. A DistributedCache file becomes private by
virtue of its permissions on the file system where the files are virtue of its permissions on the file system where the files are
uploaded, typically HDFS. If the file has no world readable access, or if uploaded, typically HDFS. If the file has no world readable access, or if
the directory path leading to the file has no world executable access for the directory path leading to the file has no world executable access for
@ -821,7 +821,7 @@ DistributedCache files can be private or public, that determines how they can be
* "Public" DistributedCache files are cached in a global directory and the * "Public" DistributedCache files are cached in a global directory and the
file access is setup such that they are publicly visible to all users. file access is setup such that they are publicly visible to all users.
These files can be shared by tasks and jobs of all users on the slaves. A These files can be shared by tasks and jobs of all users on the workers. A
DistributedCache file becomes public by virtue of its permissions on the DistributedCache file becomes public by virtue of its permissions on the
file system where the files are uploaded, typically HDFS. If the file has file system where the files are uploaded, typically HDFS. If the file has
world readable access, AND if the directory path leading to the file has world readable access, AND if the directory path leading to the file has

29
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ReliabilityTest.java
View File

@ -60,10 +60,10 @@ import org.apache.hadoop.util.ToolRunner;
* build/hadoop-<version>-test.jar MRReliabilityTest -libjars * build/hadoop-<version>-test.jar MRReliabilityTest -libjars
* build/hadoop-<version>-examples.jar [-scratchdir <dir>]" * build/hadoop-<version>-examples.jar [-scratchdir <dir>]"
* *
* The scratchdir is optional and by default the current directory on the client * The scratchdir is optional and by default the current directory on
* will be used as the scratch space. Note that password-less SSH must be set up * the client will be used as the scratch space. Note that password-less
* between the client machine from where the test is submitted, and the cluster * SSH must be set up between the client machine from where the test is
* nodes where the test runs. * submitted, and the cluster nodes where the test runs.
* *
* The test should be run on a <b>free</b> cluster where there is no other parallel * The test should be run on a <b>free</b> cluster where there is no other parallel
* job submission going on. Submission of other jobs while the test runs can cause * job submission going on. Submission of other jobs while the test runs can cause
@ -223,7 +223,8 @@ public class ReliabilityTest extends Configured implements Tool {
JobID jobId = jobs[jobs.length - 1].getJobID(); JobID jobId = jobs[jobs.length - 1].getJobID();
RunningJob rJob = jc.getJob(jobId); RunningJob rJob = jc.getJob(jobId);
if(rJob.isComplete()) { if(rJob.isComplete()) {
LOG.error("The last job returned by the querying JobTracker is complete :" + LOG.error("The last job returned by the querying "
+"JobTracker is complete :" +
rJob.getJobID() + " .Exiting the test"); rJob.getJobID() + " .Exiting the test");
System.exit(-1); System.exit(-1);
} }
@ -255,12 +256,12 @@ public class ReliabilityTest extends Configured implements Tool {
private float threshold = 0.2f; private float threshold = 0.2f;
private boolean onlyMapsProgress; private boolean onlyMapsProgress;
private int numIterations; private int numIterations;
final private String slavesFile = dir + "/_reliability_test_slaves_file_"; final private String workersFile = dir + "/_reliability_test_workers_file_";
final String shellCommand = normalizeCommandPath("bin/slaves.sh"); final private String shellCommand = normalizeCommandPath("bin/workers.sh");
final private String STOP_COMMAND = "ps uwwx | grep java | grep " + final private String stopCommand = "ps uwwx | grep java | grep " +
"org.apache.hadoop.mapred.TaskTracker"+ " |" + "org.apache.hadoop.mapred.TaskTracker"+ " |" +
" grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s STOP"; " grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s STOP";
final private String RESUME_COMMAND = "ps uwwx | grep java | grep " + final private String resumeCommand = "ps uwwx | grep java | grep " +
"org.apache.hadoop.mapred.TaskTracker"+ " |" + "org.apache.hadoop.mapred.TaskTracker"+ " |" +
" grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s CONT"; " grep -v grep | tr -s ' ' | cut -d ' ' -f2 | xargs kill -s CONT";
//Only one instance must be active at any point //Only one instance must be active at any point
@ -336,7 +337,7 @@ public class ReliabilityTest extends Configured implements Tool {
int count = 0; int count = 0;
FileOutputStream fos = new FileOutputStream(new File(slavesFile)); FileOutputStream fos = new FileOutputStream(new File(workersFile));
LOG.info(new Date() + " Stopping a few trackers"); LOG.info(new Date() + " Stopping a few trackers");
for (String tracker : trackerNamesList) { for (String tracker : trackerNamesList) {
@ -355,17 +356,17 @@ public class ReliabilityTest extends Configured implements Tool {
private void startTaskTrackers() throws Exception { private void startTaskTrackers() throws Exception {
LOG.info(new Date() + " Resuming the stopped trackers"); LOG.info(new Date() + " Resuming the stopped trackers");
runOperationOnTT("resume"); runOperationOnTT("resume");
new File(slavesFile).delete(); new File(workersFile).delete();
} }
private void runOperationOnTT(String operation) throws IOException { private void runOperationOnTT(String operation) throws IOException {
Map<String,String> hMap = new HashMap<String,String>(); Map<String,String> hMap = new HashMap<String,String>();
hMap.put("HADOOP_SLAVES", slavesFile); hMap.put("HADOOP_WORKERS", workersFile);
StringTokenizer strToken; StringTokenizer strToken;
if (operation.equals("suspend")) { if (operation.equals("suspend")) {
strToken = new StringTokenizer(STOP_COMMAND, " "); strToken = new StringTokenizer(stopCommand, " ");
} else { } else {
strToken = new StringTokenizer(RESUME_COMMAND, " "); strToken = new StringTokenizer(resumeCommand, " ");
} }
String commandArgs[] = new String[strToken.countTokens() + 1]; String commandArgs[] = new String[strToken.countTokens() + 1];
int i = 0; int i = 0;

19
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestLazyOutput.java
View File

@ -44,11 +44,11 @@ import static org.junit.Assert.assertTrue;
* 0 byte files * 0 byte files
*/ */
public class TestLazyOutput { public class TestLazyOutput {
private static final int NUM_HADOOP_SLAVES = 3; private static final int NUM_HADOOP_WORKERS = 3;
private static final int NUM_MAPS_PER_NODE = 2; private static final int NUM_MAPS_PER_NODE = 2;
private static final Path INPUT = new Path("/testlazy/input"); private static final Path INPUTPATH = new Path("/testlazy/input");
private static final List<String> input = private static final List<String> INPUTLIST =
Arrays.asList("All","Roads","Lead","To","Hadoop"); Arrays.asList("All","Roads","Lead","To","Hadoop");
@ -98,7 +98,7 @@ public class TestLazyOutput {
job.setJobName("test-lazy-output"); job.setJobName("test-lazy-output");
FileInputFormat.setInputPaths(job, INPUT); FileInputFormat.setInputPaths(job, INPUTPATH);
FileOutputFormat.setOutputPath(job, output); FileOutputFormat.setOutputPath(job, output);
job.setInputFormat(TextInputFormat.class); job.setInputFormat(TextInputFormat.class);
job.setMapOutputKeyClass(LongWritable.class); job.setMapOutputKeyClass(LongWritable.class);
@ -123,10 +123,10 @@ public class TestLazyOutput {
public void createInput(FileSystem fs, int numMappers) throws Exception { public void createInput(FileSystem fs, int numMappers) throws Exception {
for (int i =0; i < numMappers; i++) { for (int i =0; i < numMappers; i++) {
OutputStream os = fs.create(new Path(INPUT, OutputStream os = fs.create(new Path(INPUTPATH,
"text" + i + ".txt")); "text" + i + ".txt"));
Writer wr = new OutputStreamWriter(os); Writer wr = new OutputStreamWriter(os);
for(String inp : input) { for(String inp : INPUTLIST) {
wr.write(inp+"\n"); wr.write(inp+"\n");
} }
wr.close(); wr.close();
@ -142,13 +142,14 @@ public class TestLazyOutput {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters // Start the mini-MR and mini-DFS clusters
dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_HADOOP_SLAVES) dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_HADOOP_WORKERS)
.build(); .build();
fileSys = dfs.getFileSystem(); fileSys = dfs.getFileSystem();
mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1); mr = new MiniMRCluster(NUM_HADOOP_WORKERS,
fileSys.getUri().toString(), 1);
int numReducers = 2; int numReducers = 2;
int numMappers = NUM_HADOOP_SLAVES * NUM_MAPS_PER_NODE; int numMappers = NUM_HADOOP_WORKERS * NUM_MAPS_PER_NODE;
createInput(fileSys, numMappers); createInput(fileSys, numMappers);
Path output1 = new Path("/testlazy/output1"); Path output1 = new Path("/testlazy/output1");

13
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/pipes/TestPipes.java
View File

@ -56,11 +56,11 @@ public class TestPipes {
private static Path cppExamples = private static Path cppExamples =
new Path(System.getProperty("install.c++.examples")); new Path(System.getProperty("install.c++.examples"));
static Path wordCountSimple = private static Path wordCountSimple =
new Path(cppExamples, "bin/wordcount-simple"); new Path(cppExamples, "bin/wordcount-simple");
static Path wordCountPart = private static Path wordCountPart =
new Path(cppExamples, "bin/wordcount-part"); new Path(cppExamples, "bin/wordcount-part");
static Path wordCountNoPipes = private static Path wordCountNoPipes =
new Path(cppExamples,"bin/wordcount-nopipe"); new Path(cppExamples,"bin/wordcount-nopipe");
static Path nonPipedOutDir; static Path nonPipedOutDir;
@ -80,10 +80,11 @@ public class TestPipes {
Path inputPath = new Path("testing/in"); Path inputPath = new Path("testing/in");
Path outputPath = new Path("testing/out"); Path outputPath = new Path("testing/out");
try { try {
final int numSlaves = 2; final int numWorkers = 2;
Configuration conf = new Configuration(); Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).numDataNodes(numSlaves).build(); dfs = new MiniDFSCluster.Builder(conf).numDataNodes(numWorkers).build();
mr = new MiniMRCluster(numSlaves, dfs.getFileSystem().getUri().toString(), 1); mr = new MiniMRCluster(numWorkers,
dfs.getFileSystem().getUri().toString(), 1);
writeInputFile(dfs.getFileSystem(), inputPath); writeInputFile(dfs.getFileSystem(), inputPath);
runProgram(mr, dfs, wordCountSimple, runProgram(mr, dfs, wordCountSimple,
inputPath, outputPath, 3, 2, twoSplitOutput, null); inputPath, outputPath, 3, 2, twoSplitOutput, null);

26
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestMapReduceLazyOutput.java
View File

@ -50,13 +50,16 @@ import static org.junit.Assert.assertTrue;
* 0 byte files * 0 byte files
*/ */
public class TestMapReduceLazyOutput { public class TestMapReduceLazyOutput {
private static final int NUM_HADOOP_SLAVES = 3; private static final int NUM_HADOOP_WORKERS = 3;
private static final int NUM_MAPS_PER_NODE = 2; private static final int NUM_MAPS_PER_NODE = 2;
private static final Path INPUT = new Path("/testlazy/input"); private static final Path INPUTPATH = new Path("/testlazy/input");
private static final List<String> input = private static final List<String> INPUTLIST =
Arrays.asList("All","Roads","Lead","To","Hadoop"); Arrays.asList("All","Roads","Lead","To","Hadoop");
/**
* Test mapper.
*/
public static class TestMapper public static class TestMapper
extends Mapper<LongWritable, Text, LongWritable, Text>{ extends Mapper<LongWritable, Text, LongWritable, Text>{
@ -70,7 +73,9 @@ public class TestMapReduceLazyOutput {
} }
} }
/**
* Test Reducer.
*/
public static class TestReducer public static class TestReducer
extends Reducer<LongWritable,Text,LongWritable,Text> { extends Reducer<LongWritable,Text,LongWritable,Text> {
@ -91,7 +96,7 @@ public class TestMapReduceLazyOutput {
throws Exception { throws Exception {
Job job = Job.getInstance(conf, "Test-Lazy-Output"); Job job = Job.getInstance(conf, "Test-Lazy-Output");
FileInputFormat.setInputPaths(job, INPUT); FileInputFormat.setInputPaths(job, INPUTPATH);
FileOutputFormat.setOutputPath(job, output); FileOutputFormat.setOutputPath(job, output);
job.setJarByClass(TestMapReduceLazyOutput.class); job.setJarByClass(TestMapReduceLazyOutput.class);
@ -113,10 +118,10 @@ public class TestMapReduceLazyOutput {
public void createInput(FileSystem fs, int numMappers) throws Exception { public void createInput(FileSystem fs, int numMappers) throws Exception {
for (int i =0; i < numMappers; i++) { for (int i =0; i < numMappers; i++) {
OutputStream os = fs.create(new Path(INPUT, OutputStream os = fs.create(new Path(INPUTPATH,
"text" + i + ".txt")); "text" + i + ".txt"));
Writer wr = new OutputStreamWriter(os); Writer wr = new OutputStreamWriter(os);
for(String inp : input) { for(String inp : INPUTLIST) {
wr.write(inp+"\n"); wr.write(inp+"\n");
} }
wr.close(); wr.close();
@ -132,13 +137,14 @@ public class TestMapReduceLazyOutput {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters // Start the mini-MR and mini-DFS clusters
dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_HADOOP_SLAVES) dfs = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_HADOOP_WORKERS)
.build(); .build();
fileSys = dfs.getFileSystem(); fileSys = dfs.getFileSystem();
mr = new MiniMRCluster(NUM_HADOOP_SLAVES, fileSys.getUri().toString(), 1); mr = new MiniMRCluster(NUM_HADOOP_WORKERS,
fileSys.getUri().toString(), 1);
int numReducers = 2; int numReducers = 2;
int numMappers = NUM_HADOOP_SLAVES * NUM_MAPS_PER_NODE; int numMappers = NUM_HADOOP_WORKERS * NUM_MAPS_PER_NODE;
createInput(fileSys, numMappers); createInput(fileSys, numMappers);
Path output1 = new Path("/testlazy/output1"); Path output1 = new Path("/testlazy/output1");

10
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestBinaryTokenFile.java
View File

@ -99,7 +99,8 @@ public class TestBinaryTokenFile {
if (binaryTokenCollection.size() != 1) { if (binaryTokenCollection.size() != 1) {
throw new RuntimeException("The token collection read from file ["+tokenFile+"] must have size = 1."); throw new RuntimeException("The token collection read from file ["+tokenFile+"] must have size = 1.");
} }
final Token<? extends TokenIdentifier> binTok = binaryTokenCollection.iterator().next(); final Token<? extends TokenIdentifier> binTok = binaryTokenCollection
.iterator().next();
System.out.println("The token read from binary file: t = [" + binTok + "]"); System.out.println("The token read from binary file: t = [" + binTok + "]");
// Verify that dt is same as the token in the file: // Verify that dt is same as the token in the file:
if (!dt.equals(binTok)) { if (!dt.equals(binTok)) {
@ -169,7 +170,7 @@ public class TestBinaryTokenFile {
new Path(System.getProperty("test.build.data","/tmp")); new Path(System.getProperty("test.build.data","/tmp"));
private static final Path binaryTokenFileName = new Path(TEST_DIR, "tokenFile.binary"); private static final Path binaryTokenFileName = new Path(TEST_DIR, "tokenFile.binary");
private static final int numSlaves = 1; // num of data nodes private static final int NUMWORKERS = 1; // num of data nodes
private static final int noOfNMs = 1; private static final int noOfNMs = 1;
private static Path p1; private static Path p1;
@ -183,7 +184,7 @@ public class TestBinaryTokenFile {
final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); final MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
builder.checkExitOnShutdown(true); builder.checkExitOnShutdown(true);
builder.numDataNodes(numSlaves); builder.numDataNodes(NUMWORKERS);
builder.format(true); builder.format(true);
builder.racks(null); builder.racks(null);
dfsCluster = builder.build(); dfsCluster = builder.build();
@ -192,7 +193,8 @@ public class TestBinaryTokenFile {
mrCluster.init(conf); mrCluster.init(conf);
mrCluster.start(); mrCluster.start();
NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem()).startThreads(); NameNodeAdapter.getDtSecretManager(dfsCluster.getNamesystem())
.startThreads();
FileSystem fs = dfsCluster.getFileSystem(); FileSystem fs = dfsCluster.getFileSystem();
p1 = new Path("file1"); p1 = new Path("file1");

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/TestMRCredentials.java
View File

@ -51,7 +51,7 @@ public class TestMRCredentials {
static final int NUM_OF_KEYS = 10; static final int NUM_OF_KEYS = 10;
private static MiniMRClientCluster mrCluster; private static MiniMRClientCluster mrCluster;
private static MiniDFSCluster dfsCluster; private static MiniDFSCluster dfsCluster;
private static int numSlaves = 1; private static int numWorkers = 1;
private static JobConf jConf; private static JobConf jConf;
@SuppressWarnings("deprecation") @SuppressWarnings("deprecation")
@ -59,7 +59,7 @@ public class TestMRCredentials {
public static void setUp() throws Exception { public static void setUp() throws Exception {
System.setProperty("hadoop.log.dir", "logs"); System.setProperty("hadoop.log.dir", "logs");
Configuration conf = new Configuration(); Configuration conf = new Configuration();
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(numSlaves) dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(numWorkers)
.build(); .build();
jConf = new JobConf(conf); jConf = new JobConf(conf);
FileSystem.setDefaultUri(conf, dfsCluster.getFileSystem().getUri().toString()); FileSystem.setDefaultUri(conf, dfsCluster.getFileSystem().getUri().toString());
@ -80,7 +80,7 @@ public class TestMRCredentials {
} }
public static void createKeysAsJson (String fileName) public static void createKeysAsJson(String fileName)
throws FileNotFoundException, IOException{ throws FileNotFoundException, IOException{
StringBuilder jsonString = new StringBuilder(); StringBuilder jsonString = new StringBuilder();
jsonString.append("{"); jsonString.append("{");

6
hadoop-yarn-project/hadoop-yarn/bin/start-yarn.sh
View File

@ -62,7 +62,7 @@ else
"${HADOOP_YARN_HOME}/bin/yarn" \ "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--daemon start \ --daemon start \
--slaves \ --workers \
--hostnames "${RMHOSTS}" \ --hostnames "${RMHOSTS}" \
resourcemanager resourcemanager
fi fi
@ -71,7 +71,7 @@ fi
echo "Starting nodemanagers" echo "Starting nodemanagers"
"${HADOOP_YARN_HOME}/bin/yarn" \ "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--slaves \ --workers \
--daemon start \ --daemon start \
nodemanager nodemanager
@ -80,7 +80,7 @@ PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.ad
if [[ -n ${PROXYSERVER} ]]; then if [[ -n ${PROXYSERVER} ]]; then
"${HADOOP_YARN_HOME}/bin/yarn" \ "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--slaves \ --workers \
--hostnames "${PROXYSERVER}" \ --hostnames "${PROXYSERVER}" \
--daemon start \ --daemon start \
proxyserver proxyserver

6
hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.sh
View File

@ -62,7 +62,7 @@ else
"${HADOOP_YARN_HOME}/bin/yarn" \ "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--daemon stop \ --daemon stop \
--slaves \ --workers \
--hostnames "${RMHOSTS}" \ --hostnames "${RMHOSTS}" \
resourcemanager resourcemanager
fi fi
@ -71,7 +71,7 @@ fi
echo "Stopping nodemanagers" echo "Stopping nodemanagers"
"${HADOOP_YARN_HOME}/bin/yarn" \ "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--slaves \ --workers \
--daemon stop \ --daemon stop \
nodemanager nodemanager
@ -81,7 +81,7 @@ if [[ -n ${PROXYSERVER} ]]; then
echo "Stopping proxy server [${PROXYSERVER}]" echo "Stopping proxy server [${PROXYSERVER}]"
"${HADOOP_YARN_HOME}/bin/yarn" \ "${HADOOP_YARN_HOME}/bin/yarn" \
--config "${HADOOP_CONF_DIR}" \ --config "${HADOOP_CONF_DIR}" \
--slaves \ --workers \
--hostnames "${PROXYSERVER}" \ --hostnames "${PROXYSERVER}" \
--daemon stop \ --daemon stop \
proxyserver proxyserver

12
hadoop-yarn-project/hadoop-yarn/bin/yarn
View File

@ -26,10 +26,10 @@ function hadoop_usage
{ {
hadoop_add_option "--buildpaths" "attempt to add class files from build tree" hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon" hadoop_add_option "--daemon (start|status|stop)" "operate on a daemon"
hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in slave mode" hadoop_add_option "--hostnames list[,of,host,names]" "hosts to use in worker mode"
hadoop_add_option "--loglevel level" "set the log4j level for this command" hadoop_add_option "--loglevel level" "set the log4j level for this command"
hadoop_add_option "--hosts filename" "list of hosts to use in slave mode" hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--slaves" "turn on slave mode" hadoop_add_option "--workers" "turn on worker mode"
hadoop_add_subcommand "application" "prints application(s) report/kill application" hadoop_add_subcommand "application" "prints application(s) report/kill application"
hadoop_add_subcommand "applicationattempt" "prints applicationattempt(s) report" hadoop_add_subcommand "applicationattempt" "prints applicationattempt(s) report"
@ -41,7 +41,7 @@ function hadoop_usage
hadoop_add_subcommand "jar <jar>" "run a jar file" hadoop_add_subcommand "jar <jar>" "run a jar file"
hadoop_add_subcommand "logs" "dump container logs" hadoop_add_subcommand "logs" "dump container logs"
hadoop_add_subcommand "node" "prints node report(s)" hadoop_add_subcommand "node" "prints node report(s)"
hadoop_add_subcommand "nodemanager" "run a nodemanager on each slave" hadoop_add_subcommand "nodemanager" "run a nodemanager on each worker"
hadoop_add_subcommand "proxyserver" "run the web app proxy server" hadoop_add_subcommand "proxyserver" "run the web app proxy server"
hadoop_add_subcommand "queue" "prints queue information" hadoop_add_subcommand "queue" "prints queue information"
hadoop_add_subcommand "resourcemanager" "run the ResourceManager" hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
@ -266,8 +266,8 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}" hadoop_verify_user "${HADOOP_SUBCMD}"
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}" hadoop_common_worker_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
exit $? exit $?
fi fi

2
hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd
View File

@ -64,7 +64,7 @@ if not defined YARN_CONF_DIR (
@rem @rem
if "%1" == "--hosts" ( if "%1" == "--hosts" (
set YARN_SLAVES=%YARN_CONF_DIR%\%2 set YARN_WORKERS=%YARN_CONF_DIR%\%2
shift shift
shift shift
) )

2
hadoop-yarn-project/hadoop-yarn/bin/yarn-config.sh
View File

@ -51,7 +51,7 @@ function hadoop_subproject_init
hadoop_deprecate_envvar YARN_OPTS HADOOP_OPTS hadoop_deprecate_envvar YARN_OPTS HADOOP_OPTS
hadoop_deprecate_envvar YARN_SLAVES HADOOP_SLAVES hadoop_deprecate_envvar YARN_SLAVES HADOOP_WORKERS
HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_HOME}" HADOOP_YARN_HOME="${HADOOP_YARN_HOME:-$HADOOP_HOME}"

6
hadoop-yarn-project/hadoop-yarn/bin/yarn-daemons.sh
View File

@ -47,13 +47,13 @@ daemonmode=$1
shift shift
hadoop_error "WARNING: Use of this script to ${daemonmode} YARN daemons is deprecated." hadoop_error "WARNING: Use of this script to ${daemonmode} YARN daemons is deprecated."
hadoop_error "WARNING: Attempting to execute replacement \"yarn --slaves --daemon ${daemonmode}\" instead." hadoop_error "WARNING: Attempting to execute replacement \"yarn --workers --daemon ${daemonmode}\" instead."
# #
# Original input was usually: # Original input was usually:
# yarn-daemons.sh (shell options) (start|stop) nodemanager (daemon options) # yarn-daemons.sh (shell options) (start|stop) nodemanager (daemon options)
# we're going to turn this into # we're going to turn this into
# yarn --slaves --daemon (start|stop) (rest of options) # yarn --workers --daemon (start|stop) (rest of options)
# #
for (( i = 0; i < ${#HADOOP_USER_PARAMS[@]}; i++ )) for (( i = 0; i < ${#HADOOP_USER_PARAMS[@]}; i++ ))
do do
@ -64,5 +64,5 @@ do
fi fi
done done
${yarnscript} --slaves --daemon "${daemonmode}" "${HADOOP_USER_PARAMS[@]}" ${yarnscript} --workers --daemon "${daemonmode}" "${HADOOP_USER_PARAMS[@]}"

2
hadoop-yarn-project/hadoop-yarn/pom.xml
View File

@ -54,7 +54,7 @@
<artifactId>apache-rat-plugin</artifactId> <artifactId>apache-rat-plugin</artifactId>
<configuration> <configuration>
<excludes> <excludes>
<exclude>conf/slaves</exclude> <exclude>conf/workers</exclude>
<exclude>conf/container-executor.cfg</exclude> <exclude>conf/container-executor.cfg</exclude>
<exclude>dev-support/jdiff/**</exclude> <exclude>dev-support/jdiff/**</exclude>
</excludes> </excludes>