HADOOP-13673. Update scripts to be smarter when running with privilege
Signed-off-by: Andrew Wang <wang@apache.org> Signed-off-by: Ravi Prakash <raviprak@apache.org>
This commit is contained in:
Allen Wittenauer
parent
acda1a5fee
commit
0eb4b513b7
|
|
@ -183,13 +183,24 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# = 0 ]; then
|
||||
# now that we have support code, let's abs MYNAME so we can use it later
|
||||
MYNAME=$(hadoop_abs "${MYNAME}")
|
||||
|
||||
if [[ $# = 0 ]]; then
|
||||
hadoop_exit_with_usage 1
|
||||
fi
|
||||
|
||||
HADOOP_SUBCMD=$1
|
||||
shift
|
||||
|
||||
if hadoop_need_reexec hadoop "${HADOOP_SUBCMD}"; then
|
||||
hadoop_uservar_su hadoop "${HADOOP_SUBCMD}" \
|
||||
"${MYNAME}" \
|
||||
"--reexec" \
|
||||
"${HADOOP_USER_PARAMS[@]}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
||||
|
||||
HADOOP_SUBCMD_ARGS=("$@")
|
||||
|
|
|
|||
|
|
@ -41,6 +41,42 @@ function hadoop_debug
|
|||
fi
|
||||
}
|
||||
|
||||
## @description Given a filename or dir, return the absolute version of it
|
||||
## @description This works as an alternative to readlink, which isn't
|
||||
## @description portable.
|
||||
## @audience public
|
||||
## @stability stable
|
||||
## @param fsobj
|
||||
## @replaceable no
|
||||
## @return 0 success
|
||||
## @return 1 failure
|
||||
## @return stdout abspath
|
||||
function hadoop_abs
|
||||
{
|
||||
declare obj=$1
|
||||
declare dir
|
||||
declare fn
|
||||
declare dirret
|
||||
|
||||
if [[ ! -e ${obj} ]]; then
|
||||
return 1
|
||||
elif [[ -d ${obj} ]]; then
|
||||
dir=${obj}
|
||||
else
|
||||
dir=$(dirname -- "${obj}")
|
||||
fn=$(basename -- "${obj}")
|
||||
fn="/${fn}"
|
||||
fi
|
||||
|
||||
dir=$(cd -P -- "${dir}" >/dev/null 2>/dev/null && pwd -P)
|
||||
dirret=$?
|
||||
if [[ ${dirret} = 0 ]]; then
|
||||
echo "${dir}${fn}"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
## @description Given variable $1 delete $2 from it
|
||||
## @audience public
|
||||
## @stability stable
|
||||
|
|
@ -79,6 +115,101 @@ function hadoop_verify_entry
|
|||
[[ ${!1} =~ \ ${2}\ ]]
|
||||
}
|
||||
|
||||
## @description Check if we are running with privilege
|
||||
## @description by default, this implementation looks for
|
||||
## @description EUID=0. For OSes that have true privilege
|
||||
## @description separation, this should be something more complex
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable yes
|
||||
## @return 1 = no priv
|
||||
## @return 0 = priv
|
||||
function hadoop_privilege_check
|
||||
{
|
||||
[[ "${EUID}" = 0 ]]
|
||||
}
|
||||
|
||||
## @description Execute a command via su when running as root
|
||||
## @description if the given user is found or exit with
|
||||
## @description failure if not.
|
||||
## @description otherwise just run it. (This is intended to
|
||||
## @description be used by the start-*/stop-* scripts.)
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable yes
|
||||
## @param user
|
||||
## @param commandstring
|
||||
## @return exitstatus
|
||||
function hadoop_su
|
||||
{
|
||||
declare user=$1
|
||||
shift
|
||||
declare idret
|
||||
|
||||
if hadoop_privilege_check; then
|
||||
id -u "${user}" >/dev/null 2>&1
|
||||
idret=$?
|
||||
if [[ ${idret} != 0 ]]; then
|
||||
hadoop_error "ERROR: Refusing to run as root: ${user} account is not found. Aborting."
|
||||
return 1
|
||||
else
|
||||
su -l "${user}" -- "$@"
|
||||
fi
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
## @description Execute a command via su when running as root
|
||||
## @description with extra support for commands that might
|
||||
## @description legitimately start as root (e.g., datanode)
|
||||
## @description (This is intended to
|
||||
## @description be used by the start-*/stop-* scripts.)
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
## @param user
|
||||
## @param commandstring
|
||||
## @return exitstatus
|
||||
function hadoop_uservar_su
|
||||
{
|
||||
|
||||
## startup matrix:
|
||||
#
|
||||
# if $EUID != 0, then exec
|
||||
# if $EUID =0 then
|
||||
# if hdfs_subcmd_user is defined, call hadoop_su to exec
|
||||
# if hdfs_subcmd_user is not defined, error
|
||||
#
|
||||
# For secure daemons, this means both the secure and insecure env vars need to be
|
||||
# defined. e.g., HDFS_DATANODE_USER=root HDFS_DATANODE_SECURE_USER=hdfs
|
||||
# This function will pick up the "normal" var, switch to that user, then
|
||||
# execute the command which will then pick up the "secure" version.
|
||||
#
|
||||
|
||||
declare program=$1
|
||||
declare command=$2
|
||||
shift 2
|
||||
|
||||
declare uprogram
|
||||
declare ucommand
|
||||
declare uvar
|
||||
|
||||
if hadoop_privilege_check; then
|
||||
uvar=$(hadoop_get_verify_uservar "${program}" "${command}")
|
||||
|
||||
if [[ -n "${!uvar}" ]]; then
|
||||
hadoop_su "${!uvar}" "$@"
|
||||
else
|
||||
hadoop_error "ERROR: Attempting to launch ${program} ${command} as root"
|
||||
hadoop_error "ERROR: but there is no ${uvar} defined. Aborting launch."
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
## @description Add a subcommand to the usage output
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
|
|
@ -343,6 +474,9 @@ function hadoop_bootstrap
|
|||
# daemonization
|
||||
HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
|
||||
|
||||
# by default, we have not been self-re-execed
|
||||
HADOOP_REEXECED_CMD=false
|
||||
|
||||
# shellcheck disable=SC2034
|
||||
HADOOP_SUBCMD_SECURESERVICE=false
|
||||
|
||||
|
|
@ -624,9 +758,10 @@ function hadoop_basic_init
|
|||
fi
|
||||
|
||||
# if for some reason the shell doesn't have $USER defined
|
||||
# let's define it as 'hadoop'
|
||||
# (e.g., ssh'd in to execute a command)
|
||||
# let's get the effective username and use that
|
||||
USER=${USER:-$(id -nu)}
|
||||
HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-$USER}
|
||||
HADOOP_IDENT_STRING=${HADOOP_IDENT_STRING:-hadoop}
|
||||
HADOOP_LOG_DIR=${HADOOP_LOG_DIR:-"${HADOOP_HOME}/logs"}
|
||||
HADOOP_LOGFILE=${HADOOP_LOGFILE:-hadoop.log}
|
||||
HADOOP_LOGLEVEL=${HADOOP_LOGLEVEL:-INFO}
|
||||
|
|
@ -1400,8 +1535,7 @@ function hadoop_verify_secure_prereq
|
|||
# and you are using pfexec, you'll probably want to change
|
||||
# this.
|
||||
|
||||
# ${EUID} comes from the shell itself!
|
||||
if [[ "${EUID}" -ne 0 ]] && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
if ! hadoop_privilege_check && [[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
hadoop_error "ERROR: You must be a privileged user in order to run a secure service."
|
||||
exit 1
|
||||
else
|
||||
|
|
@ -1994,20 +2128,18 @@ function hadoop_secure_daemon_handler
|
|||
esac
|
||||
}
|
||||
|
||||
## @description Verify that ${USER} is allowed to execute the
|
||||
## @description given subcommand.
|
||||
## @description Get the environment variable used to validate users
|
||||
## @audience public
|
||||
## @stability stable
|
||||
## @replaceable yes
|
||||
## @param subcommand
|
||||
## @return will exit on failure conditions
|
||||
function hadoop_verify_user
|
||||
## @return string
|
||||
function hadoop_get_verify_uservar
|
||||
{
|
||||
declare program=$1
|
||||
declare command=$2
|
||||
declare uprogram
|
||||
declare ucommand
|
||||
declare uvar
|
||||
|
||||
if [[ -z "${BASH_VERSINFO[0]}" ]] \
|
||||
|| [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
|
||||
|
|
@ -2018,7 +2150,25 @@ function hadoop_verify_user
|
|||
ucommand=${command^^}
|
||||
fi
|
||||
|
||||
uvar="${uprogram}_${ucommand}_USER"
|
||||
echo "${uprogram}_${ucommand}_USER"
|
||||
}
|
||||
|
||||
## @description Verify that ${USER} is allowed to execute the
|
||||
## @description given subcommand.
|
||||
## @audience public
|
||||
## @stability stable
|
||||
## @replaceable yes
|
||||
## @param command
|
||||
## @param subcommand
|
||||
## @return return 0 on success
|
||||
## @return exit 1 on failure
|
||||
function hadoop_verify_user
|
||||
{
|
||||
declare program=$1
|
||||
declare command=$2
|
||||
declare uvar
|
||||
|
||||
uvar=$(hadoop_get_verify_uservar "${program}" "${command}")
|
||||
|
||||
if [[ -n ${!uvar} ]]; then
|
||||
if [[ ${!uvar} != "${USER}" ]]; then
|
||||
|
|
@ -2026,6 +2176,42 @@ function hadoop_verify_user
|
|||
exit 1
|
||||
fi
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
## @description Verify that ${USER} is allowed to execute the
|
||||
## @description given subcommand.
|
||||
## @audience public
|
||||
## @stability stable
|
||||
## @replaceable yes
|
||||
## @param subcommand
|
||||
## @return 1 on no re-exec needed
|
||||
## @return 0 on need to re-exec
|
||||
function hadoop_need_reexec
|
||||
{
|
||||
declare program=$1
|
||||
declare command=$2
|
||||
declare uvar
|
||||
|
||||
# we've already been re-execed, bail
|
||||
|
||||
if [[ "${HADOOP_REEXECED_CMD}" = true ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
# if we have privilege, and the _USER is defined, and _USER is
|
||||
# set to someone who isn't us, then yes, we should re-exec.
|
||||
# otherwise no, don't re-exec and let the system deal with it.
|
||||
|
||||
if hadoop_privilege_check; then
|
||||
uvar=$(hadoop_get_verify_uservar "${program}" "${command}")
|
||||
if [[ -n ${!uvar} ]]; then
|
||||
if [[ ${!uvar} != "${USER}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
## @description Add custom (program)_(command)_OPTS to HADOOP_OPTS.
|
||||
|
|
@ -2228,6 +2414,15 @@ function hadoop_parse_args
|
|||
shift
|
||||
((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+2))
|
||||
;;
|
||||
--reexec)
|
||||
shift
|
||||
if [[ "${HADOOP_REEXECED_CMD}" = true ]]; then
|
||||
hadoop_error "ERROR: re-exec fork bomb prevention: --reexec already called"
|
||||
exit 1
|
||||
fi
|
||||
HADOOP_REEXECED_CMD=true
|
||||
((HADOOP_PARSE_COUNTER=HADOOP_PARSE_COUNTER+1))
|
||||
;;
|
||||
--workers)
|
||||
shift
|
||||
# shellcheck disable=SC2034
|
||||
|
|
|
|||
|
|
@ -15,10 +15,14 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
echo "This script is deprecated. Use start-dfs.sh and start-yarn.sh instead."
|
||||
exit 1
|
||||
|
||||
|
||||
## @description catch the ctrl-c
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_abort_startall()
|
||||
{
|
||||
exit 1
|
||||
}
|
||||
|
||||
# let's locate libexec...
|
||||
if [[ -n "${HADOOP_HOME}" ]]; then
|
||||
|
|
@ -38,6 +42,16 @@ else
|
|||
echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hadoop-config.sh." 2>&1
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! hadoop_privilege_check; then
|
||||
trap hadoop_abort_startall INT
|
||||
hadoop_error "WARNING: Attempting to start all Apache Hadoop daemons as ${USER} in 10 seconds."
|
||||
hadoop_error "WARNING: This is not a recommended production deployment configuration."
|
||||
hadoop_error "WARNING: Use CTRL-C to abort."
|
||||
sleep 10
|
||||
trap - INT
|
||||
fi
|
||||
|
||||
# start hdfs daemons if hdfs is present
|
||||
if [[ -f "${HADOOP_HDFS_HOME}/sbin/start-dfs.sh" ]]; then
|
||||
"${HADOOP_HDFS_HOME}/sbin/start-dfs.sh" --config "${HADOOP_CONF_DIR}"
|
||||
|
|
@ -49,4 +63,3 @@ if [[ -f "${HADOOP_YARN_HOME}/sbin/start-yarn.sh" ]]; then
|
|||
fi
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,12 +15,17 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## @description catch the ctrl-c
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_abort_stopall()
|
||||
{
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Stop all hadoop daemons. Run this on master node.
|
||||
|
||||
echo "This script is deprecated. Use stop-dfs.sh and stop-yarn.sh instead."
|
||||
exit 1
|
||||
|
||||
# let's locate libexec...
|
||||
if [[ -n "${HADOOP_HOME}" ]]; then
|
||||
HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
|
||||
|
|
@ -40,6 +45,14 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if ! hadoop_privilege_check; then
|
||||
trap hadoop_abort_stopall INT
|
||||
hadoop_error "WARNING: Stopping all Apache Hadoop daemons as ${USER} in 10 seconds."
|
||||
hadoop_error "WARNING: Use CTRL-C to abort."
|
||||
sleep 10
|
||||
trap - INT
|
||||
fi
|
||||
|
||||
# stop hdfs daemons if hdfs is present
|
||||
if [[ -f "${HADOOP_HDFS_HOME}/sbin/stop-dfs.sh" ]]; then
|
||||
"${HADOOP_HDFS_HOME}/sbin/stop-dfs.sh" --config "${HADOOP_CONF_DIR}"
|
||||
|
|
|
|||
|
|
@ -103,6 +103,15 @@ In addition, daemons that run in an extra security mode also support `(command)_
|
|||
|
||||
Apache Hadoop provides a way to do a user check per-subcommand. While this method is easily circumvented and should not be considered a security-feature, it does provide a mechanism by which to prevent accidents. For example, setting `HDFS_NAMENODE_USER=hdfs` will make the `hdfs namenode` and `hdfs --daemon start namenode` commands verify that the user running the commands are the hdfs user by checking the `USER` environment variable. This also works for non-daemons. Setting `HADOOP_DISTCP_USER=jane` will verify that `USER` is set to `jane` before being allowed to execute the `hadoop distcp` command.
|
||||
|
||||
If a \_USER environment variable exists and commands are run with a privilege (e.g., as root; see hadoop_privilege_check in the API documentation), execution will switch to the specified user. For commands that support user account switching for security and therefore have a SECURE\_USER variable, the base \_USER variable needs to be the user that is expected to be used to switch to the SECURE\_USER account. For example:
|
||||
|
||||
```bash
|
||||
HDFS_DATANODE_USER=root
|
||||
HDFS_DATANODE_SECURE_USER=hdfs
|
||||
```
|
||||
|
||||
Be aware that if the \-\-workers flag is used, the user switch happens *after* ssh is invoked. The multi-daemon start and stop commands in sbin will, however, switch (if appropriate) prior and will therefore use the keys of the specified \_USER.
|
||||
|
||||
## Developer and Advanced Administrator Environment
|
||||
|
||||
### Shell Profiles
|
||||
|
|
|
|||
|
|
@ -16,9 +16,9 @@
|
|||
|
||||
setup() {
|
||||
|
||||
TMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}"
|
||||
mkdir -p ${TMP}
|
||||
TMP=$(cd -P -- "${TMP}" >/dev/null && pwd -P)
|
||||
RELTMP="${BATS_TEST_DIRNAME}/../../../target/test-dir/bats.$$.${RANDOM}"
|
||||
mkdir -p ${RELTMP}
|
||||
TMP=$(cd -P -- "${RELTMP}" >/dev/null && pwd -P)
|
||||
export TMP
|
||||
TESTBINDIR=$(cd -P -- "$(pwd)" >/dev/null && pwd -P)
|
||||
HADOOP_LIBEXEC_DIR=${TESTBINDIR}/../../main/bin
|
||||
|
|
|
|||
|
|
@ -0,0 +1,65 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load hadoop-functions_test_helper
|
||||
|
||||
create_fake () {
|
||||
mkdir ${TMP}/j
|
||||
touch ${TMP}/j/k
|
||||
ln -s j ${TMP}/l
|
||||
}
|
||||
|
||||
|
||||
@test "hadoop_abs (simple not exist)" {
|
||||
run hadoop_abs fake
|
||||
[ "${status}" -eq 1 ]
|
||||
}
|
||||
|
||||
@test "hadoop_abs (simple dir)" {
|
||||
create_fake
|
||||
run hadoop_abs "${TMP}/j"
|
||||
[ "${output}" = "${TMP}/j" ]
|
||||
}
|
||||
|
||||
@test "hadoop_abs (simple file)" {
|
||||
create_fake
|
||||
run hadoop_abs "${TMP}/j/k"
|
||||
[ "${output}" = "${TMP}/j/k" ]
|
||||
}
|
||||
|
||||
@test "hadoop_abs (relative file1)" {
|
||||
create_fake
|
||||
run hadoop_abs "${TMP}/j/../j/k"
|
||||
[ "${output}" = "${TMP}/j/k" ]
|
||||
}
|
||||
|
||||
@test "hadoop_abs (relative file2)" {
|
||||
create_fake
|
||||
run hadoop_abs "${RELTMP}/j/../j/k"
|
||||
[ "${output}" = "${TMP}/j/k" ]
|
||||
}
|
||||
|
||||
@test "hadoop_abs (relative dir)" {
|
||||
create_fake
|
||||
fred=$(cd -P -- ".." >/dev/null && pwd -P)
|
||||
run hadoop_abs ".."
|
||||
[ "${output}" = "${fred}" ]
|
||||
}
|
||||
|
||||
@test "hadoop_abs (symlink)" {
|
||||
create_fake
|
||||
run hadoop_abs "${TMP}/l"
|
||||
[ "${output}" = "${TMP}/j" ]
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load hadoop-functions_test_helper
|
||||
|
||||
@test "hadoop_get_verify_uservar" {
|
||||
run hadoop_get_verify_uservar cool program
|
||||
[ "${output}" = "COOL_PROGRAM_USER" ]
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
load hadoop-functions_test_helper
|
||||
|
||||
@test "hadoop_privilege_check " {
|
||||
if [ ${EUID} = 0 ]; then
|
||||
result=0
|
||||
else
|
||||
result=1
|
||||
fi
|
||||
run hadoop_privilege_check
|
||||
[ "${status}" = "${result}" ]
|
||||
}
|
||||
|
|
@ -239,6 +239,9 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# now that we have support code, let's abs MYNAME so we can use it later
|
||||
MYNAME=$(hadoop_abs "${MYNAME}")
|
||||
|
||||
if [[ $# = 0 ]]; then
|
||||
hadoop_exit_with_usage 1
|
||||
fi
|
||||
|
|
@ -246,6 +249,14 @@ fi
|
|||
HADOOP_SUBCMD=$1
|
||||
shift
|
||||
|
||||
if hadoop_need_reexec hdfs "${HADOOP_SUBCMD}"; then
|
||||
hadoop_uservar_su hdfs "${HADOOP_SUBCMD}" \
|
||||
"${MYNAME}" \
|
||||
"--reexec" \
|
||||
"${HADOOP_USER_PARAMS[@]}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
||||
|
||||
HADOOP_SUBCMD_ARGS=("$@")
|
||||
|
|
|
|||
|
|
@ -20,6 +20,21 @@
|
|||
# Optinally upgrade or rollback dfs state.
|
||||
# Run this on master node.
|
||||
|
||||
## startup matrix:
|
||||
#
|
||||
# if $EUID != 0, then exec
|
||||
# if $EUID =0 then
|
||||
# if hdfs_subcmd_user is defined, su to that user, exec
|
||||
# if hdfs_subcmd_user is not defined, error
|
||||
#
|
||||
# For secure daemons, this means both the secure and insecure env vars need to be
|
||||
# defined. e.g., HDFS_DATANODE_USER=root HADOOP_SECURE_DN_USER=hdfs
|
||||
#
|
||||
|
||||
## @description usage info
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_usage
|
||||
{
|
||||
echo "Usage: start-dfs.sh [-upgrade|-rollback] [-clusterId]"
|
||||
|
|
@ -45,7 +60,6 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# get arguments
|
||||
if [[ $# -ge 1 ]]; then
|
||||
startOpt="$1"
|
||||
|
|
@ -77,32 +91,25 @@ if [[ -z "${NAMENODES}" ]]; then
|
|||
fi
|
||||
|
||||
echo "Starting namenodes on [${NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs namenode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
--daemon start \
|
||||
namenode ${nameStartOpt}
|
||||
|
||||
HADOOP_JUMBO_RETCOUNTER=$?
|
||||
|
||||
#---------------------------------------------------------
|
||||
# datanodes (using default workers file)
|
||||
|
||||
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
|
||||
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
hadoop_error "ERROR: Attempting to start secure cluster, skipping datanodes. "
|
||||
hadoop_error "ERROR: Run start-secure-dns.sh as root or configure "
|
||||
hadoop_error "ERROR: \${HADOOP_SECURE_COMMAND} to complete startup."
|
||||
else
|
||||
|
||||
echo "Starting datanodes"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
echo "Starting datanodes"
|
||||
hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
datanode ${dataStartOpt}
|
||||
fi
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
|
||||
#---------------------------------------------------------
|
||||
# secondary namenodes (if any)
|
||||
|
|
@ -113,8 +120,8 @@ if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
|||
|
||||
if [[ "${NAMENODES}" =~ , ]]; then
|
||||
|
||||
hadoop_error "ERROR: Highly available NameNode is configured."
|
||||
hadoop_error "ERROR: Skipping SecondaryNameNode."
|
||||
hadoop_error "WARNING: Highly available NameNode is configured."
|
||||
hadoop_error "WARNING: Skipping SecondaryNameNode."
|
||||
|
||||
else
|
||||
|
||||
|
|
@ -124,12 +131,13 @@ if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
|||
|
||||
echo "Starting secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs secondarynamenode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${SECONDARY_NAMENODES}" \
|
||||
--daemon start \
|
||||
secondarynamenode
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
@ -143,12 +151,13 @@ case "${SHARED_EDITS_DIR}" in
|
|||
JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
||||
echo "Starting journal nodes [${JOURNAL_NODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${JOURNAL_NODES}" \
|
||||
--daemon start \
|
||||
journalnode
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
;;
|
||||
esac
|
||||
|
||||
|
|
@ -158,12 +167,15 @@ AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automati
|
|||
if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
|
||||
echo "Starting ZK Failover Controllers on NN hosts [${NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs zkfc "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
--daemon start \
|
||||
zkfc
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
fi
|
||||
|
||||
exit ${HADOOP_JUMBO_RETCOUNTER}
|
||||
|
||||
# eof
|
||||
|
|
|
|||
|
|
@ -17,8 +17,12 @@
|
|||
|
||||
# Run as root to start secure datanodes in a security-enabled cluster.
|
||||
|
||||
|
||||
function hadoop_usage {
|
||||
## @description usage info
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_usage()
|
||||
{
|
||||
echo "Usage: start-secure-dns.sh"
|
||||
}
|
||||
|
||||
|
|
@ -42,12 +46,9 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
echo "Starting datanodes"
|
||||
hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
datanode
|
||||
else
|
||||
echo hadoop_usage_and_exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -19,9 +19,13 @@
|
|||
# Stop hadoop dfs daemons.
|
||||
# Run this on master node.
|
||||
|
||||
## @description usage info
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_usage
|
||||
{
|
||||
echo "Usage: stop-dfs.sh [-upgrade|-rollback] [-clusterId]"
|
||||
echo "Usage: stop-dfs.sh"
|
||||
}
|
||||
|
||||
this="${BASH_SOURCE-$0}"
|
||||
|
|
@ -55,7 +59,7 @@ fi
|
|||
|
||||
echo "Stopping namenodes on [${NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs namenode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
|
|
@ -65,21 +69,13 @@ echo "Stopping namenodes on [${NAMENODES}]"
|
|||
#---------------------------------------------------------
|
||||
# datanodes (using default workers file)
|
||||
|
||||
if [[ -n "${HADOOP_SECURE_DN_USER}" ]] &&
|
||||
[[ -z "${HADOOP_SECURE_COMMAND}" ]]; then
|
||||
echo "ERROR: Attempting to stop secure cluster, skipping datanodes. "
|
||||
echo "Run stop-secure-dns.sh as root or configure "
|
||||
echo "\${HADOOP_SECURE_COMMAND} to complete stop."
|
||||
else
|
||||
echo "Stopping datanodes"
|
||||
|
||||
echo "Stopping datanodes"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
datanode
|
||||
fi
|
||||
|
||||
#---------------------------------------------------------
|
||||
# secondary namenodes (if any)
|
||||
|
|
@ -93,7 +89,7 @@ fi
|
|||
if [[ -n "${SECONDARY_NAMENODES}" ]]; then
|
||||
echo "Stopping secondary namenodes [${SECONDARY_NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs secondarynamenode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${SECONDARY_NAMENODES}" \
|
||||
|
|
@ -111,7 +107,7 @@ case "${SHARED_EDITS_DIR}" in
|
|||
JOURNAL_NODES=$(echo "${SHARED_EDITS_DIR}" | sed 's,qjournal://\([^/]*\)/.*,\1,g; s/;/ /g; s/:[0-9]*//g')
|
||||
echo "Stopping journal nodes [${JOURNAL_NODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs journalnode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${JOURNAL_NODES}" \
|
||||
|
|
@ -126,7 +122,7 @@ AUTOHA_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey dfs.ha.automati
|
|||
if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
|
||||
echo "Stopping ZK Failover Controllers on NN hosts [${NAMENODES}]"
|
||||
|
||||
"${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
hadoop_uservar_su hdfs zkfc "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--hostnames "${NAMENODES}" \
|
||||
|
|
|
|||
|
|
@ -17,8 +17,12 @@
|
|||
|
||||
# Run as root to stop secure datanodes in a security-enabled cluster.
|
||||
|
||||
|
||||
function hadoop_usage {
|
||||
## @description usage info
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_usage()
|
||||
{
|
||||
echo "Usage: stop-secure-dns.sh"
|
||||
}
|
||||
|
||||
|
|
@ -42,12 +46,8 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${EUID}" -eq 0 ]] && [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then
|
||||
exec "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
hadoop_uservar_su hdfs datanode "${HADOOP_HDFS_HOME}/bin/hdfs" \
|
||||
--workers \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
datanode
|
||||
else
|
||||
echo hadoop_usage_and_exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -121,6 +121,8 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# now that we have support code, let's abs MYNAME so we can use it later
|
||||
MYNAME=$(hadoop_abs "${MYNAME}")
|
||||
|
||||
if [ $# = 0 ]; then
|
||||
hadoop_exit_with_usage 1
|
||||
|
|
@ -129,6 +131,14 @@ fi
|
|||
HADOOP_SUBCMD=$1
|
||||
shift
|
||||
|
||||
if hadoop_need_reexec mapred "${HADOOP_SUBCMD}"; then
|
||||
hadoop_uservar_su mapred "${HADOOP_SUBCMD}" \
|
||||
"${MYNAME}" \
|
||||
"--reexec" \
|
||||
"${HADOOP_USER_PARAMS[@]}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
||||
|
||||
HADOOP_SUBCMD_ARGS=("$@")
|
||||
|
|
|
|||
|
|
@ -15,14 +15,17 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
MYNAME="${BASH_SOURCE-$0}"
|
||||
|
||||
## @description usage info
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_usage
|
||||
{
|
||||
hadoop_generate_usage "${MYNAME}" false
|
||||
}
|
||||
|
||||
MYNAME="${BASH_SOURCE-$0}"
|
||||
|
||||
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
|
||||
|
||||
# let's locate libexec...
|
||||
|
|
@ -42,14 +45,17 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
HADOOP_JUMBO_RETCOUNTER=0
|
||||
|
||||
# start resourceManager
|
||||
HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-)
|
||||
if [[ ${HARM} = "false" ]]; then
|
||||
echo "Starting resourcemanager"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
resourcemanager
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
else
|
||||
logicals=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.rm-ids 2>&-)
|
||||
logicals=${logicals//,/ }
|
||||
|
|
@ -59,30 +65,35 @@ else
|
|||
RMHOSTS="${RMHOSTS} ${rmhost}"
|
||||
done
|
||||
echo "Starting resourcemanagers on [${RMHOSTS}]"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon start \
|
||||
--workers \
|
||||
--hostnames "${RMHOSTS}" \
|
||||
resourcemanager
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
fi
|
||||
|
||||
# start nodemanager
|
||||
echo "Starting nodemanagers"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn nodemanager "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--workers \
|
||||
--daemon start \
|
||||
nodemanager
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
|
||||
|
||||
# start proxyserver
|
||||
PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:)
|
||||
if [[ -n ${PROXYSERVER} ]]; then
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn proxyserver "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--workers \
|
||||
--hostnames "${PROXYSERVER}" \
|
||||
--daemon start \
|
||||
proxyserver
|
||||
(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
|
||||
fi
|
||||
|
||||
exit ${HADOOP_JUMBO_RETCOUNTER}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,17 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
MYNAME="${BASH_SOURCE-$0}"
|
||||
|
||||
## @description usage info
|
||||
## @audience private
|
||||
## @stability evolving
|
||||
## @replaceable no
|
||||
function hadoop_usage
|
||||
{
|
||||
hadoop_generate_usage "${MYNAME}" false
|
||||
}
|
||||
|
||||
MYNAME="${BASH_SOURCE-$0}"
|
||||
|
||||
bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P)
|
||||
|
||||
# let's locate libexec...
|
||||
|
|
@ -44,7 +47,7 @@ fi
|
|||
|
||||
# stop nodemanager
|
||||
echo "Stopping nodemanagers"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn nodemanager "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--workers \
|
||||
--daemon stop \
|
||||
|
|
@ -54,7 +57,7 @@ echo "Stopping nodemanagers"
|
|||
HARM=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.resourcemanager.ha.enabled 2>&-)
|
||||
if [[ ${HARM} = "false" ]]; then
|
||||
echo "Stopping resourcemanager"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
resourcemanager
|
||||
|
|
@ -67,7 +70,7 @@ else
|
|||
RMHOSTS="${RMHOSTS} ${rmhost}"
|
||||
done
|
||||
echo "Stopping resourcemanagers on [${RMHOSTS}]"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn resourcemanager "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--daemon stop \
|
||||
--workers \
|
||||
|
|
@ -79,7 +82,7 @@ fi
|
|||
PROXYSERVER=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey yarn.web-proxy.address 2>&- | cut -f1 -d:)
|
||||
if [[ -n ${PROXYSERVER} ]]; then
|
||||
echo "Stopping proxy server [${PROXYSERVER}]"
|
||||
"${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
hadoop_uservar_su yarn proxyserver "${HADOOP_YARN_HOME}/bin/yarn" \
|
||||
--config "${HADOOP_CONF_DIR}" \
|
||||
--workers \
|
||||
--hostnames "${PROXYSERVER}" \
|
||||
|
|
|
|||
|
|
@ -219,6 +219,9 @@ else
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# now that we have support code, let's abs MYNAME so we can use it later
|
||||
MYNAME=$(hadoop_abs "${MYNAME}")
|
||||
|
||||
# if no args specified, show usage
|
||||
if [[ $# = 0 ]]; then
|
||||
hadoop_exit_with_usage 1
|
||||
|
|
@ -228,6 +231,14 @@ fi
|
|||
HADOOP_SUBCMD=$1
|
||||
shift
|
||||
|
||||
if hadoop_need_reexec yarn "${HADOOP_SUBCMD}"; then
|
||||
hadoop_uservar_su yarn "${HADOOP_SUBCMD}" \
|
||||
"${MYNAME}" \
|
||||
"--reexec" \
|
||||
"${HADOOP_USER_PARAMS[@]}"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
|
||||
|
||||
HADOOP_SUBCMD_ARGS=("$@")
|
||||
|
|
|
|||
Loading…
Reference in New Issue