HADOOP-7655. Provide a small validation script that smoke tests the installed cluster. Contributed by Arpit Gupta.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1183544 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Matthew Foley 2011-10-14 22:45:36 +00:00
parent 1caff31a74
commit ffbe012b91
2 changed files with 184 additions and 0 deletions

View File

@ -68,6 +68,9 @@ Release 0.23.0 - Unreleased
IMPROVEMENTS
HADOOP-7655. Provide a small validation script that smoke tests the installed
cluster. (Arpit Gupta via mattf)
HADOOP-7042. Updates to test-patch.sh to include failed test names and
improve other messaging. (nigel)

View File

@ -0,0 +1,181 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###############################################################################
# Run the following jobs to validate a hadoop cluster
## teragen
## terasort
## teravalidate
# If they all pass 0 will be returned and 1 otherwise
# The test will work for both secure and unsecure deploys. If the kerberos-realm
# is passed we will assume that the deploy is secure and proceed with a kinit before
# running the validation jobs.
################################################################################
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
. "$bin"/../libexec/hadoop-config.sh
usage() {
echo "
usage: $0 <parameters>
Optional parameters:
-h Display this message
--user=hdfs
--user_keytab=/home/hdfs/hdfs.keytab
--kerberos-realm=KERBEROS.EXAMPLE.COM Set Kerberos realm
"
exit 1
}
OPTS=$(getopt \
-n $0 \
-o '' \
-l 'user:' \
-l 'user-keytab:' \
-l 'kerberos-realm:' \
-o 'h' \
-- "$@")
if [ $? != 0 ] ; then
usage
fi
eval set -- "${OPTS}"
while true ; do
case "$1" in
--user)
TEST_USER=$2; shift 2
AUTOMATED=1
;;
--user-keytab)
USER_KEYTAB_FILE=$2; shift 2
AUTOMATED=1
;;
--kerberos-realm)
KERBEROS_REALM=$2; shift 2
AUTOMATED=1
;;
--)
shift ; break
;;
*)
echo "Unknown option: $1"
usage
exit 1
;;
esac
done
#set the hadoop command and the path to the hadoop examples jar
HADOOP_CMD="${HADOOP_PREFIX}/bin/hadoop --config $HADOOP_CONF_DIR"
#find the hadoop examples jar
HADOOP_EXAMPLES_JAR=''
#find under HADOOP_PREFIX (tar ball install)
HADOOP_EXAMPLES_JAR=`find ${HADOOP_PREFIX} -name 'hadoop-examples-*.jar' | head -n1`
#if its not found look under /usr/share/hadoop (rpm/deb installs)
if [ "$HADOOP_EXAMPLES_JAR" == '' ]
then
HADOOP_EXAMPLES_JAR=`find /usr/share/hadoop -name 'hadoop-examples-*.jar' | head -n1`
fi
#if it is still empty then dont run the tests
if [ "$HADOOP_EXAMPLES_JAR" == '' ]
then
echo "Did not find hadoop-examples-*.jar under '${HADOOP_PREFIX} or '/usr/share/hadoop'"
exit 1
fi
# do a kinit if secure
if [ "${KERBEROS_REALM}" != "" ]; then
# Determine kerberos location base on Linux distro.
if [ -e /etc/lsb-release ]; then
KERBEROS_BIN=/usr/bin
else
KERBEROS_BIN=/usr/kerberos/bin
fi
kinit_cmd="su -c '${KERBEROS_BIN}/kinit -kt ${USER_KEYTAB_FILE} ${TEST_USER}' ${TEST_USER}"
echo $kinit_cmd
eval $kinit_cmd
if [ $? -ne 0 ]
then
echo "kinit command did not run successfully."
exit 1
fi
fi
#dir where to store the data on hdfs. The data is relative of the users home dir on hdfs.
PARENT_DIR="validate_deploy_`date +%s`"
TERA_GEN_OUTPUT_DIR="${PARENT_DIR}/tera_gen_data"
TERA_SORT_OUTPUT_DIR="${PARENT_DIR}/tera_sort_data"
TERA_VALIDATE_OUTPUT_DIR="${PARENT_DIR}/tera_validate_data"
#tera gen cmd
TERA_GEN_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teragen 10000 $TERA_GEN_OUTPUT_DIR' $TEST_USER"
#tera sort cmd
TERA_SORT_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR terasort $TERA_GEN_OUTPUT_DIR $TERA_SORT_OUTPUT_DIR' $TEST_USER"
#tera validate cmd
TERA_VALIDATE_CMD="su -c '$HADOOP_CMD jar $HADOOP_EXAMPLES_JAR teravalidate $TERA_SORT_OUTPUT_DIR $TERA_VALIDATE_OUTPUT_DIR' $TEST_USER"
echo "Starting teragen...."
#run tera gen
echo $TERA_GEN_CMD
eval $TERA_GEN_CMD
if [ $? -ne 0 ]; then
echo "tera gen failed."
exit 1
fi
echo "Teragen passed starting terasort...."
#run tera sort
echo $TERA_SORT_CMD
eval $TERA_SORT_CMD
if [ $? -ne 0 ]; then
echo "tera sort failed."
exit 1
fi
echo "Terasort passed starting teravalidate...."
#run tera validate
echo $TERA_VALIDATE_CMD
eval $TERA_VALIDATE_CMD
if [ $? -ne 0 ]; then
echo "tera validate failed."
exit 1
fi
echo "teragen, terasort, teravalidate passed."
echo "Cleaning the data created by tests: $PARENT_DIR"
CLEANUP_CMD="su -c '$HADOOP_CMD dfs -rmr -skipTrash $PARENT_DIR' $TEST_USER"
echo $CLEANUP_CMD
eval $CLEANUP_CMD
exit 0