diff --git a/dev-support/bin/dist-tools-hooks-maker b/dev-support/bin/dist-tools-hooks-maker new file mode 100755 index 00000000000..a98dffa7ad2 --- /dev/null +++ b/dev-support/bin/dist-tools-hooks-maker @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +VERSION=${1:-3.0.0-SNAPSHOT} +TARGETDIR=${2:-/tmp/target} +TOOLSDIR=${3:-/tmp/tools} + +function getfilename +{ + declare module=$1 + declare modtype=$2 + + if [[ ${modtype} = builtin ]]; then + echo "${TARGETDIR}/hadoop-${VERSION}/libexec/tools/${module}.sh" + else + echo "${TARGETDIR}/hadoop-${VERSION}/libexec/shellprofile.d/${module}.sh" + fi +} + +function header +{ + declare fn=$1 + + cat >>"${fn}" <<-'TOKEN' +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +# IMPORTANT: This file is automatically generated by hadoop-dist at +# -Pdist time. +# +# +TOKEN + +} + +function optional_prologue +{ + declare fn=$1 + declare module=$2 + + if [[ -z "${OPTMODS}" ]]; then + OPTMODS=${module} + else + OPTMODS="${OPTMODS},${module}" + fi + + { + echo "if hadoop_verify_entry HADOOP_TOOLS_OPTIONS \"${module}\"; then" + echo " hadoop_add_profile \"${module}\"" + echo "fi" + echo "" + echo "function _${module}_hadoop_classpath" + echo "{" + } >> "${fn}" +} + +function builtin_prologue +{ + declare fn=$1 + declare module=$2 + + { + echo "" + echo "function hadoop_classpath_tools_${module}" + echo "{" + } >> "${fn}" +} + +function dependencywork +{ + declare fn=$1 + declare module=$2 + declare depfn=$3 + + declare depline + declare jarname + + while read -r depline; do + jarname=$(echo "${depline}" | awk -F: '{print $2"-"$4".jar"}') + + if [[ -f "${TARGETDIR}/hadoop-${VERSION}/share/hadoop/tools/lib/${jarname}" ]]; then + { + echo " if [[ -f \"\${HADOOP_TOOLS_HOME}/\${HADOOP_TOOLS_LIB_JARS_DIR}/${jarname}\" ]]; then" + echo " hadoop_add_classpath \"\${HADOOP_TOOLS_HOME}/\${HADOOP_TOOLS_LIB_JARS_DIR}/${jarname}\"" + echo " fi" + } >> "${fn}" + + elif [[ -f "${TARGETDIR}/hadoop-${VERSION}/share/hadoop/common/${jarname}" + || -f "${TARGETDIR}/hadoop-${VERSION}/share/hadoop/common/lib/${jarname}" ]]; then + true + else + echo "ERROR: ${module} has missing dependencies: ${jarname}" + fi + done < <(grep compile "${depfn}") + + { + echo " hadoop_add_classpath \"\${HADOOP_TOOLS_HOME}/\${HADOOP_TOOLS_LIB_JARS_DIR}/${module}-${VERSION}.jar\"" + echo "}" + echo "" + } >> "${fn}" +} + +function document_optionals +{ + echo "Rewriting ${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh" + sed -e "s^@@@HADOOP_OPTIONAL_TOOLS@@@^${OPTMODS}^" \ + "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh" \ + > "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh.new" + mv "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh.new" \ + "${TARGETDIR}/hadoop-${VERSION}/etc/hadoop/hadoop-env.sh" +} + +function process +{ + declare fn + declare basefn + declare modtype + declare module + declare newfile + declare newdir + + while read -r fn; do + basefn=${fn##*/} + module=$(echo "${basefn}" | cut -f1 -d.) + modtype=$(echo "${basefn}" | cut -f2 -d.) + modtype=${modtype##tools-} + + newfile=$(getfilename "${module}" "${modtype}") + newdir=$(dirname "${newfile}") + mkdir -p "${newdir}" + + if [[ -f "${newfile}" ]]; then + rm "${newfile}" + fi + + touch "${newfile}" + + header "${newfile}" "${module}" + + "${modtype}_prologue" "${newfile}" "${module}" + + dependencywork "${newfile}" "${module}" "${fn}" + + chmod a+rx "${newfile}" + + done < <(find "${TOOLSDIR}" -name '*.tools-builtin.txt' -o -name '*.tools-optional.txt') + + document_optionals +} + +process diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 766becd50f6..46eaf273687 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -114,7 +114,7 @@ case ${COMMAND} in ;; archive) CLASS=org.apache.hadoop.tools.HadoopArchives - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-archives ;; checknative) CLASS=org.apache.hadoop.util.NativeLibraryChecker @@ -133,11 +133,11 @@ case ${COMMAND} in ;; distch) CLASS=org.apache.hadoop.tools.DistCh - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-extras ;; distcp) CLASS=org.apache.hadoop.tools.DistCp - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-distcp ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" @@ -146,7 +146,9 @@ case ${COMMAND} in echo "HADOOP_COMMON_LIB_JARS_DIR='${HADOOP_COMMON_LIB_JARS_DIR}'" echo "HADOOP_COMMON_LIB_NATIVE_DIR='${HADOOP_COMMON_LIB_NATIVE_DIR}'" echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" - echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'" + echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'" + echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'" + echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; fs) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index fa52e226902..7f293b64754 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -41,6 +41,44 @@ function hadoop_debug fi } +## @description Given variable $1 delete $2 from it +## @audience public +## @stability stable +## @replaceable no +function hadoop_delete_entry +{ + if [[ ${!1} =~ \ ${2}\ ]] ; then + hadoop_debug "Removing ${2} from ${1}" + eval "${1}"=\""${!1// ${2} }"\" + fi +} + +## @description Given variable $1 add $2 to it +## @audience public +## @stability stable +## @replaceable no +function hadoop_add_entry +{ + if [[ ! ${!1} =~ \ ${2}\ ]] ; then + hadoop_debug "Adding ${2} to ${1}" + #shellcheck disable=SC2140 + eval "${1}"=\""${!1} ${2} "\" + fi +} + +## @description Given variable $1 determine if $2 is in it +## @audience public +## @stability stable +## @replaceable no +## @return 0 = yes, 1 = no +function hadoop_verify_entry +{ + # this unfortunately can't really be tested by bats. :( + # so if this changes, be aware that unit tests effectively + # do this function in them + [[ ${!1} =~ \ ${2}\ ]] +} + ## @description Add a subcommand to the usage output ## @audience private ## @stability evolving @@ -264,10 +302,9 @@ function hadoop_bootstrap YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"} MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"} MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"} - - # setup a default HADOOP_TOOLS_PATH - hadoop_deprecate_envvar TOOL_PATH HADOOP_TOOLS_PATH - HADOOP_TOOLS_PATH=${HADOOP_TOOLS_PATH:-${HADOOP_PREFIX}/share/hadoop/tools/lib/*} + HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_PREFIX}} + HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"} + HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"} # usage output set to zero hadoop_reset_usage @@ -322,6 +359,7 @@ function hadoop_exec_hadoopenv if [[ -z "${HADOOP_ENV_PROCESSED}" ]]; then if [[ -f "${HADOOP_CONF_DIR}/hadoop-env.sh" ]]; then export HADOOP_ENV_PROCESSED=true + # shellcheck disable=SC1090 . "${HADOOP_CONF_DIR}/hadoop-env.sh" fi fi @@ -334,6 +372,7 @@ function hadoop_exec_hadoopenv function hadoop_exec_userfuncs { if [[ -e "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" ]]; then + # shellcheck disable=SC1090 . "${HADOOP_CONF_DIR}/hadoop-user-functions.sh" fi } @@ -348,6 +387,7 @@ function hadoop_exec_hadooprc { if [[ -f "${HOME}/.hadooprc" ]]; then hadoop_debug "Applying the user's .hadooprc" + # shellcheck disable=SC1090 . "${HOME}/.hadooprc" fi } @@ -373,11 +413,22 @@ function hadoop_import_shellprofiles files2=(${HADOOP_CONF_DIR}/shellprofile.d/*.sh) fi + # enable bundled shellprofiles that come + # from hadoop-tools. This converts the user-facing HADOOP_OPTIONAL_TOOLS + # to the HADOOP_TOOLS_OPTIONS that the shell profiles expect. + # See dist-tools-hooks-maker for how the example HADOOP_OPTIONAL_TOOLS + # gets populated into hadoop-env.sh + + for i in ${HADOOP_OPTIONAL_TOOLS//,/ }; do + hadoop_add_entry HADOOP_TOOLS_OPTIONS "${i}" + done + for i in "${files1[@]}" "${files2[@]}" do if [[ -n "${i}" && -f "${i}" ]]; then hadoop_debug "Profiles: importing ${i}" + # shellcheck disable=SC1090 . "${i}" fi done @@ -945,34 +996,25 @@ function hadoop_add_common_to_classpath hadoop_add_classpath "${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR}"'/*' } -## @description Add the HADOOP_TOOLS_PATH to the classpath +## @description Run libexec/tools/module.sh to add to the classpath ## @description environment ## @audience private ## @stability evolving ## @replaceable yes -function hadoop_add_to_classpath_toolspath +## @param module +function hadoop_add_to_classpath_tools { - declare -a array - declare -i c=0 - declare -i j - declare -i i - declare idx + declare module=$1 - if [[ -n "${HADOOP_TOOLS_PATH}" ]]; then - hadoop_debug "Adding HADOOP_TOOLS_PATH to CLASSPATH" - oldifs=${IFS} - IFS=: - for idx in ${HADOOP_TOOLS_PATH}; do - array[${c}]=${idx} - ((c=c+1)) - done - IFS=${oldifs} - ((j=c-1)) || ${QATESTMODE} - - for ((i=0; i<=j; i++)); do - hadoop_add_classpath "${array[$i]}" after - done + if [[ -f "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh" ]]; then + # shellcheck disable=SC1090 + . "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh" + else + hadoop_error "ERROR: Tools helper ${HADOOP_LIBEXEC_DIR}/tools/${module}.sh was not found." + fi + if declare -f hadoop_classpath_tools_${module} >/dev/null 2>&1; then + "hadoop_classpath_tools_${module}" fi } diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example old mode 100644 new mode 100755 index a3f7bde1bbc..faa431740c0 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-layout.sh.example @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Copyright 2014 The Apache Software Foundation # # Licensed to the Apache Software Foundation (ASF) under one @@ -87,7 +88,10 @@ # Misc paths #### -# setup a default HADOOP_TOOLS_PATH, where things like distcp lives +# This is where things like distcp, S3, and other things live # note that this path only gets added for certain commands and not -# part of the general classpath -# export HADOOP_TOOLS_PATH="${HADOOP_PREFIX}/share/hadoop/tools/lib/*" +# part of the general classpath unless HADOOP_OPTIONAL_TOOLS is used +# to configure them in +# export HADOOP_TOOLS_HOME=${HADOOP_PREFIX} +# export HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"} +# export HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"} diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index 04aad8b3562..3c554aa8d02 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -49,7 +49,7 @@ # preferred. Many sites configure these options outside of Hadoop, # such as in /etc/profile.d -# The java implementation to use. By default, this environment +# The java implementation to use. By default, this environment # variable is REQUIRED on ALL platforms except OS X! # export JAVA_HOME= @@ -64,15 +64,15 @@ # path. # export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop -# The maximum amount of heap to use (Java -Xmx). If no unit -# is provided, it will be converted to MB. Daemons will +# The maximum amount of heap to use (Java -Xmx). If no unit +# is provided, it will be converted to MB. Daemons will # prefer any Xmx setting in their respective _OPT variable. # There is no default; the JVM will autoscale based upon machine # memory size. # export HADOOP_HEAPSIZE_MAX= -# The minimum amount of heap to use (Java -Xms). If no unit -# is provided, it will be converted to MB. Daemons will +# The minimum amount of heap to use (Java -Xms). If no unit +# is provided, it will be converted to MB. Daemons will # prefer any Xms setting in their respective _OPT variable. # There is no default; the JVM will autoscale based upon machine # memory size. @@ -107,8 +107,8 @@ case ${HADOOP_OS_TYPE} in esac # Extra Java runtime options for some Hadoop commands -# and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for -# such commands. In most cases, # this should be left empty and +# and clients (i.e., hdfs dfs -blah). These get appended to HADOOP_OPTS for +# such commands. In most cases, # this should be left empty and # let users supply it on the command line. # export HADOOP_CLIENT_OPTS="" @@ -146,6 +146,11 @@ esac # names starting with a '-' are treated as negative matches. For example, # export HADOOP_CLIENT_CLASSLOADER_SYSTEM_CLASSES="-org.apache.hadoop.UserClass,java.,javax.,org.apache.hadoop." +# Enable optional, bundled Hadoop features +# This is a comma delimited list. It may NOT be overridden via .hadooprc +# Entries may be added/removed as needed. +# export HADOOP_OPTIONAL_TOOLS="@@@HADOOP_OPTIONAL_TOOLS@@@" + ### # Options for remote shell connectivity ### @@ -181,7 +186,7 @@ esac # non-secure) # -# Where (primarily) daemon log files are stored. # $HADOOP_PREFIX/logs +# Where (primarily) daemon log files are stored. # $HADOOP_PREFIX/logs # by default. # Java property: hadoop.log.dir # export HADOOP_LOG_DIR=${HADOOP_PREFIX}/logs @@ -201,7 +206,7 @@ esac # Java property: hadoop.root.logger # export HADOOP_ROOT_LOGGER=INFO,console -# Default log4j setting for daemons spawned explicitly by +# Default log4j setting for daemons spawned explicitly by # --daemon option of hadoop, hdfs, mapred and yarn command. # Java property: hadoop.root.logger # export HADOOP_DAEMON_ROOT_LOGGER=INFO,RFA diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_tools.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_tools.bats new file mode 100644 index 00000000000..df0ff01e07a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_tools.bats @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +toolsetup () { + HADOOP_LIBEXEC_DIR="${TMP}/libexec" + mkdir -p "${HADOOP_LIBEXEC_DIR}/tools" +} + +@test "hadoop_classpath_tools (load)" { + toolsetup + echo "unittest=libexec" > "${HADOOP_LIBEXEC_DIR}/tools/test.sh" + hadoop_add_to_classpath_tools test + [ -n "${unittest}" ] +} + + +@test "hadoop_classpath_tools (not exist)" { + toolsetup + hadoop_add_to_classpath_tools test + [ -z "${unittest}" ] +} + +@test "hadoop_classpath_tools (function)" { + toolsetup + { + echo "function hadoop_classpath_tools_test {" + echo " unittest=libexec" + echo " }" + } > "${HADOOP_LIBEXEC_DIR}/tools/test.sh" + hadoop_add_to_classpath_tools test + declare -f + [ -n "${unittest}" ] +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_toolspath.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_toolspath.bats deleted file mode 100644 index a8a9bca8f7a..00000000000 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_to_classpath_toolspath.bats +++ /dev/null @@ -1,74 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load hadoop-functions_test_helper - -freetheclasses () { - local j - - for j in HADOOP_TOOLS_PATH \ - CLASSPATH; do - unset ${j} - done -} - -createdirs () { - local j - - for j in new old foo bar baz; do - mkdir -p "${TMP}/${j}" - done -} - -@test "hadoop_add_to_classpath_toolspath (nothing)" { - freetheclasses - hadoop_add_to_classpath_toolspath - [ -z "${CLASSPATH}" ] -} - -@test "hadoop_add_to_classpath_toolspath (none)" { - freetheclasses - CLASSPATH=test - hadoop_add_to_classpath_toolspath - [ "${CLASSPATH}" = "test" ] -} - -@test "hadoop_add_to_classpath_toolspath (only)" { - freetheclasses - createdirs - HADOOP_TOOLS_PATH="${TMP}/new" - hadoop_add_to_classpath_toolspath - [ "${CLASSPATH}" = "${TMP}/new" ] -} - -@test "hadoop_add_to_classpath_toolspath (1+1)" { - freetheclasses - createdirs - CLASSPATH=${TMP}/foo - HADOOP_TOOLS_PATH=${TMP}/foo - hadoop_add_to_classpath_toolspath - echo ">${CLASSPATH}<" - [ ${CLASSPATH} = "${TMP}/foo" ] -} - -@test "hadoop_add_to_classpath_toolspath (3+2)" { - freetheclasses - createdirs - CLASSPATH=${TMP}/foo:${TMP}/bar:${TMP}/baz - HADOOP_TOOLS_PATH=${TMP}/new:${TMP}/old - hadoop_add_to_classpath_toolspath - echo ">${CLASSPATH}<" - [ ${CLASSPATH} = "${TMP}/foo:${TMP}/bar:${TMP}/baz:${TMP}/new:${TMP}/old" ] -} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats index 6a5e3f270c7..ae20248dadf 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_basic_init.bats @@ -38,7 +38,8 @@ basicinitsetup () { dirvars="HADOOP_COMMON_HOME \ HADOOP_MAPRED_HOME \ HADOOP_HDFS_HOME \ - HADOOP_YARN_HOME" + HADOOP_YARN_HOME \ + HADOOP_TOOLS_HOME" for j in ${testvars}; do unset ${j} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats index 0fd5d214121..9114c707336 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_bootstrap.bats @@ -15,13 +15,13 @@ load hadoop-functions_test_helper -@test "hadoop_deprecate_envvar (no libexec)" { +@test "hadoop_bootstrap (no libexec)" { unset HADOOP_LIBEXEC_DIR run hadoop_bootstrap [ "${status}" -eq 1 ] } -@test "hadoop_deprecate_envvar (libexec)" { +@test "hadoop_bootstrap (libexec)" { unset HADOOP_PREFIX unset HADOOP_COMMON_DIR unset HADOOP_COMMON_LIB_JARS_DIR @@ -31,7 +31,9 @@ load hadoop-functions_test_helper unset YARN_LIB_JARS_DIR unset MAPRED_DIR unset MAPRED_LIB_JARS_DIR - unset TOOL_PATH + unset HADOOP_TOOLS_HOME + unset HADOOP_TOOLS_DIR + unset HADOOP_TOOLS_LIB_JARS_DIR unset HADOOP_OS_TYPE hadoop_bootstrap @@ -46,6 +48,9 @@ load hadoop-functions_test_helper [ -n ${YARN_LIB_JARS_DIR} ] [ -n ${MAPRED_DIR} ] [ -n ${MAPRED_LIB_JARS_DIR} ] - [ -n ${TOOL_PATH} ] [ -n ${HADOOP_OS_TYPE} ] -} + [ -n ${HADOOP_TOOLS_PATH} ] + [ -n ${HADOOP_TOOLS_HOME} ] + [ -n ${HADOOP_TOOLS_DIR} ] + [ -n ${HADOOP_TOOLS_LIB_JARS_DIR} ] +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_entry_tests.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_entry_tests.bats new file mode 100644 index 00000000000..20be3e546c3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_entry_tests.bats @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load hadoop-functions_test_helper + +@test "hadoop_add_entry (positive 1)" { + hadoop_add_entry testvar foo + echo ">${testvar}<" + [ "${testvar}" = " foo " ] +} + +@test "hadoop_add_entry (negative)" { + hadoop_add_entry testvar foo + hadoop_add_entry testvar foo + echo ">${testvar}<" + [ "${testvar}" = " foo " ] +} + +@test "hadoop_add_entry (positive 2)" { + hadoop_add_entry testvar foo + hadoop_add_entry testvar foo + hadoop_add_entry testvar bar + echo ">${testvar}<" + [ "${testvar}" = " foo bar " ] +} + +@test "hadoop_add_entry (positive 3)" { + hadoop_add_entry testvar foo + hadoop_add_entry testvar foo + hadoop_add_entry testvar bar + hadoop_add_entry testvar bar + hadoop_add_entry testvar baz + hadoop_add_entry testvar baz + + echo ">${testvar}<" + [ "${testvar}" = " foo bar baz " ] +} diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_shellprofile.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_shellprofile.bats index d6e0a2563e0..305067b9c7c 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_shellprofile.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_shellprofile.bats @@ -66,6 +66,13 @@ _test_hadoop_finalize () { [ -z "${unittest}" ] } +@test "hadoop_import_shellprofiles (H_O_T)" { + HADOOP_OPTIONAL_TOOLS=1,2 + shellprofilesetup + hadoop_import_shellprofiles + [ "${HADOOP_TOOLS_OPTIONS}" == " 1 2 " ] +} + @test "hadoop_add_profile+hadoop_shellprofiles_init" { hadoop_add_profile test hadoop_shellprofiles_init diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 42e74cbe7c4..8323bc64ace 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -103,6 +103,24 @@ + + toolshooks + prepare-package + + exec + + + ${shell-executable} + ${basedir} + false + + ${basedir}/../dev-support/bin/dist-tools-hooks-maker + ${project.version} + ${project.build.directory} + ${basedir}/../hadoop-tools + + + tar package diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index a6644d1af3e..bcd04d16085 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -141,7 +141,9 @@ case ${COMMAND} in echo "HDFS_DIR='${HDFS_DIR}'" echo "HDFS_LIB_JARS_DIR='${HDFS_LIB_JARS_DIR}'" echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" - echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'" + echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'" + echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'" + echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; erasurecode) @@ -165,7 +167,6 @@ case ${COMMAND} in ;; haadmin) CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin - hadoop_add_to_classpath_toolspath hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 66a19aee31c..fab5b87f2e0 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -69,13 +69,13 @@ case ${COMMAND} in ;; archive) CLASS=org.apache.hadoop.tools.HadoopArchives - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-archives hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; archive-logs) CLASS=org.apache.hadoop.tools.HadoopArchiveLogs - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-archive-logs hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; @@ -84,7 +84,7 @@ case ${COMMAND} in ;; distcp) CLASS=org.apache.hadoop.tools.DistCp - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-distcp hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" ;; @@ -94,7 +94,9 @@ case ${COMMAND} in echo "MAPRED_DIR='${MAPRED_DIR}'" echo "MAPRED_LIB_JARS_DIR='${MAPRED_LIB_JARS_DIR}'" echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" - echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'" + echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'" + echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'" + echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; historyserver) diff --git a/hadoop-tools/hadoop-archive-logs/pom.xml b/hadoop-tools/hadoop-archive-logs/pom.xml index 7e7da77efa3..f3fc9880b18 100644 --- a/hadoop-tools/hadoop-archive-logs/pom.xml +++ b/hadoop-tools/hadoop-archive-logs/pom.xml @@ -172,6 +172,23 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + org.codehaus.mojo findbugs-maven-plugin diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml index f04d7fc252c..cc98e632206 100644 --- a/hadoop-tools/hadoop-archives/pom.xml +++ b/hadoop-tools/hadoop-archives/pom.xml @@ -128,6 +128,23 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml index eb87d77b7cc..f4228460eae 100644 --- a/hadoop-tools/hadoop-aws/pom.xml +++ b/hadoop-tools/hadoop-aws/pom.xml @@ -90,6 +90,23 @@ 3600 + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 5e6eb95a650..af3541fafb1 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -19,6 +19,9 @@ JAR file, `hadoop-aws.jar` also declares a transitive dependency on all external artifacts which are needed for this support —enabling downstream applications to easily use this support. +To make it part of Apache Hadoop's default classpath, simply make sure that +HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aws' in the list. + Features 1. The "classic" `s3:` filesystem for storing objects in Amazon S3 Storage @@ -30,7 +33,7 @@ higher performance. The specifics of using these filesystems are documented below. -## Warning: Object Stores are not filesystems. +## Warning #1: Object Stores are not filesystems. Amazon S3 is an example of "an object store". In order to achieve scalability and especially high availability, S3 has —as many other cloud object stores have diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index e9b3af775f2..7d3aa5fff7c 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -82,6 +82,24 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + + diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/index.md b/hadoop-tools/hadoop-azure/src/site/markdown/index.md index a4a761521d5..43c551c4d5b 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/index.md @@ -34,6 +34,9 @@ The built jar file, named hadoop-azure.jar, also declares transitive dependencie on the additional artifacts it requires, notably the [Azure Storage SDK for Java](https://github.com/Azure/azure-storage-java). +To make it part of Apache Hadoop's default classpath, simply make sure that +HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-azure' in the list. + ## Features * Read and write data stored in an Azure Blob Storage account. diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml index 946952189b6..40445cb9ab0 100644 --- a/hadoop-tools/hadoop-datajoin/pom.xml +++ b/hadoop-tools/hadoop-datajoin/pom.xml @@ -132,6 +132,22 @@ org.apache.maven.plugins maven-jar-plugin + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + ${project.basedir}/../../hadoop-dist/target/hadoop-tools-deps/${project.artifactId}.txt + + + + diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 7099cec9b2c..b64a85839c3 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -163,6 +163,17 @@ ${project.build.directory}/lib + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml index 5f758758f10..f8851efae09 100644 --- a/hadoop-tools/hadoop-extras/pom.xml +++ b/hadoop-tools/hadoop-extras/pom.xml @@ -137,6 +137,23 @@ org.apache.maven.plugins maven-jar-plugin + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml index db84f8fa371..33b5488f072 100644 --- a/hadoop-tools/hadoop-gridmix/pom.xml +++ b/hadoop-tools/hadoop-gridmix/pom.xml @@ -163,6 +163,23 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + diff --git a/hadoop-tools/hadoop-kafka/pom.xml b/hadoop-tools/hadoop-kafka/pom.xml index c0667ee3391..f000682c5c3 100644 --- a/hadoop-tools/hadoop-kafka/pom.xml +++ b/hadoop-tools/hadoop-kafka/pom.xml @@ -88,6 +88,23 @@ 3600 + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml index 542a52367f8..20a52fa2704 100644 --- a/hadoop-tools/hadoop-openstack/pom.xml +++ b/hadoop-tools/hadoop-openstack/pom.xml @@ -85,6 +85,23 @@ false + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt + + + + diff --git a/hadoop-tools/hadoop-openstack/src/site/markdown/index.md b/hadoop-tools/hadoop-openstack/src/site/markdown/index.md index 0eeb2747278..1815f60c613 100644 --- a/hadoop-tools/hadoop-openstack/src/site/markdown/index.md +++ b/hadoop-tools/hadoop-openstack/src/site/markdown/index.md @@ -54,6 +54,9 @@ Introduction This module enables Apache Hadoop applications -including MapReduce jobs, read and write data to and from instances of the [OpenStack Swift object store](http://www.openstack.org/software/openstack-storage/). +To make it part of Apache Hadoop's default classpath, simply make sure that +HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-openstack' in the list. + Features -------- @@ -440,7 +443,9 @@ If the host is declared, the proxy port must be set to a valid integer value. The `hadoop-openstack` JAR -or any dependencies- may not be on your classpath. -If it is a remote MapReduce job that is failing, make sure that the JAR is installed on the servers in the cluster -or that the job submission process uploads the JAR file to the distributed cache. +Make sure that the: +* JAR is installed on the servers in the cluster. +* 'hadoop-openstack' is on the HADOOP_OPTIONAL_TOOLS entry in hadoop-env.sh or that the job submission process uploads the JAR file to the distributed cache. #### Failure to Authenticate diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index 6828b1c1f1b..d11cd170287 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -132,6 +132,23 @@ org.apache.maven.plugins maven-jar-plugin + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml index 0539a0885e0..6d199bd492e 100644 --- a/hadoop-tools/hadoop-sls/pom.xml +++ b/hadoop-tools/hadoop-sls/pom.xml @@ -177,6 +177,23 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh index c5b6d844bca..f9bfaef2db0 100644 --- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh +++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh @@ -55,7 +55,7 @@ function parse_args() function calculate_classpath() { - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-rumen } function run_sls_generator() diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh index 29b0de743a5..30fd60a4440 100644 --- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh +++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh @@ -70,7 +70,7 @@ function parse_args() function calculate_classpath { - hadoop_add_to_classpath_toolspath + hadoop_add_to_classpath_tools hadoop-sls hadoop_debug "Injecting ${HADOOP_PREFIX}/share/hadoop/tools/sls/html into CLASSPATH" hadoop_add_classpath "${HADOOP_PREFIX}/share/hadoop/tools/sls/html" } diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 5ea10f63182..b16e1326954 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -174,6 +174,23 @@ + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + + ${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index da4ce4042f4..cb2364b3957 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -103,7 +103,9 @@ case "${COMMAND}" in echo "YARN_DIR='${YARN_DIR}'" echo "YARN_LIB_JARS_DIR='${YARN_LIB_JARS_DIR}'" echo "HADOOP_CONF_DIR='${HADOOP_CONF_DIR}'" - echo "HADOOP_TOOLS_PATH='${HADOOP_TOOLS_PATH}'" + echo "HADOOP_TOOLS_HOME='${HADOOP_TOOLS_HOME}'" + echo "HADOOP_TOOLS_DIR='${HADOOP_TOOLS_DIR}'" + echo "HADOOP_TOOLS_LIB_JARS_DIR='${HADOOP_TOOLS_LIB_JARS_DIR}'" exit 0 ;; jar)