From 2a303865391657648281270114bccc32fd4c051a Mon Sep 17 00:00:00 2001 From: Colin Patrick Mccabe Date: Fri, 15 Jan 2016 14:05:21 -0800 Subject: [PATCH] HADOOP-12662. The build should fail if a -Dbundle option fails (Kai Zheng via cmccabe) --- BUILDING.txt | 9 +- .../hadoop-common/CHANGES.txt | 3 + .../hadoop-common/src/CMakeLists.txt | 2 +- hadoop-project-dist/pom.xml | 88 ++++++++++++++----- 4 files changed, 76 insertions(+), 26 deletions(-) diff --git a/BUILDING.txt b/BUILDING.txt index fc68a0bdbbd..6a098fa6d05 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -149,7 +149,8 @@ Maven build goals: installed snappy using a package manager. * Use -Dbundle.snappy to copy the contents of the snappy.lib directory into the final tar file. This option requires that -Dsnappy.lib is also given, - and it ignores the -Dsnappy.prefix option. + and it ignores the -Dsnappy.prefix option. If -Dsnappy.lib isn't given, the + bundling and building will fail. OpenSSL build options: @@ -170,7 +171,8 @@ Maven build goals: installed openssl using a package manager. * Use -Dbundle.openssl to copy the contents of the openssl.lib directory into the final tar file. This option requires that -Dopenssl.lib is also given, - and it ignores the -Dopenssl.prefix option. + and it ignores the -Dopenssl.prefix option. If -Dopenssl.lib isn't given, the + bundling and building will fail. Tests options: @@ -205,7 +207,8 @@ Maven build goals: files. * Use -Dbundle.isal to copy the contents of the isal.lib directory into the final tar file. This option requires that -Disal.lib is also given, - and it ignores the -Disal.prefix option. + and it ignores the -Disal.prefix option. If -Disal.lib isn't given, the + bundling and building will fail. ---------------------------------------------------------------------------------- Building components separately diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f3e6ff892a7..4b2c9e726ae 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -671,6 +671,9 @@ Release 2.9.0 - UNRELEASED HADOOP-12713. Disable spurious checkstyle checks. (wang) + HADOOP-12662. The build should fail if a -Dbundle option fails (Kai Zheng + via cmccabe) + BUG FIXES HADOOP-12655. TestHttpServer.testBindAddress bind port range is wider diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index eaa8d88ccc0..51fb0fe49c3 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -99,7 +99,7 @@ hadoop_set_find_shared_library_version("2") find_library(ISAL_LIBRARY NAMES isal PATHS ${CUSTOM_ISAL_PREFIX} ${CUSTOM_ISAL_PREFIX}/lib - ${CUSTOM_ISAL_PREFIX}/lib64 ${CUSTOM_ISAL_LIB /usr/lib}) + ${CUSTOM_ISAL_PREFIX}/lib64 ${CUSTOM_ISAL_LIB} /usr/lib) set(CMAKE_FIND_LIBRARY_SUFFIXES ${STORED_CMAKE_FIND_LIBRARY_SUFFIXES}) if (ISAL_LIBRARY) GET_FILENAME_COMPONENT(HADOOP_ISAL_LIBRARY ${ISAL_LIBRARY} NAME) diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index 81773d66cf8..8e11bebe8c5 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -41,7 +41,7 @@ UNDEF false false - true + false false false @@ -324,52 +324,96 @@ + + # Bundle a native library if requested. Exit 1 in case error happens. + # Usage: bundle_native_lib bundleOption libOption libPattern libDir + function bundle_native_lib() { + bundleOption="$1" + libOption="$2" + libPattern="$3" + libDir="$4" + + echo "Checking to bundle with:" + echo "bundleOption=${bundleOption}, libOption=${libOption}, libDir=${libDir}, pattern=${libPattern}" + + if [[ "${bundleOption}" != "true" ]]; then + return + fi + + if [[ -z "${libDir}" ]] || [[ ! -d ${libDir} ]]; then + echo "The required option $libOption isn't given or invalid. Bundling the lib failed" + exit 1 + fi + + cd "$libDir" + $$TAR *$libPattern* | (cd $${TARGET_DIR}/; $$UNTAR) + if [[ $? -ne 0 ]]; then + echo "Bundling library with ${libOption} failed " + exit 1 + fi + } + BUILD_DIR="${project.build.directory}" TAR='tar cf -' UNTAR='tar xfBp -' LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib" - if [ -d $${LIB_DIR} ] ; then + + set -o pipefail + + if [[ -d $${LIB_DIR} ]]; then TARGET_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/lib/native" mkdir -p $${TARGET_DIR} cd $${LIB_DIR} $$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR) - if [ "X${bundle.snappy}" = "Xtrue" ] ; then - cd "${snappy.lib}" - $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR) - fi - if [ "X${bundle.openssl}" = "Xtrue" ] ; then - cd "${openssl.lib}" - $$TAR *crypto* | (cd $${TARGET_DIR}/; $$UNTAR) - fi - if [ "X${bundle.isal}" = "Xtrue" ] ; then - if [ "X${isal.lib}" != "X" ]; then - cd "${isal.lib}" - $$TAR *isa* | (cd $${TARGET_DIR}/; $$UNTAR) - else - echo "The required option isal.lib isn't given, bundling ISA-L skipped" - fi + if [[ $? -ne 0 ]]; then + echo "Bundling lib files failed" + exit 1 fi + + echo bundle_native_lib "${bundle.snappy}" "snappy.lib" "snappy" "${snappy.lib}" + bundle_native_lib "${bundle.snappy}" "snappy.lib" "snappy" "${snappy.lib}" + + echo bundle_native_lib "${bundle.openssl}" "openssl.lib" "crypto" "${openssl.lib}" + bundle_native_lib "${bundle.openssl}" "openssl.lib" "crypto" "${openssl.lib}" + + echo bundle_native_lib "${bundle.isal}" "isal.lib" "isa" "${isal.lib}" + bundle_native_lib "${bundle.isal}" "isal.lib" "isa" "${isal.lib}" fi + BIN_DIR="${BUILD_DIR}/bin" - if [ -d $${BIN_DIR} ] ; then + if [[ -d $${BIN_DIR} ]] ; then TARGET_BIN_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/bin" mkdir -p $${TARGET_BIN_DIR} cd $${BIN_DIR} $$TAR * | (cd $${TARGET_BIN_DIR}/; $$UNTAR) - if [ "${bundle.snappy.in.bin}" = "true" ] ; then - if [ "${bundle.snappy}" = "true" ] ; then + if [[ $? -ne 0 ]]; then + echo "Bundling bin files failed" + exit 1 + fi + + if [[ "${bundle.snappy.in.bin}" == "true" ]]; then + if [[ "${bundle.snappy}" == "true" ]]; then cd "${snappy.lib}" $$TAR *snappy* | (cd $${TARGET_BIN_DIR}/; $$UNTAR) + if [[ $? -ne 0 ]]; then + echo "Bundling snappy bin files failed" + exit 1 + fi fi fi - if [ "${bundle.openssl.in.bin}" = "true" ] ; then - if [ "${bundle.openssl}" = "true" ] ; then + if [[ "${bundle.openssl.in.bin}" == "true" ]]; then + if [[ "${bundle.openssl}" == "true" ]]; then cd "${openssl.lib}" $$TAR *crypto* | (cd $${TARGET_BIN_DIR}/; $$UNTAR) + if [[ $? -ne 0 ]]; then + echo "Bundling openssl bin files failed" + exit 1 + fi fi fi fi +