From 7d8452040d1ade90a4eb7328eb79bc9a0ca7e205 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 5 Jan 2016 15:51:06 -0500 Subject: [PATCH] HDFS-9325. libhdfs++ Allow the location of hadoop source tree resources to be passed to CMake during a build. Contributed by Bob Hansen. --- .../hadoop-hdfs-native-client/pom.xml | 4 +- .../src/main/native/libhdfspp/CMakeLists.txt | 93 ++++++++++++++++--- .../libhdfspp/lib/bindings/c/CMakeLists.txt | 5 +- .../libhdfspp/lib/common/CMakeLists.txt | 8 +- .../libhdfspp/lib/connection/CMakeLists.txt | 5 +- .../native/libhdfspp/lib/fs/CMakeLists.txt | 5 +- .../native/libhdfspp/lib/proto/CMakeLists.txt | 6 +- .../libhdfspp/lib/reader/CMakeLists.txt | 5 +- .../native/libhdfspp/lib/rpc/CMakeLists.txt | 5 +- .../native/libhdfspp/tests/CMakeLists.txt | 18 ++-- .../third_party/gmock-1.7.0/CMakeLists.txt | 3 +- 11 files changed, 121 insertions(+), 36 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index 7156b041127..2582d34e09b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -147,7 +147,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - + - + diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 67442591633..d8e8c59fa00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -18,6 +18,8 @@ project (libhdfspp) +cmake_minimum_required(VERSION 2.8) + enable_testing() include (CTest) @@ -59,10 +61,53 @@ add_custom_target(doc ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/doc/Doxy COMMENT "Generating API documentation with Doxygen" VERBATIM) endif(DOXYGEN_FOUND) + +# Copy files from the hadoop tree into the output/extern directory if +# they've changed +function (copy_on_demand input_src_glob input_dest_dir) + get_filename_component(src_glob ${input_src_glob} REALPATH) + get_filename_component(dest_dir ${input_dest_dir} REALPATH) + get_filename_component(src_dir ${src_glob} DIRECTORY) + message(STATUS "Syncing ${src_glob} to ${dest_dir}") + + file(GLOB_RECURSE src_files ${src_glob}) + foreach(src_path ${src_files}) + file(RELATIVE_PATH relative_src ${src_dir} ${src_path}) + set(dest_path "${dest_dir}/${relative_src}") + add_custom_command(TARGET copy_hadoop_files + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${src_path}" "${dest_path}" + ) + endforeach() +endfunction() + +# If we're building in the hadoop tree, pull the Hadoop files that +# libhdfspp depends on. This allows us to ensure that +# the distribution will have a consistent set of headers and +# .proto files +if(HADOOP_BUILD) + set(HADOOP_IMPORT_DIR ${PROJECT_BINARY_DIR}/extern) + get_filename_component(HADOOP_IMPORT_DIR ${HADOOP_IMPORT_DIR} REALPATH) + + add_custom_target(copy_hadoop_files ALL) + + # Gather the Hadoop files and resources that libhdfs++ needs to build + copy_on_demand(../libhdfs/include/*.h* ${HADOOP_IMPORT_DIR}/include) + copy_on_demand(${CMAKE_CURRENT_LIST_DIR}/../../../../../hadoop-hdfs-client/src/main/proto/*.proto ${HADOOP_IMPORT_DIR}/proto/hdfs) + copy_on_demand(${CMAKE_CURRENT_LIST_DIR}/../../../../../../hadoop-common-project/hadoop-common/src/main/proto/*.proto ${HADOOP_IMPORT_DIR}/proto/hadoop) + copy_on_demand(${CMAKE_CURRENT_LIST_DIR}/../../../../../../hadoop-common-project/hadoop-common/src/test/proto/*.proto ${HADOOP_IMPORT_DIR}/proto/hadoop_test) +else(HADOOP_BUILD) + set(HADOOP_IMPORT_DIR ${CMAKE_CURRENT_LIST_DIR}/extern) +endif(HADOOP_BUILD) + +# Paths to find the imported files +set(PROTO_HDFS_DIR ${HADOOP_IMPORT_DIR}/proto/hdfs) +set(PROTO_HADOOP_DIR ${HADOOP_IMPORT_DIR}/proto/hadoop) +set(PROTO_HADOOP_TEST_DIR ${HADOOP_IMPORT_DIR}/proto/hadoop_test) + include_directories( include lib - ../libhdfs/include + ${HADOOP_IMPORT_DIR}/include ) include_directories( SYSTEM @@ -75,9 +120,6 @@ include_directories( SYSTEM ${OPENSSL_INCLUDE_DIR} ) -set(PROTO_HDFS_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../../../hadoop-hdfs-client/src/main/proto) -set(PROTO_HADOOP_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../../../../hadoop-common-project/hadoop-common/src/main/proto) -set(PROTO_HADOOP_TEST_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../../../../hadoop-common-project/hadoop-common/src/test/proto) add_subdirectory(third_party/gmock-1.7.0) add_subdirectory(lib) @@ -88,20 +130,43 @@ add_subdirectory(tests) set(EMPTY_FILE_CC ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/empty.cc) file(WRITE ${EMPTY_FILE_CC} "") -hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC}) +# Build the output libraries if(NEED_LINK_DL) set(LIB_DL dl) endif() -SET(LIBHDFSPP_SUBLIBS bindings_c fs rpc reader proto common) -IF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") - # linking a shared library from static ones requires --whole-archive - SET(LIBHDFSPP_SUBLIBS -Wl,--whole-archive ${LIBHDFSPP_SUBLIBS} -Wl,--no-whole-archive) -ENDIF(${CMAKE_SYSTEM_NAME} MATCHES "Linux") - -hadoop_target_link_dual_libraries(hdfspp - ${LIBHDFSPP_SUBLIBS} +set(LIBHDFSPP_ALL_OBJECTS $ $ $ $ $ $ $) +if (HADOOP_BUILD) + hadoop_add_dual_library(hdfspp ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) + hadoop_target_link_dual_libraries(hdfspp ${LIB_DL} ${PROTOBUF_LIBRARY} ${OPENSSL_LIBRARIES} -) + ) +else (HADOOP_BUILD) + add_library(hdfspp_static STATIC ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) + target_link_libraries(hdfspp_static + ${LIB_DL} + ${PROTOBUF_LIBRARY} + ${OPENSSL_LIBRARIES} + ) + add_library(hdfspp SHARED ${EMPTY_FILE_CC} ${LIBHDFSPP_ALL_OBJECTS}) + target_link_libraries(hdfspp_static + ${LIB_DL} + ${PROTOBUF_LIBRARY} + ${OPENSSL_LIBRARIES} + ) +endif (HADOOP_BUILD) +set(LIBHDFSPP_VERSION "0.1.0") +set_target_properties(hdfspp PROPERTIES + SOVERSION ${LIBHDFSPP_VERSION}) + +# Set up make install targets +# Can be installed to a particular location via "make DESTDIR=... install" +file(GLOB_RECURSE LIBHDFSPP_HEADER_FILES "${CMAKE_CURRENT_LIST_DIR}/include/*.h*") +file(GLOB_RECURSE LIBHDFS_HEADER_FILES "${HADOOP_IMPORT_DIR}/include/*.h*") +install(FILES ${LIBHDFSPP_HEADER_FILES} DESTINATION /include/libhdfspp) +install(FILES ${LIBHDFS_HEADER_FILES} DESTINATION /include/libhdfs) + +install(TARGETS hdfspp_static ARCHIVE DESTINATION /lib) +install(TARGETS hdfspp LIBRARY DESTINATION /lib) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt index 664518ac684..98b081ff335 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt @@ -16,5 +16,6 @@ # under the License. -add_library(bindings_c hdfs.cc) -add_dependencies(bindings_c fs rpc reader proto common fs rpc reader proto common) +add_library(bindings_c_obj OBJECT hdfs.cc) +add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common) +add_library(bindings_c $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt index 488870ad85d..c02e3db8201 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/CMakeLists.txt @@ -15,4 +15,10 @@ # specific language governing permissions and limitations # under the License. -add_library(common base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc util.cc retry_policy.cc) +if(NEED_LINK_DL) + set(LIB_DL dl) +endif() + +add_library(common_obj OBJECT base64.cc status.cc sasl_digest_md5.cc hdfs_public_api.cc options.cc configuration.cc configuration_loader.cc hdfs_configuration.cc util.cc retry_policy.cc) +add_library(common $) +target_link_libraries(common ${LIB_DL}) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/CMakeLists.txt index 1c12b2ee060..fda51a174d0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/CMakeLists.txt @@ -16,5 +16,6 @@ # limitations under the License. # -add_library(connection datanodeconnection.cc) -add_dependencies(connection proto) +add_library(connection_obj OBJECT datanodeconnection.cc) +add_dependencies(connection_obj proto) +add_library(connection $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt index d87b2bad727..7d750b2aae7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt @@ -16,5 +16,6 @@ # limitations under the License. # -add_library(fs filesystem.cc filehandle.cc bad_datanode_tracker.cc) -add_dependencies(fs proto) +add_library(fs_obj OBJECT filesystem.cc filehandle.cc bad_datanode_tracker.cc) +add_dependencies(fs_obj proto) +add_library(fs $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt index 63fa5a73fdf..a5b5c4cfcd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt @@ -79,4 +79,8 @@ gen_hrpc(HRPC_SRCS ${PROTO_HDFS_DIR}/ClientNamenodeProtocol.proto ) -add_library(proto ${PROTO_SRCS} ${PROTO_HDRS} ${HRPC_SRCS}) +add_library(proto_obj OBJECT ${PROTO_SRCS} ${PROTO_HDRS} ${HRPC_SRCS}) +if(HADOOP_BUILD) + add_dependencies(proto_obj copy_hadoop_files) +endif(HADOOP_BUILD) +add_library(proto $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/CMakeLists.txt index 0dcae29e88d..d4bb837236b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/CMakeLists.txt @@ -16,5 +16,6 @@ # limitations under the License. # -add_library(reader block_reader.cc datatransfer.cc) -add_dependencies(reader proto) +add_library(reader_obj OBJECT block_reader.cc datatransfer.cc) +add_dependencies(reader_obj proto) +add_library(reader $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt index 6ab54ef4c02..b82bcf619e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/CMakeLists.txt @@ -17,5 +17,6 @@ # include_directories(${OPENSSL_INCLUDE_DIRS}) -add_library(rpc rpc_connection.cc rpc_engine.cc) -add_dependencies(rpc proto) +add_library(rpc_obj OBJECT rpc_connection.cc rpc_engine.cc) +add_dependencies(rpc_obj proto) +add_library(rpc $) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 6490484c8bf..3dee80110fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -32,9 +32,8 @@ include_directories( ${LIBHDFS_SRC_DIR} ${OS_DIR} ) -add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc) - -add_library(test_common OBJECT mock_connection.cc) +add_library(test_common_obj OBJECT mock_connection.cc) +add_library(test_common $) set(PROTOBUF_IMPORT_DIRS ${PROTO_HADOOP_TEST_DIR}) @@ -56,8 +55,8 @@ function(add_memcheck_test name binary) endfunction(add_memcheck_test) -add_executable(remote_block_reader_test remote_block_reader_test.cc $) -target_link_libraries(remote_block_reader_test reader proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +add_executable(remote_block_reader_test remote_block_reader_test.cc) +target_link_libraries(remote_block_reader_test test_common reader proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(remote_block_reader remote_block_reader_test) add_executable(sasl_digest_md5_test sasl_digest_md5_test.cc) @@ -69,8 +68,8 @@ target_link_libraries(retry_policy_test common gmock_main ${CMAKE_THREAD_LIBS_IN add_memcheck_test(retry_policy retry_policy_test) include_directories(${CMAKE_CURRENT_BINARY_DIR}) -add_executable(rpc_engine_test rpc_engine_test.cc ${PROTO_TEST_SRCS} ${PROTO_TEST_HDRS} $) -target_link_libraries(rpc_engine_test rpc proto common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) +add_executable(rpc_engine_test rpc_engine_test.cc ${PROTO_TEST_SRCS} ${PROTO_TEST_HDRS}) +target_link_libraries(rpc_engine_test test_common rpc proto common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_memcheck_test(rpc_engine rpc_engine_test) add_executable(bad_datanode_test bad_datanode_test.cc) @@ -89,6 +88,11 @@ add_executable(hdfs_configuration_test hdfs_configuration_test.cc) target_link_libraries(hdfs_configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT}) add_test(hdfs_configuration hdfs_configuration_test) +#This test requires a great deal of Hadoop Java infrastructure to run. +if(HADOOP_BUILD) +add_library(hdfspp_test_shim_static STATIC hdfs_shim.c libhdfs_wrapper.c libhdfspp_wrapper.cc ${LIBHDFSPP_BINDING_C}/hdfs.cc) + build_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static expect.c test_libhdfs_threaded.c ${OS_DIR}/thread.c) link_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static fs reader rpc proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} native_mini_dfs ${JAVA_JVM_LIBRARY}) add_libhdfs_test(libhdfs_threaded hdfspp_test_shim_static) +endif(HADOOP_BUILD) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/gmock-1.7.0/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/gmock-1.7.0/CMakeLists.txt index 6c9c82e83ce..3ebde862b4d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/gmock-1.7.0/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/third_party/gmock-1.7.0/CMakeLists.txt @@ -17,4 +17,5 @@ # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-field-initializers -Wno-unused-const-variable") -add_library(gmock_main gmock-gtest-all.cc gmock_main.cc) +add_library(gmock_main_obj OBJECT gmock-gtest-all.cc gmock_main.cc) +add_library(gmock_main $)