diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index 6d05fe139fe..9eb9bcffb1b 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -63,6 +63,7 @@ RUN apt-get -q update && apt-get -q install -y \ pkg-config \ python \ python2.7 \ + pylint \ python-pip \ python-pkg-resources \ python-setuptools \ @@ -71,7 +72,8 @@ RUN apt-get -q update && apt-get -q install -y \ software-properties-common \ snappy \ sudo \ - zlib1g-dev + zlib1g-dev \ + valgrind ####### # OpenJDK 8 diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml index 64fafc8dd8a..069bf1a7e38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml @@ -33,6 +33,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> false hdfs false + false + + + @@ -143,11 +147,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - + + + @@ -202,13 +208,16 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> compile cmake-compile - ${basedir}/src - - ${project.build.directory}/native/javah - ${sun.arch.data.model} - ${require.fuse} - - ${project.build.directory} + + + + + + + + + + @@ -227,6 +236,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt index 128527abe24..6338d5d9b06 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt @@ -19,14 +19,26 @@ project (libhdfspp) enable_testing() +include (CTest) find_package(Doxygen) find_package(OpenSSL REQUIRED) find_package(Protobuf REQUIRED) find_package(Threads) +find_program(MEMORYCHECK_COMMAND valgrind HINTS ${VALGRIND_DIR} ) +set(MEMORYCHECK_COMMAND_OPTIONS "--trace-children=yes --leak-check=full --error-exitcode=1") +message(STATUS "valgrind location: ${MEMORYCHECK_COMMAND}") + +if (REQUIRE_VALGRIND AND MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" ) + message(FATAL_ERROR "valgrind was required but not found. " + "The path can be included via a -DVALGRIND_DIR=... flag passed to CMake.") +endif (REQUIRE_VALGRIND AND MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" ) + + add_definitions(-DASIO_STANDALONE -DASIO_CPP11_DATE_TIME) + if(UNIX) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -pedantic -std=c++11 -g -fPIC -fno-strict-aliasing") endif() diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt index 60e71d61886..6490484c8bf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/CMakeLists.txt @@ -43,34 +43,47 @@ protobuf_generate_cpp(PROTO_TEST_SRCS PROTO_TEST_HDRS ${PROTO_HADOOP_TEST_DIR}/test_rpc_service.proto ) +# Shamelessly stolen from +# http://stackoverflow.com/questions/9303711/how-do-i-make-ctest-run-a-program-with-valgrind-without-dart +function(add_memcheck_test name binary) + add_test(${name} ${binary} ${ARGN}) + + if(NOT MEMORYCHECK_COMMAND MATCHES "MEMORYCHECK_COMMAND-NOTFOUND" AND NOT SKIP_VALGRIND) + set(memcheck_command "${MEMORYCHECK_COMMAND} ${MEMORYCHECK_COMMAND_OPTIONS}") + separate_arguments(memcheck_command) + add_test(memcheck_${name} ${memcheck_command} ./${binary} ${ARGN}) + endif() +endfunction(add_memcheck_test) + + add_executable(remote_block_reader_test remote_block_reader_test.cc $) target_link_libraries(remote_block_reader_test reader proto common connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) -add_test(remote_block_reader remote_block_reader_test) +add_memcheck_test(remote_block_reader remote_block_reader_test) add_executable(sasl_digest_md5_test sasl_digest_md5_test.cc) target_link_libraries(sasl_digest_md5_test common ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) -add_test(sasl_digest_md5 sasl_digest_md5_test) +add_memcheck_test(sasl_digest_md5 sasl_digest_md5_test) add_executable(retry_policy_test retry_policy_test.cc) target_link_libraries(retry_policy_test common gmock_main ${CMAKE_THREAD_LIBS_INIT}) -add_test(retry_policy retry_policy_test) +add_memcheck_test(retry_policy retry_policy_test) include_directories(${CMAKE_CURRENT_BINARY_DIR}) add_executable(rpc_engine_test rpc_engine_test.cc ${PROTO_TEST_SRCS} ${PROTO_TEST_HDRS} $) target_link_libraries(rpc_engine_test rpc proto common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) -add_test(rpc_engine rpc_engine_test) +add_memcheck_test(rpc_engine rpc_engine_test) add_executable(bad_datanode_test bad_datanode_test.cc) target_link_libraries(bad_datanode_test rpc reader proto fs bindings_c rpc proto common reader connection ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} gmock_main ${CMAKE_THREAD_LIBS_INIT}) -add_test(bad_datanode bad_datanode_test) +add_memcheck_test(bad_datanode bad_datanode_test) add_executable(node_exclusion_test node_exclusion_test.cc) target_link_libraries(node_exclusion_test fs gmock_main common ${PROTOBUF_LIBRARIES} ${OPENSSL_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) -add_test(node_exclusion node_exclusion_test) +add_memcheck_test(node_exclusion node_exclusion_test) add_executable(configuration_test configuration_test.cc) target_link_libraries(configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT}) -add_test(configuration configuration_test) +add_memcheck_test(configuration configuration_test) add_executable(hdfs_configuration_test hdfs_configuration_test.cc) target_link_libraries(hdfs_configuration_test common gmock_main ${CMAKE_THREAD_LIBS_INIT}) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc index 0f69195fc6d..771d85f4742 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc @@ -242,5 +242,9 @@ int main(int argc, char *argv[]) { // The following line must be executed to initialize Google Mock // (and Google Test) before running the tests. ::testing::InitGoogleMock(&argc, argv); - return RUN_ALL_TESTS(); + int exit_code = RUN_ALL_TESTS(); + + // Clean up static data and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + return exit_code; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc index 6f3122e7816..7d926ba710f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc @@ -410,5 +410,9 @@ int main(int argc, char *argv[]) { // The following line must be executed to initialize Google Mock // (and Google Test) before running the tests. ::testing::InitGoogleMock(&argc, argv); - return RUN_ALL_TESTS(); + int exit_code = RUN_ALL_TESTS(); + + // Clean up static data and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + return exit_code; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc index 063bceeb60e..71e39780682 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc @@ -395,5 +395,9 @@ int main(int argc, char *argv[]) { // The following line must be executed to initialize Google Mock // (and Google Test) before running the tests. ::testing::InitGoogleMock(&argc, argv); - return RUN_ALL_TESTS(); + int exit_code = RUN_ALL_TESTS(); + + // Clean up static data and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + return exit_code; }