From ed4466296862bfc5bd4409c2d1250ac387c776da Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Wed, 2 Feb 2022 11:12:17 +0530 Subject: [PATCH] HDFS-16445. Make HDFS count, mkdir, rm cross platform (#3945) --- .../libhdfspp/tests/tools/CMakeLists.txt | 9 ++ .../libhdfspp/tests/tools/hdfs-count-mock.cc | 63 ++++++++ .../libhdfspp/tests/tools/hdfs-count-mock.h | 68 +++++++++ .../libhdfspp/tests/tools/hdfs-mkdir-mock.cc | 83 +++++++++++ .../libhdfspp/tests/tools/hdfs-mkdir-mock.h | 70 +++++++++ .../libhdfspp/tests/tools/hdfs-rm-mock.cc | 60 ++++++++ .../libhdfspp/tests/tools/hdfs-rm-mock.h | 68 +++++++++ .../libhdfspp/tests/tools/hdfs-tool-tests.cc | 59 ++++++++ .../libhdfspp/tests/tools/hdfs-tool-tests.h | 94 ++++++++++++ .../native/libhdfspp/tools/CMakeLists.txt | 9 +- .../libhdfspp/tools/hdfs-count/CMakeLists.txt | 27 ++++ .../libhdfspp/tools/hdfs-count/hdfs-count.cc | 125 ++++++++++++++++ .../libhdfspp/tools/hdfs-count/hdfs-count.h | 94 ++++++++++++ .../native/libhdfspp/tools/hdfs-count/main.cc | 52 +++++++ .../libhdfspp/tools/hdfs-mkdir/CMakeLists.txt | 27 ++++ .../libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc | 140 ++++++++++++++++++ .../libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h | 105 +++++++++++++ .../native/libhdfspp/tools/hdfs-mkdir/main.cc | 52 +++++++ .../libhdfspp/tools/hdfs-rm/CMakeLists.txt | 27 ++++ .../native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc | 114 ++++++++++++++ .../native/libhdfspp/tools/hdfs-rm/hdfs-rm.h | 92 ++++++++++++ .../native/libhdfspp/tools/hdfs-rm/main.cc | 52 +++++++ .../main/native/libhdfspp/tools/hdfs_count.cc | 93 ------------ .../main/native/libhdfspp/tools/hdfs_mkdir.cc | 98 ------------ .../main/native/libhdfspp/tools/hdfs_rm.cc | 90 ----------- 25 files changed, 1484 insertions(+), 287 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h create mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc delete mode 100644 hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt index e18892840e1..56755adac3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt @@ -32,6 +32,9 @@ add_executable(hdfs_tool_tests hdfs-du-mock.cc hdfs-copy-to-local-mock.cc hdfs-move-to-local-mock.cc + hdfs-count-mock.cc + hdfs-mkdir-mock.cc + hdfs-rm-mock.cc main.cc) target_include_directories(hdfs_tool_tests PRIVATE ../tools @@ -48,6 +51,9 @@ target_include_directories(hdfs_tool_tests PRIVATE ../../tools/hdfs-chmod ../../tools/hdfs-copy-to-local ../../tools/hdfs-move-to-local + ../../tools/hdfs-count + ../../tools/hdfs-mkdir + ../../tools/hdfs-rm ../../tools/hdfs-cat) target_link_libraries(hdfs_tool_tests PRIVATE gmock_main @@ -63,5 +69,8 @@ target_link_libraries(hdfs_tool_tests PRIVATE hdfs_chmod_lib hdfs_copyToLocal_lib hdfs_moveToLocal_lib + hdfs_count_lib + hdfs_mkdir_lib + hdfs_rm_lib hdfs_cat_lib) add_test(hdfs_tool_tests hdfs_tool_tests) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc new file mode 100644 index 00000000000..649a7103607 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.cc @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-count-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +CountMock::~CountMock() = default; + +void CountMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(false, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassQOptAndPath) { + const auto arg1 = args[0]; + const auto arg2 = args[1]; + EXPECT_CALL(*this, HandlePath(true, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h new file mode 100644 index 00000000000..6f0e5c00cea --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-count-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_COUNT_MOCK +#define LIBHDFSPP_TOOLS_HDFS_COUNT_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-count.h" + +namespace hdfs::tools::test { +/** + * {@class CountMock} is an {@class Count} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class CountMock : public hdfs::tools::Count { +public: + /** + * {@inheritdoc} + */ + CountMock(const int argc, char **argv) : Count(argc, argv) {} + + // Abiding to the Rule of 5 + CountMock(const CountMock &) = delete; + CountMock(CountMock &&) = delete; + CountMock &operator=(const CountMock &) = delete; + CountMock &operator=(CountMock &&) = delete; + ~CountMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc new file mode 100644 index 00000000000..54ed0b0990e --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.cc @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "hdfs-mkdir-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +MkdirMock::~MkdirMock() = default; + +void MkdirMock::SetExpectations( + std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = + test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + const std::optional permissions = std::nullopt; + EXPECT_CALL(*this, HandlePath(false, permissions, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassPOptAndPath) { + const auto arg1 = args[1]; + const std::optional permissions = std::nullopt; + EXPECT_CALL(*this, HandlePath(true, permissions, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassMOptPermissionsAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[2]; + const auto permissions = std::optional(arg1); + EXPECT_CALL(*this, HandlePath(false, permissions, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassMPOptsPermissionsAndAPath) { + const auto arg1 = args[1]; + const auto arg2 = args[3]; + const auto permissions = std::optional(arg1); + EXPECT_CALL(*this, HandlePath(true, permissions, arg2)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h new file mode 100644 index 00000000000..e112a14cd20 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-mkdir-mock.h @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_MKDIR_MOCK +#define LIBHDFSPP_TOOLS_HDFS_MKDIR_MOCK + +#include +#include +#include +#include +#include + +#include + +#include "hdfs-mkdir.h" + +namespace hdfs::tools::test { +/** + * {@class MkdirMock} is an {@class Mkdir} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class MkdirMock : public hdfs::tools::Mkdir { +public: + /** + * {@inheritdoc} + */ + MkdirMock(const int argc, char **argv) : Mkdir(argc, argv) {} + + // Abiding to the Rule of 5 + MkdirMock(const MkdirMock &) = delete; + MkdirMock(MkdirMock &&) = delete; + MkdirMock &operator=(const MkdirMock &) = delete; + MkdirMock &operator=(MkdirMock &&) = delete; + ~MkdirMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, + (bool, const std::optional &, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc new file mode 100644 index 00000000000..29cf95fcced --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.cc @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include "hdfs-rm-mock.h" +#include "hdfs-tool-tests.h" + +namespace hdfs::tools::test { +RmMock::~RmMock() = default; + +void RmMock::SetExpectations(std::function()> test_case, + const std::vector &args) const { + // Get the pointer to the function that defines the test case + const auto test_case_func = test_case.target (*)()>(); + ASSERT_NE(test_case_func, nullptr); + + // Set the expected method calls and their corresponding arguments for each + // test case + if (*test_case_func == &CallHelp) { + EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true)); + return; + } + + if (*test_case_func == &PassAPath) { + const auto arg1 = args[0]; + EXPECT_CALL(*this, HandlePath(false, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } + + if (*test_case_func == &PassRecursivePath) { + const auto arg1 = args[1]; + EXPECT_CALL(*this, HandlePath(true, arg1)) + .Times(1) + .WillOnce(testing::Return(true)); + } +} +} // namespace hdfs::tools::test diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h new file mode 100644 index 00000000000..632716bf0a6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-rm-mock.h @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_RM_MOCK +#define LIBHDFSPP_TOOLS_HDFS_RM_MOCK + +#include +#include +#include +#include + +#include + +#include "hdfs-rm.h" + +namespace hdfs::tools::test { +/** + * {@class RmMock} is an {@class Rm} whereby it mocks the + * HandleHelp and HandlePath methods for testing their functionality. + */ +class RmMock : public hdfs::tools::Rm { +public: + /** + * {@inheritdoc} + */ + RmMock(const int argc, char **argv) : Rm(argc, argv) {} + + // Abiding to the Rule of 5 + RmMock(const RmMock &) = delete; + RmMock(RmMock &&) = delete; + RmMock &operator=(const RmMock &) = delete; + RmMock &operator=(RmMock &&) = delete; + ~RmMock() override; + + /** + * Defines the methods and the corresponding arguments that are expected + * to be called on this instance of {@link HdfsTool} for the given test case. + * + * @param test_case An {@link std::function} object that points to the + * function defining the test case + * @param args The arguments that are passed to this test case + */ + void SetExpectations(std::function()> test_case, + const std::vector &args = {}) const; + + MOCK_METHOD(bool, HandleHelp, (), (const, override)); + + MOCK_METHOD(bool, HandlePath, (const bool, const std::string &), + (const, override)); +}; +} // namespace hdfs::tools::test + +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc index 8fa1ef812a4..7678834a955 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc @@ -25,13 +25,16 @@ #include "hdfs-chmod-mock.h" #include "hdfs-chown-mock.h" #include "hdfs-copy-to-local-mock.h" +#include "hdfs-count-mock.h" #include "hdfs-create-snapshot-mock.h" #include "hdfs-delete-snapshot-mock.h" #include "hdfs-df-mock.h" #include "hdfs-disallow-snapshot-mock.h" #include "hdfs-du-mock.h" +#include "hdfs-mkdir-mock.h" #include "hdfs-move-to-local-mock.h" #include "hdfs-rename-snapshot-mock.h" +#include "hdfs-rm-mock.h" #include "hdfs-tool-test-fixtures.h" #include "hdfs-tool-tests.h" @@ -110,6 +113,27 @@ INSTANTIATE_TEST_SUITE_P( testing::Values(CallHelp, Pass2Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsCount, HdfsToolBasicTest, + testing::Values(CallHelp, + PassAPath, + PassQOptAndPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMkdir, HdfsToolBasicTest, + testing::Values( + CallHelp, + PassAPath, + PassPOptAndPath, + PassMOptPermissionsAndAPath, + PassMPOptsPermissionsAndAPath)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRm, HdfsToolBasicTest, + testing::Values(CallHelp, + PassAPath, + PassRecursivePath)); + // Negative tests INSTANTIATE_TEST_SUITE_P( HdfsAllowSnapshot, HdfsToolNegativeTestThrows, @@ -153,6 +177,41 @@ INSTANTIATE_TEST_SUITE_P( HdfsMoveToLocal, HdfsToolNegativeTestThrows, testing::Values(Pass3Paths)); +INSTANTIATE_TEST_SUITE_P( + HdfsCount, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursive)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMkdir, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursive, + PassMOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRm, HdfsToolNegativeTestThrows, + testing::Values(Pass2Paths, + Pass3Paths, + PassNOptAndAPath, + PassRecursiveOwnerAndAPath, + PassMOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsRm, HdfsToolNegativeTestNoThrow, + testing::Values(PassRecursive)); + +INSTANTIATE_TEST_SUITE_P( + HdfsMkdir, HdfsToolNegativeTestNoThrow, + testing::Values(PassPOpt)); + +INSTANTIATE_TEST_SUITE_P( + HdfsCount, HdfsToolNegativeTestNoThrow, + testing::Values(PassQOpt)); + INSTANTIATE_TEST_SUITE_P( HdfsMoveToLocal, HdfsToolNegativeTestNoThrow, testing::Values(PassAPath)); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h index a34435cd638..12dbc6c01ce 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.h @@ -177,4 +177,98 @@ template std::unique_ptr PassRecursivePermissionsAndAPath() { return hdfs_tool; } +template std::unique_ptr PassQOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-q"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassQOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassQOptAndPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-q"); + static std::string arg2("a/b/c"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassQOptAndPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassPOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-p"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassPOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassMOpt() { + constexpr auto argc = 2; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-m"); + + static char *argv[] = {exe.data(), arg1.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassMOpt, {arg1}); + return hdfs_tool; +} + +template std::unique_ptr PassPOptAndPath() { + constexpr auto argc = 3; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-p"); + static std::string arg2("a/b/c"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassPOptAndPath, {arg1, arg2}); + return hdfs_tool; +} + +template std::unique_ptr PassMOptPermissionsAndAPath() { + constexpr auto argc = 4; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-m"); + static std::string arg2("757"); + static std::string arg3("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassMOptPermissionsAndAPath, + {arg1, arg2, arg3}); + return hdfs_tool; +} + +template std::unique_ptr PassMPOptsPermissionsAndAPath() { + constexpr auto argc = 5; + static std::string exe("hdfs_tool_name"); + static std::string arg1("-m"); + static std::string arg2("757"); + static std::string arg3("-p"); + static std::string arg4("g/h/i"); + + static char *argv[] = {exe.data(), arg1.data(), arg2.data(), arg3.data(), + arg4.data()}; + + auto hdfs_tool = std::make_unique(argc, argv); + hdfs_tool->SetExpectations(PassMPOptsPermissionsAndAPath, + {arg1, arg2, arg3, arg4}); + return hdfs_tool; +} + #endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt index 27f95523194..aa82c018b38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt @@ -44,11 +44,9 @@ add_subdirectory(hdfs-chmod) add_executable(hdfs_find hdfs_find.cc) target_link_libraries(hdfs_find tools_common hdfspp_static) -add_executable(hdfs_mkdir hdfs_mkdir.cc) -target_link_libraries(hdfs_mkdir tools_common hdfspp_static) +add_subdirectory(hdfs-mkdir) -add_executable(hdfs_rm hdfs_rm.cc) -target_link_libraries(hdfs_rm tools_common hdfspp_static) +add_subdirectory(hdfs-rm) add_executable(hdfs_ls hdfs_ls.cc) target_link_libraries(hdfs_ls tools_common hdfspp_static) @@ -56,8 +54,7 @@ target_link_libraries(hdfs_ls tools_common hdfspp_static) add_executable(hdfs_stat hdfs_stat.cc) target_link_libraries(hdfs_stat tools_common hdfspp_static) -add_executable(hdfs_count hdfs_count.cc) -target_link_libraries(hdfs_count tools_common hdfspp_static) +add_subdirectory(hdfs-count) add_subdirectory(hdfs-df) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt new file mode 100644 index 00000000000..fbf21fecbe2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_count_lib STATIC $ hdfs-count.cc) +target_include_directories(hdfs_count_lib PRIVATE ../../tools hdfs-count ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_count_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_count main.cc) +target_include_directories(hdfs_count PRIVATE ../../tools) +target_link_libraries(hdfs_count PRIVATE hdfs_count_lib) + +install(TARGETS hdfs_count RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc new file mode 100644 index 00000000000..fca5c9713b6 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.cc @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "hdfs-count.h" +#include "tools_common.h" + +namespace hdfs::tools { +Count::Count(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Count::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options( + "help,h", + "Count the number of directories, files and bytes under the given path"); + add_options("show-quota,q", "Output additional columns before the rest: " + "QUOTA, SPACE_QUOTA, SPACE_CONSUMED"); + add_options("path", po::value(), + "The path to the file that needs to be count-ed"); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Count::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_count [OPTION] FILE" << std::endl + << std::endl + << "Count the number of directories, files and bytes under the path " + "that match the specified FILE pattern." + << std::endl + << "The output columns with -count are: DIR_COUNT, FILE_COUNT, " + "CONTENT_SIZE, PATHNAME" + << std::endl + << std::endl + << " -q output additional columns before the rest: QUOTA, " + "SPACE_QUOTA, SPACE_CONSUMED" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_count hdfs://localhost.localdomain:8020/dir" << std::endl + << "hdfs_count -q /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Count::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS count tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto show_quota = opt_val_.count("show-quota") > 0; + return HandlePath(show_quota, path); + } + + return false; +} + +bool Count::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Count::HandlePath(const bool show_quota, const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (fs == nullptr) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + hdfs::ContentSummary content_summary; + const auto status = fs->GetContentSummary(uri.get_path(), content_summary); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + + std::cout << content_summary.str(show_quota) << std::endl; + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h new file mode 100644 index 00000000000..473c60967fe --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/hdfs-count.h @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_COUNT +#define LIBHDFSPP_TOOLS_HDFS_COUNT + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Count} is an {@class HdfsTool} that counts the number of directories, + * files and bytes under the given path. + */ +class Count : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Count(int argc, char **argv); + + // Abiding to the Rule of 5 + Count(const Count &) = default; + Count(Count &&) = default; + Count &operator=(const Count &) = delete; + Count &operator=(Count &&) = delete; + ~Count() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param show_quota Output additional columns before the rest: QUOTA, + * SPACE_QUOTA, SPACE_CONSUMED. + * @param path The path to the directory for which the files, directories and + * bytes need to be counted. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(bool show_quota, + const std::string &path) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc new file mode 100644 index 00000000000..807ad3b2c39 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-count/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-count.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS count " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Count count(argc, argv); + auto success = false; + + try { + success = count.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt new file mode 100644 index 00000000000..5efce8de86b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_mkdir_lib STATIC $ hdfs-mkdir.cc) +target_include_directories(hdfs_mkdir_lib PRIVATE ../../tools hdfs-mkdir ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_mkdir_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_mkdir main.cc) +target_include_directories(hdfs_mkdir PRIVATE ../../tools) +target_link_libraries(hdfs_mkdir PRIVATE hdfs_mkdir_lib) + +install(TARGETS hdfs_mkdir RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc new file mode 100644 index 00000000000..9c406fb959f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.cc @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "hdfs-mkdir.h" +#include "tools_common.h" + +namespace hdfs::tools { +Mkdir::Mkdir(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Mkdir::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Create directory if it does not exist"); + add_options("create-parents,p", "Create parent directories as needed"); + add_options( + "mode,m", po::value(), + "Set the permissions for the new directory (and newly created parents if " + "any). The permissions are specified in octal representation"); + add_options("path", po::value(), + "The path to the directory that needs to be created"); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Mkdir::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_mkdir [OPTION] DIRECTORY" << std::endl + << std::endl + << "Create the DIRECTORY(ies), if they do not already exist." + << std::endl + << std::endl + << " -p make parent directories as needed" << std::endl + << " -m MODE set file mode (octal permissions) for the new " + "DIRECTORY(ies)" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2" << std::endl + << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir" + << std::endl; + return desc.str(); +} + +bool Mkdir::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS mkdir tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto create_parents = opt_val_.count("create-parents") > 0; + const auto permissions = + opt_val_.count("mode") > 0 + ? std::optional(opt_val_["mode"].as()) + : std::nullopt; + return HandlePath(create_parents, permissions, path); + } + + return false; +} + +bool Mkdir::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Mkdir::HandlePath(const bool create_parents, + const std::optional &permissions, + const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (fs == nullptr) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + const auto status = + fs->Mkdirs(uri.get_path(), GetPermissions(permissions), create_parents); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + + return true; +} + +uint16_t Mkdir::GetPermissions(const std::optional &permissions) { + if (permissions) { + // TODO : Handle the error returned by std::strtol. + return static_cast( + std::strtol(permissions.value().c_str(), nullptr, 8)); + } + + return hdfs::FileSystem::GetDefaultPermissionMask(); +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h new file mode 100644 index 00000000000..60875f3c7af --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/hdfs-mkdir.h @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_MKDIR +#define LIBHDFSPP_TOOLS_HDFS_MKDIR + +#include +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Mkdir} is an {@class HdfsTool} that creates directory if it does not + * exist. + */ +class Mkdir : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Mkdir(int argc, char **argv); + + // Abiding to the Rule of 5 + Mkdir(const Mkdir &) = default; + Mkdir(Mkdir &&) = default; + Mkdir &operator=(const Mkdir &) = delete; + Mkdir &operator=(Mkdir &&) = delete; + ~Mkdir() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param create_parents Creates parent directories as needed if this boolean + * is set to true. + * @param permissions An octal representation of the permissions to be stamped + * to each directory that gets created. + * @param path The path in the filesystem where the directory must be created. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool + HandlePath(bool create_parents, const std::optional &permissions, + const std::string &path) const; + + /** + * @param permissions The permissions string to convert to octal value. + * @return The octal representation of the permissions supplied as parameter + * to this tool. + */ + [[nodiscard]] static uint16_t + GetPermissions(const std::optional &permissions); + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc new file mode 100644 index 00000000000..3f70ef6dd57 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-mkdir/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-mkdir.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS mkdir " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Mkdir mkdir(argc, argv); + auto success = false; + + try { + success = mkdir.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt new file mode 100644 index 00000000000..70a7db51674 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_library(hdfs_rm_lib STATIC $ hdfs-rm.cc) +target_include_directories(hdfs_rm_lib PRIVATE ../../tools hdfs-rm ${Boost_INCLUDE_DIRS}) +target_link_libraries(hdfs_rm_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static) + +add_executable(hdfs_rm main.cc) +target_include_directories(hdfs_rm PRIVATE ../../tools) +target_link_libraries(hdfs_rm PRIVATE hdfs_rm_lib) + +install(TARGETS hdfs_rm RUNTIME DESTINATION bin) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc new file mode 100644 index 00000000000..ef1a85649d2 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.cc @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "hdfs-rm.h" +#include "tools_common.h" + +namespace hdfs::tools { +Rm::Rm(const int argc, char **argv) : HdfsTool(argc, argv) {} + +bool Rm::Initialize() { + auto add_options = opt_desc_.add_options(); + add_options("help,h", "Remove/unlink the files or directories."); + add_options("recursive,R", + "Remove the directories and their contents recursively."); + add_options("path", po::value(), + "The path to the file that needs to be removed."); + + // We allow only one argument to be passed to this tool. An exception is + // thrown if multiple arguments are passed. + pos_opt_desc_.add("path", 1); + + po::store(po::command_line_parser(argc_, argv_) + .options(opt_desc_) + .positional(pos_opt_desc_) + .run(), + opt_val_); + po::notify(opt_val_); + return true; +} + +std::string Rm::GetDescription() const { + std::stringstream desc; + desc << "Usage: hdfs_rm [OPTION] FILE" << std::endl + << std::endl + << "Remove (unlink) the FILE(s) or directory(ies)." << std::endl + << std::endl + << " -R remove directories and their contents recursively" + << std::endl + << " -h display this help and exit" << std::endl + << std::endl + << "Examples:" << std::endl + << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file" << std::endl + << "hdfs_rm -R /dir1/dir2" << std::endl; + return desc.str(); +} + +bool Rm::Do() { + if (!Initialize()) { + std::cerr << "Unable to initialize HDFS rm tool" << std::endl; + return false; + } + + if (!ValidateConstraints()) { + std::cout << GetDescription(); + return false; + } + + if (opt_val_.count("help") > 0) { + return HandleHelp(); + } + + if (opt_val_.count("path") > 0) { + const auto path = opt_val_["path"].as(); + const auto recursive = opt_val_.count("recursive") > 0; + return HandlePath(recursive, path); + } + + return false; +} + +bool Rm::HandleHelp() const { + std::cout << GetDescription(); + return true; +} + +bool Rm::HandlePath(const bool recursive, const std::string &path) const { + // Building a URI object from the given uri_path + auto uri = hdfs::parse_path_or_exit(path); + + const auto fs = hdfs::doConnect(uri, false); + if (fs == nullptr) { + std::cerr << "Could not connect the file system." << std::endl; + return false; + } + + const auto status = fs->Delete(uri.get_path(), recursive); + if (!status.ok()) { + std::cerr << "Error: " << status.ToString() << std::endl; + return false; + } + return true; +} +} // namespace hdfs::tools diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h new file mode 100644 index 00000000000..af5faf65be4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/hdfs-rm.h @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_TOOLS_HDFS_RM +#define LIBHDFSPP_TOOLS_HDFS_RM + +#include + +#include + +#include "hdfs-tool.h" + +namespace hdfs::tools { +/** + * {@class Rm} is an {@class HdfsTool} that removes/unlinks the files or + * directories. + */ +class Rm : public HdfsTool { +public: + /** + * {@inheritdoc} + */ + Rm(int argc, char **argv); + + // Abiding to the Rule of 5 + Rm(const Rm &) = default; + Rm(Rm &&) = default; + Rm &operator=(const Rm &) = delete; + Rm &operator=(Rm &&) = delete; + ~Rm() override = default; + + /** + * {@inheritdoc} + */ + [[nodiscard]] std::string GetDescription() const override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Do() override; + +protected: + /** + * {@inheritdoc} + */ + [[nodiscard]] bool Initialize() override; + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool ValidateConstraints() const override { return argc_ > 1; } + + /** + * {@inheritdoc} + */ + [[nodiscard]] bool HandleHelp() const override; + + /** + * Handle the path argument that's passed to this tool. + * + * @param recursive Perform this operation recursively on the sub-directories. + * @param path The path to the file/directory that needs to be removed. + * + * @return A boolean indicating the result of this operation. + */ + [[nodiscard]] virtual bool HandlePath(bool recursive, + const std::string &path) const; + +private: + /** + * A boost data-structure containing the description of positional arguments + * passed to the command-line. + */ + po::positional_options_description pos_opt_desc_; +}; +} // namespace hdfs::tools +#endif diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc new file mode 100644 index 00000000000..16577f52a84 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-rm/main.cc @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +#include "hdfs-rm.h" + +int main(int argc, char *argv[]) { + const auto result = std::atexit([]() -> void { + // Clean up static data on exit and prevent valgrind memory leaks + google::protobuf::ShutdownProtobufLibrary(); + }); + if (result != 0) { + std::cerr << "Error: Unable to schedule clean-up tasks for HDFS rm " + "tool, exiting" + << std::endl; + std::exit(EXIT_FAILURE); + } + + hdfs::tools::Rm rm(argc, argv); + auto success = false; + + try { + success = rm.Do(); + } catch (const std::exception &e) { + std::cerr << "Error: " << e.what() << std::endl; + } + + if (!success) { + std::exit(EXIT_FAILURE); + } + return 0; +} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc deleted file mode 100644 index 345ccc6e279..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_count.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_count [OPTION] FILE" - << std::endl - << std::endl << "Count the number of directories, files and bytes under the path that match the specified FILE pattern." - << std::endl << "The output columns with -count are: DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME" - << std::endl - << std::endl << " -q output additional columns before the rest: QUOTA, SPACE_QUOTA, SPACE_CONSUMED" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_count hdfs://localhost.localdomain:8020/dir" - << std::endl << "hdfs_count -q /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool quota = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "qh")) != -1) { - switch (input) - { - case 'q': - quota = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::ContentSummary content_summary; - hdfs::Status status = fs->GetContentSummary(uri.get_path(), content_summary); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - std::cout << content_summary.str(quota) << std::endl; - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc deleted file mode 100644 index 3ccc6017b58..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_mkdir.cc +++ /dev/null @@ -1,98 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_mkdir [OPTION] DIRECTORY" - << std::endl - << std::endl << "Create the DIRECTORY(ies), if they do not already exist." - << std::endl - << std::endl << " -p make parent directories as needed" - << std::endl << " -m MODE set file mode (octal permissions) for the new DIRECTORY(ies)" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_mkdir hdfs://localhost.localdomain:8020/dir1/dir2" - << std::endl << "hdfs_mkdir -p /extant_dir/non_extant_dir/non_extant_dir/new_dir" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool create_parents = false; - uint16_t permissions = hdfs::FileSystem::GetDefaultPermissionMask(); - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "pm:h")) != -1) { - switch (input) - { - case 'p': - create_parents = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case 'm': - //Get octal permissions for the new DIRECTORY(ies) - permissions = strtol(optarg, NULL, 8); - break; - case '?': - if (optopt == 'm') - std::cerr << "Option -" << (char) optopt << " requires an argument." << std::endl; - else if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, false); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->Mkdirs(uri.get_path(), permissions, create_parents); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc deleted file mode 100644 index 7056cf9a674..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_rm.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. -*/ - -#include -#include -#include "tools_common.h" - -void usage(){ - std::cout << "Usage: hdfs_rm [OPTION] FILE" - << std::endl - << std::endl << "Remove (unlink) the FILE(s) or directory(ies)." - << std::endl - << std::endl << " -R remove directories and their contents recursively" - << std::endl << " -h display this help and exit" - << std::endl - << std::endl << "Examples:" - << std::endl << "hdfs_rm hdfs://localhost.localdomain:8020/dir/file" - << std::endl << "hdfs_rm -R /dir1/dir2" - << std::endl; -} - -int main(int argc, char *argv[]) { - //We should have at least 2 arguments - if (argc < 2) { - usage(); - exit(EXIT_FAILURE); - } - - bool recursive = false; - int input; - - //Using GetOpt to read in the values - opterr = 0; - while ((input = getopt(argc, argv, "Rh")) != -1) { - switch (input) - { - case 'R': - recursive = true; - break; - case 'h': - usage(); - exit(EXIT_SUCCESS); - case '?': - if (isprint(optopt)) - std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl; - else - std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl; - usage(); - exit(EXIT_FAILURE); - default: - exit(EXIT_FAILURE); - } - } - std::string uri_path = argv[optind]; - - //Building a URI object from the given uri_path - hdfs::URI uri = hdfs::parse_path_or_exit(uri_path); - - std::shared_ptr fs = hdfs::doConnect(uri, true); - if (!fs) { - std::cerr << "Could not connect the file system. " << std::endl; - exit(EXIT_FAILURE); - } - - hdfs::Status status = fs->Delete(uri.get_path(), recursive); - if (!status.ok()) { - std::cerr << "Error: " << status.ToString() << std::endl; - exit(EXIT_FAILURE); - } - - // Clean up static data and prevent valgrind memory leaks - google::protobuf::ShutdownProtobufLibrary(); - return 0; -}