HDFS-16419. Make HDFS data transfer tools cross platform (#3873)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt

@@ -30,6 +30,8 @@ add_executable(hdfs_tool_tests
                 hdfs-tool-tests.cc
                 hdfs-df-mock.cc
                 hdfs-du-mock.cc
+                hdfs-copy-to-local-mock.cc
+                hdfs-move-to-local-mock.cc
                 main.cc)
 target_include_directories(hdfs_tool_tests PRIVATE
                             ../tools
@@ -44,6 +46,8 @@ target_include_directories(hdfs_tool_tests PRIVATE
                             ../../tools/hdfs-chown
                             ../../tools/hdfs-chgrp
                             ../../tools/hdfs-chmod
+                            ../../tools/hdfs-copy-to-local
+                            ../../tools/hdfs-move-to-local
                             ../../tools/hdfs-cat)
 target_link_libraries(hdfs_tool_tests PRIVATE
                        gmock_main
@@ -57,5 +61,7 @@ target_link_libraries(hdfs_tool_tests PRIVATE
                        hdfs_chown_lib
                        hdfs_chgrp_lib
                        hdfs_chmod_lib
+                       hdfs_copyToLocal_lib
+                       hdfs_moveToLocal_lib
                        hdfs_cat_lib)
 add_test(hdfs_tool_tests hdfs_tool_tests)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.cc

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-copy-to-local-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+CopyToLocalMock::~CopyToLocalMock() = default;
+
+void CopyToLocalMock::SetExpectations(
+    std::function<std::unique_ptr<CopyToLocalMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<CopyToLocalMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<CopyToLocalMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &Pass2Paths<CopyToLocalMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    EXPECT_CALL(*this, HandlePath(arg1, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-copy-to-local-mock.h

@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-copy-to-local.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class CopyToLocalMock} is an {@class CopyToLocal} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class CopyToLocalMock : public hdfs::tools::CopyToLocal {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CopyToLocalMock(const int argc, char **argv) : CopyToLocal(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  CopyToLocalMock(const CopyToLocalMock &) = delete;
+  CopyToLocalMock(CopyToLocalMock &&) = delete;
+  CopyToLocalMock &operator=(const CopyToLocalMock &) = delete;
+  CopyToLocalMock &operator=(CopyToLocalMock &&) = delete;
+  ~CopyToLocalMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void
+  SetExpectations(std::function<std::unique_ptr<CopyToLocalMock>()> test_case,
+                  const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.cc

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "hdfs-move-to-local-mock.h"
+#include "hdfs-tool-tests.h"
+
+namespace hdfs::tools::test {
+MoveToLocalMock::~MoveToLocalMock() = default;
+
+void MoveToLocalMock::SetExpectations(
+    std::function<std::unique_ptr<MoveToLocalMock>()> test_case,
+    const std::vector<std::string> &args) const {
+  // Get the pointer to the function that defines the test case
+  const auto test_case_func =
+      test_case.target<std::unique_ptr<MoveToLocalMock> (*)()>();
+  ASSERT_NE(test_case_func, nullptr);
+
+  // Set the expected method calls and their corresponding arguments for each
+  // test case
+  if (*test_case_func == &CallHelp<MoveToLocalMock>) {
+    EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
+    return;
+  }
+
+  if (*test_case_func == &Pass2Paths<MoveToLocalMock>) {
+    const auto arg1 = args[0];
+    const auto arg2 = args[1];
+    EXPECT_CALL(*this, HandlePath(arg1, arg2))
+        .Times(1)
+        .WillOnce(testing::Return(true));
+  }
+}
+} // namespace hdfs::tools::test

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-move-to-local-mock.h

@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL_MOCK
+#define LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL_MOCK
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+
+#include "hdfs-move-to-local.h"
+
+namespace hdfs::tools::test {
+/**
+ * {@class MoveToLocalMock} is an {@class MoveToLocal} whereby it mocks the
+ * HandleHelp and HandlePath methods for testing their functionality.
+ */
+class MoveToLocalMock : public hdfs::tools::MoveToLocal {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  MoveToLocalMock(const int argc, char **argv) : MoveToLocal(argc, argv) {}
+
+  // Abiding to the Rule of 5
+  MoveToLocalMock(const MoveToLocalMock &) = delete;
+  MoveToLocalMock(MoveToLocalMock &&) = delete;
+  MoveToLocalMock &operator=(const MoveToLocalMock &) = delete;
+  MoveToLocalMock &operator=(MoveToLocalMock &&) = delete;
+  ~MoveToLocalMock() override;
+
+  /**
+   * Defines the methods and the corresponding arguments that are expected
+   * to be called on this instance of {@link HdfsTool} for the given test case.
+   *
+   * @param test_case An {@link std::function} object that points to the
+   * function defining the test case
+   * @param args The arguments that are passed to this test case
+   */
+  void
+  SetExpectations(std::function<std::unique_ptr<MoveToLocalMock>()> test_case,
+                  const std::vector<std::string> &args = {}) const;
+
+  MOCK_METHOD(bool, HandleHelp, (), (const, override));
+
+  MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
+              (const, override));
+};
+} // namespace hdfs::tools::test
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc

@@ -24,11 +24,13 @@
 #include "hdfs-chgrp-mock.h"
 #include "hdfs-chmod-mock.h"
 #include "hdfs-chown-mock.h"
+#include "hdfs-copy-to-local-mock.h"
 #include "hdfs-create-snapshot-mock.h"
 #include "hdfs-delete-snapshot-mock.h"
 #include "hdfs-df-mock.h"
 #include "hdfs-disallow-snapshot-mock.h"
 #include "hdfs-du-mock.h"
+#include "hdfs-move-to-local-mock.h"
 #include "hdfs-rename-snapshot-mock.h"
 #include "hdfs-tool-test-fixtures.h"
 #include "hdfs-tool-tests.h"
@@ -98,6 +100,16 @@ INSTANTIATE_TEST_SUITE_P(
                     PassOwnerAndAPath<hdfs::tools::test::ChgrpMock>,
                     PassRecursiveOwnerAndAPath<hdfs::tools::test::ChgrpMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCopyToLocal, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::CopyToLocalMock>,
+                    Pass2Paths<hdfs::tools::test::CopyToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMoveToLocal, HdfsToolBasicTest,
+    testing::Values(CallHelp<hdfs::tools::test::MoveToLocalMock>,
+                    Pass2Paths<hdfs::tools::test::MoveToLocalMock>));
+
 // Negative tests
 INSTANTIATE_TEST_SUITE_P(
     HdfsAllowSnapshot, HdfsToolNegativeTestThrows,
@@ -133,6 +145,22 @@ INSTANTIATE_TEST_SUITE_P(
     HdfsCat, HdfsToolNegativeTestThrows,
     testing::Values(Pass2Paths<hdfs::tools::test::CatMock>));
 
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCopyToLocal, HdfsToolNegativeTestThrows,
+    testing::Values(Pass3Paths<hdfs::tools::test::CopyToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMoveToLocal, HdfsToolNegativeTestThrows,
+    testing::Values(Pass3Paths<hdfs::tools::test::MoveToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsMoveToLocal, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::MoveToLocalMock>));
+
+INSTANTIATE_TEST_SUITE_P(
+    HdfsCopyToLocal, HdfsToolNegativeTestNoThrow,
+    testing::Values(PassAPath<hdfs::tools::test::CopyToLocalMock>));
+
 INSTANTIATE_TEST_SUITE_P(
     HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow,
     testing::Values(PassAPath<hdfs::tools::test::DeleteSnapshotMock>));

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt

@@ -66,11 +66,9 @@ add_subdirectory(hdfs-du)
 add_executable(hdfs_get hdfs_get.cc)
 target_link_libraries(hdfs_get tools_common hdfspp_static)
 
-add_executable(hdfs_copyToLocal hdfs_copyToLocal.cc)
-target_link_libraries(hdfs_copyToLocal tools_common hdfspp_static)
+add_subdirectory(hdfs-copy-to-local)
 
-add_executable(hdfs_moveToLocal hdfs_moveToLocal.cc)
-target_link_libraries(hdfs_moveToLocal tools_common hdfspp_static)
+add_subdirectory(hdfs-move-to-local)
 
 add_executable(hdfs_setrep hdfs_setrep.cc)
 target_link_libraries(hdfs_setrep tools_common hdfspp_static)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/CMakeLists.txt

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_copyToLocal_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-copy-to-local.cc)
+target_include_directories(hdfs_copyToLocal_lib PRIVATE ../../tools hdfs-copyToLocal ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_copyToLocal_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_copyToLocal main.cc)
+target_include_directories(hdfs_copyToLocal PRIVATE ../../tools)
+target_link_libraries(hdfs_copyToLocal PRIVATE hdfs_copyToLocal_lib)
+
+install(TARGETS hdfs_copyToLocal RUNTIME DESTINATION bin)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc

@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-copy-to-local.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+CopyToLocal::CopyToLocal(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool CopyToLocal::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Copies the file from the given HDFS source path to "
+                        "the destination path on the local machine");
+
+  add_options("source", po::value<std::string>(), "The HDFS source file path");
+  add_options("target", po::value<std::string>(),
+              "The target local system file path");
+
+  // We allow only two arguments to be passed to this tool. An exception is
+  // thrown if more arguments are passed.
+  pos_opt_desc_.add("source", 1);
+  pos_opt_desc_.add("target", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool CopyToLocal::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help") > 0;
+  }
+
+  // Rest of the cases must contain exactly 2 arguments
+  return argc_ == 3;
+}
+
+std::string CopyToLocal::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE" << std::endl
+       << std::endl
+       << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
+       << std::endl
+       << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file "
+          "/home/usr/myfile"
+       << std::endl
+       << "hdfs_copyToLocal /dir/file /home/usr/dir/file" << std::endl;
+  return desc.str();
+}
+
+bool CopyToLocal::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS copyToLocal tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("source") > 0 && opt_val_.count("target") > 0) {
+    const auto source = opt_val_["source"].as<std::string>();
+    const auto target = opt_val_["target"].as<std::string>();
+    return HandlePath(source, target);
+  }
+
+  return false;
+}
+
+bool CopyToLocal::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool CopyToLocal::HandlePath(const std::string &source,
+                             const std::string &target) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(source);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  auto dst_file = std::fopen(target.c_str(), "wb");
+  if (!dst_file) {
+    std::cerr << "Unable to open the destination file: " << target << std::endl;
+    return false;
+  }
+
+  readFile(fs, uri.get_path(), 0, dst_file, false);
+  std::fclose(dst_file);
+  return true;
+}
+} // namespace hdfs::tools

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h

@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL
+#define LIBHDFSPP_TOOLS_HDFS_COPY_TO_LOCAL
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class CopyToLocal} is an {@class HdfsTool} that copies the file from the
+ * given HDFS source path to the destination path on the local machine.
+ */
+class CopyToLocal : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  CopyToLocal(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  CopyToLocal(const CopyToLocal &) = default;
+  CopyToLocal(CopyToLocal &&) = default;
+  CopyToLocal &operator=(const CopyToLocal &) = delete;
+  CopyToLocal &operator=(CopyToLocal &&) = delete;
+  ~CopyToLocal() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param source The source file path in HDFS.
+   * @param target The target file path on the local machine.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &source,
+                                        const std::string &target) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/main.cc

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-copy-to-local.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr
+        << "Error: Unable to schedule clean-up tasks for HDFS copy_to_local tool, exiting"
+        << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::CopyToLocal copy_to_local(argc, argv);
+  auto success = false;
+
+  try {
+    success = copy_to_local.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/CMakeLists.txt

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+add_library(hdfs_moveToLocal_lib STATIC  $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-move-to-local.cc)
+target_include_directories(hdfs_moveToLocal_lib PRIVATE ../../tools hdfs-moveToLocal ${Boost_INCLUDE_DIRS})
+target_link_libraries(hdfs_moveToLocal_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static)
+
+add_executable(hdfs_moveToLocal main.cc)
+target_include_directories(hdfs_moveToLocal PRIVATE ../../tools)
+target_link_libraries(hdfs_moveToLocal PRIVATE hdfs_moveToLocal_lib)
+
+install(TARGETS hdfs_moveToLocal RUNTIME DESTINATION bin)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.cc

@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdio>
+#include <iostream>
+#include <memory>
+#include <ostream>
+#include <sstream>
+#include <string>
+
+#include "hdfs-move-to-local.h"
+#include "tools_common.h"
+
+namespace hdfs::tools {
+MoveToLocal::MoveToLocal(const int argc, char **argv) : HdfsTool(argc, argv) {}
+
+bool MoveToLocal::Initialize() {
+  auto add_options = opt_desc_.add_options();
+  add_options("help,h", "Moves the file from the given HDFS source path to "
+                        "the destination path on the local machine");
+
+  add_options("source", po::value<std::string>(), "The HDFS source file path");
+  add_options("target", po::value<std::string>(),
+              "The target local system file path");
+
+  // We allow only two arguments to be passed to this tool. An exception is
+  // thrown if more arguments are passed.
+  pos_opt_desc_.add("source", 1);
+  pos_opt_desc_.add("target", 1);
+
+  po::store(po::command_line_parser(argc_, argv_)
+                .options(opt_desc_)
+                .positional(pos_opt_desc_)
+                .run(),
+            opt_val_);
+  po::notify(opt_val_);
+  return true;
+}
+
+bool MoveToLocal::ValidateConstraints() const {
+  // Only "help" is allowed as single argument
+  if (argc_ == 2) {
+    return opt_val_.count("help") > 0;
+  }
+
+  // Rest of the cases must contain exactly 2 arguments
+  return argc_ == 3;
+}
+
+std::string MoveToLocal::GetDescription() const {
+  std::stringstream desc;
+  desc << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE" << std::endl
+       << std::endl
+       << "Move SRC_FILE from hdfs to DST_FILE on the local file system."
+       << std::endl
+       << "Moving is done by copying SRC_FILE to DST_FILE, and then"
+       << std::endl
+       << "deleting DST_FILE if copy succeeded." << std::endl
+       << std::endl
+       << "  -h  display this help and exit" << std::endl
+       << std::endl
+       << "Examples:" << std::endl
+       << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file "
+          "/home/usr/myfile"
+       << std::endl
+       << "hdfs_moveToLocal /dir/file /home/usr/dir/file" << std::endl;
+  return desc.str();
+}
+
+bool MoveToLocal::Do() {
+  if (!Initialize()) {
+    std::cerr << "Unable to initialize HDFS moveToLocal tool" << std::endl;
+    return false;
+  }
+
+  if (!ValidateConstraints()) {
+    std::cout << GetDescription();
+    return false;
+  }
+
+  if (opt_val_.count("help") > 0) {
+    return HandleHelp();
+  }
+
+  if (opt_val_.count("source") > 0 && opt_val_.count("target") > 0) {
+    const auto source = opt_val_["source"].as<std::string>();
+    const auto target = opt_val_["target"].as<std::string>();
+    return HandlePath(source, target);
+  }
+
+  return false;
+}
+
+bool MoveToLocal::HandleHelp() const {
+  std::cout << GetDescription();
+  return true;
+}
+
+bool MoveToLocal::HandlePath(const std::string &source,
+                             const std::string &target) const {
+  // Building a URI object from the given path
+  auto uri = hdfs::parse_path_or_exit(source);
+
+  const auto fs = hdfs::doConnect(uri, true);
+  if (!fs) {
+    std::cerr << "Could not connect the file system. " << std::endl;
+    return false;
+  }
+
+  auto dst_file = std::fopen(target.c_str(), "wb");
+  if (!dst_file) {
+    std::cerr << "Unable to open the destination file: " << target << std::endl;
+    return false;
+  }
+
+  readFile(fs, uri.get_path(), 0, dst_file, true);
+  std::fclose(dst_file);
+  return true;
+}
+} // namespace hdfs::tools

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/hdfs-move-to-local.h

@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL
+#define LIBHDFSPP_TOOLS_HDFS_MOVE_TO_LOCAL
+
+#include <string>
+
+#include <boost/program_options.hpp>
+
+#include "hdfs-tool.h"
+
+namespace hdfs::tools {
+/**
+ * {@class MoveToLocal} is an {@class HdfsTool} that copies the file from the
+ * given HDFS source path to the destination path on the local machine.
+ */
+class MoveToLocal : public HdfsTool {
+public:
+  /**
+   * {@inheritdoc}
+   */
+  MoveToLocal(int argc, char **argv);
+
+  // Abiding to the Rule of 5
+  MoveToLocal(const MoveToLocal &) = default;
+  MoveToLocal(MoveToLocal &&) = default;
+  MoveToLocal &operator=(const MoveToLocal &) = delete;
+  MoveToLocal &operator=(MoveToLocal &&) = delete;
+  ~MoveToLocal() override = default;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] std::string GetDescription() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Do() override;
+
+protected:
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool Initialize() override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool ValidateConstraints() const override;
+
+  /**
+   * {@inheritdoc}
+   */
+  [[nodiscard]] bool HandleHelp() const override;
+
+  /**
+   * Handle the path argument that's passed to this tool.
+   *
+   * @param source The source file path in HDFS.
+   * @param target The target file path on the local machine.
+   *
+   * @return A boolean indicating the result of this operation.
+   */
+  [[nodiscard]] virtual bool HandlePath(const std::string &source,
+                                        const std::string &target) const;
+
+private:
+  /**
+   * A boost data-structure containing the description of positional arguments
+   * passed to the command-line.
+   */
+  po::positional_options_description pos_opt_desc_;
+};
+} // namespace hdfs::tools
+#endif

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-move-to-local/main.cc

@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+
+#include <google/protobuf/stubs/common.h>
+
+#include "hdfs-move-to-local.h"
+
+int main(int argc, char *argv[]) {
+  const auto result = std::atexit([]() -> void {
+    // Clean up static data on exit and prevent valgrind memory leaks
+    google::protobuf::ShutdownProtobufLibrary();
+  });
+  if (result != 0) {
+    std::cerr << "Error: Unable to schedule clean-up tasks for HDFS "
+                 "move_to_local tool, exiting"
+              << std::endl;
+    std::exit(EXIT_FAILURE);
+  }
+
+  hdfs::tools::MoveToLocal move_to_local(argc, argv);
+  auto success = false;
+
+  try {
+    success = move_to_local.Do();
+  } catch (const std::exception &e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+  }
+
+  if (!success) {
+    std::exit(EXIT_FAILURE);
+  }
+  return 0;
+}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_copyToLocal.cc

@@ -1,88 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE"
-      << std::endl
-      << std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
-      << std::endl
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
-      << std::endl << "hdfs_copyToLocal /dir/file /home/usr/dir/file"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  if (argc > 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  std::string uri_path = argv[optind];
-  std::string dest = argv[optind+1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
-  if(!dst_file){
-    std::cerr << "Unable to open the destination file: " << dest << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  readFile(fs, uri.get_path(), 0, dst_file, false);
-  std::fclose(dst_file);
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_moveToLocal.cc

@@ -1,90 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing,
-  software distributed under the License is distributed on an
-  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-  KIND, either express or implied.  See the License for the
-  specific language governing permissions and limitations
-  under the License.
-*/
-
-#include <google/protobuf/stubs/common.h>
-#include <unistd.h>
-#include "tools_common.h"
-
-void usage(){
-  std::cout << "Usage: hdfs_moveToLocal [OPTION] SRC_FILE DST_FILE"
-      << std::endl
-      << std::endl << "Move SRC_FILE from hdfs to DST_FILE on the local file system."
-      << std::endl << "Moving is done by copying SRC_FILE to DST_FILE, and then"
-      << std::endl << "deleting DST_FILE if copy succeeded."
-      << std::endl
-      << std::endl << "  -h  display this help and exit"
-      << std::endl
-      << std::endl << "Examples:"
-      << std::endl << "hdfs_moveToLocal hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
-      << std::endl << "hdfs_moveToLocal /dir/file /home/usr/dir/file"
-      << std::endl;
-}
-
-int main(int argc, char *argv[]) {
-  if (argc > 4) {
-    usage();
-    exit(EXIT_FAILURE);
-  }
-
-  int input;
-
-  //Using GetOpt to read in the values
-  opterr = 0;
-  while ((input = getopt(argc, argv, "h")) != -1) {
-    switch (input)
-    {
-    case 'h':
-      usage();
-      exit(EXIT_SUCCESS);
-    case '?':
-      if (isprint(optopt))
-        std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
-      else
-        std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
-      usage();
-      exit(EXIT_FAILURE);
-    default:
-      exit(EXIT_FAILURE);
-    }
-  }
-
-  std::string uri_path = argv[optind];
-  std::string dest = argv[optind+1];
-
-  //Building a URI object from the given uri_path
-  hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
-
-  std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
-  if (!fs) {
-    std::cerr << "Could not connect the file system. " << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
-  if(!dst_file){
-    std::cerr << "Unable to open the destination file: " << dest << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  readFile(fs, uri.get_path(), 0, dst_file, true);
-  std::fclose(dst_file);
-
-  // Clean up static data and prevent valgrind memory leaks
-  google::protobuf::ShutdownProtobufLibrary();
-  return 0;
-}