Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-16462. Make HDFS get tool cross platform (#4003)

This commit is contained in:
Gautham B A 2022-03-05 23:05:14 +05:30 committed by GitHub
parent 19561846df
commit 8f07081789
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 308 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/CMakeLists.txt
View File

@ -35,6 +35,7 @@ add_executable(hdfs_tool_tests
hdfs-count-mock.cc
hdfs-mkdir-mock.cc
hdfs-rm-mock.cc
hdfs-get-mock.cc
main.cc)
target_include_directories(hdfs_tool_tests PRIVATE
../tools
@ -54,6 +55,7 @@ target_include_directories(hdfs_tool_tests PRIVATE
../../tools/hdfs-count
../../tools/hdfs-mkdir
../../tools/hdfs-rm
../../tools/hdfs-get
../../tools/hdfs-cat)
target_link_libraries(hdfs_tool_tests PRIVATE
gmock_main
@ -72,5 +74,6 @@ target_link_libraries(hdfs_tool_tests PRIVATE
hdfs_count_lib
hdfs_mkdir_lib
hdfs_rm_lib
hdfs_get_lib
hdfs_cat_lib)
add_test(hdfs_tool_tests hdfs_tool_tests)

56
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.cc Normal file
View File

@ -0,0 +1,56 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include "hdfs-get-mock.h"
#include "hdfs-tool-tests.h"
namespace hdfs::tools::test {
GetMock::~GetMock() = default;
void GetMock::SetExpectations(
std::function<std::unique_ptr<GetMock>()> test_case,
const std::vector<std::string> &args) const {
// Get the pointer to the function that defines the test case
const auto test_case_func =
test_case.target<std::unique_ptr<GetMock> (*)()>();
ASSERT_NE(test_case_func, nullptr);
// Set the expected method calls and their corresponding arguments for each
// test case
if (*test_case_func == &CallHelp<GetMock>) {
EXPECT_CALL(*this, HandleHelp()).Times(1).WillOnce(testing::Return(true));
return;
}
if (*test_case_func == &Pass2Paths<GetMock>) {
const auto arg1 = args[0];
const auto arg2 = args[1];
EXPECT_CALL(*this, HandlePath(arg1, arg2))
.Times(1)
.WillOnce(testing::Return(true));
}
}
} // namespace hdfs::tools::test

68
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-get-mock.h Normal file
View File

@ -0,0 +1,68 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LIBHDFSPP_TOOLS_HDFS_GET_MOCK
#define LIBHDFSPP_TOOLS_HDFS_GET_MOCK
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include <gmock/gmock.h>
#include "hdfs-get.h"
namespace hdfs::tools::test {
/**
* {@class GetMock} is an {@class Get} whereby it mocks the
* HandleHelp and HandlePath methods for testing their functionality.
*/
class GetMock : public hdfs::tools::Get {
public:
/**
* {@inheritdoc}
*/
GetMock(const int argc, char **argv) : Get(argc, argv) {}
// Abiding to the Rule of 5
GetMock(const GetMock &) = delete;
GetMock(GetMock &&) = delete;
GetMock &operator=(const GetMock &) = delete;
GetMock &operator=(GetMock &&) = delete;
~GetMock() override;
/**
* Defines the methods and the corresponding arguments that are expected
* to be called on this instance of {@link HdfsTool} for the given test case.
*
* @param test_case An {@link std::function} object that points to the
* function defining the test case
* @param args The arguments that are passed to this test case
*/
void SetExpectations(std::function<std::unique_ptr<GetMock>()> test_case,
const std::vector<std::string> &args = {}) const;
MOCK_METHOD(bool, HandleHelp, (), (const, override));
MOCK_METHOD(bool, HandlePath, (const std::string &, const std::string &),
(const, override));
};
} // namespace hdfs::tools::test
#endif

14
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/tools/hdfs-tool-tests.cc
View File

@ -31,6 +31,7 @@
#include "hdfs-df-mock.h"
#include "hdfs-disallow-snapshot-mock.h"
#include "hdfs-du-mock.h"
#include "hdfs-get-mock.h"
#include "hdfs-mkdir-mock.h"
#include "hdfs-move-to-local-mock.h"
#include "hdfs-rename-snapshot-mock.h"
@ -108,6 +109,11 @@ INSTANTIATE_TEST_SUITE_P(
testing::Values(CallHelp<hdfs::tools::test::CopyToLocalMock>,
Pass2Paths<hdfs::tools::test::CopyToLocalMock>));
INSTANTIATE_TEST_SUITE_P(
HdfsGet, HdfsToolBasicTest,
testing::Values(CallHelp<hdfs::tools::test::GetMock>,
Pass2Paths<hdfs::tools::test::GetMock>));
INSTANTIATE_TEST_SUITE_P(
HdfsMoveToLocal, HdfsToolBasicTest,
testing::Values(CallHelp<hdfs::tools::test::MoveToLocalMock>,
@ -173,6 +179,10 @@ INSTANTIATE_TEST_SUITE_P(
HdfsCopyToLocal, HdfsToolNegativeTestThrows,
testing::Values(Pass3Paths<hdfs::tools::test::CopyToLocalMock>));
INSTANTIATE_TEST_SUITE_P(
HdfsGet, HdfsToolNegativeTestThrows,
testing::Values(Pass3Paths<hdfs::tools::test::GetMock>));
INSTANTIATE_TEST_SUITE_P(
HdfsMoveToLocal, HdfsToolNegativeTestThrows,
testing::Values(Pass3Paths<hdfs::tools::test::MoveToLocalMock>));
@ -220,6 +230,10 @@ INSTANTIATE_TEST_SUITE_P(
HdfsCopyToLocal, HdfsToolNegativeTestNoThrow,
testing::Values(PassAPath<hdfs::tools::test::CopyToLocalMock>));
INSTANTIATE_TEST_SUITE_P(
HdfsGet, HdfsToolNegativeTestNoThrow,
testing::Values(PassAPath<hdfs::tools::test::GetMock>));
INSTANTIATE_TEST_SUITE_P(
HdfsDeleteSnapshot, HdfsToolNegativeTestNoThrow,
testing::Values(PassAPath<hdfs::tools::test::DeleteSnapshotMock>));

3
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/CMakeLists.txt
View File

@ -60,8 +60,7 @@ add_subdirectory(hdfs-df)
add_subdirectory(hdfs-du)
add_executable(hdfs_get hdfs_get.cc)
target_link_libraries(hdfs_get tools_common hdfspp_static)
add_subdirectory(hdfs-get)
add_subdirectory(hdfs-copy-to-local)

14
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.cc
View File

@ -64,7 +64,8 @@ bool CopyToLocal::ValidateConstraints() const {
std::string CopyToLocal::GetDescription() const {
std::stringstream desc;
desc << "Usage: hdfs_copyToLocal [OPTION] SRC_FILE DST_FILE" << std::endl
desc << "Usage: hdfs_" << GetToolName() << " [OPTION] SRC_FILE DST_FILE"
<< std::endl
<< std::endl
<< "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
<< std::endl
@ -72,16 +73,19 @@ std::string CopyToLocal::GetDescription() const {
<< " -h display this help and exit" << std::endl
<< std::endl
<< "Examples:" << std::endl
<< "hdfs_copyToLocal hdfs://localhost.localdomain:8020/dir/file "
<< "hdfs_" << GetToolName()
<< " hdfs://localhost.localdomain:8020/dir/file "
"/home/usr/myfile"
<< std::endl
<< "hdfs_copyToLocal /dir/file /home/usr/dir/file" << std::endl;
<< "hdfs_" << GetToolName() << " /dir/file /home/usr/dir/file"
<< std::endl;
return desc.str();
}
bool CopyToLocal::Do() {
if (!Initialize()) {
std::cerr << "Unable to initialize HDFS copyToLocal tool" << std::endl;
std::cerr << "Unable to initialize HDFS " << GetToolName() << " tool"
<< std::endl;
return false;
}
@ -129,4 +133,6 @@ bool CopyToLocal::HandlePath(const std::string &source,
std::fclose(dst_file);
return true;
}
std::string CopyToLocal::GetToolName() const { return "copyToLocal"; }
} // namespace hdfs::tools

5
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-copy-to-local/hdfs-copy-to-local.h
View File

@ -81,6 +81,11 @@ protected:
[[nodiscard]] virtual bool HandlePath(const std::string &source,
const std::string &target) const;
/**
* @return The name of the tool.
*/
[[nodiscard]] virtual std::string GetToolName() const;
private:
/**
* A boost data-structure containing the description of positional arguments

27
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/CMakeLists.txt Normal file
View File

@ -0,0 +1,27 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
add_library(hdfs_get_lib STATIC $<TARGET_OBJECTS:hdfs_tool_obj> hdfs-get.cc)
target_include_directories(hdfs_get_lib PRIVATE ../../tools hdfs-get ${Boost_INCLUDE_DIRS})
target_link_libraries(hdfs_get_lib PRIVATE Boost::boost Boost::program_options tools_common hdfspp_static hdfs_copyToLocal_lib)
add_executable(hdfs_get main.cc)
target_include_directories(hdfs_get PRIVATE ../../tools)
target_link_libraries(hdfs_get PRIVATE hdfs_get_lib)
install(TARGETS hdfs_get RUNTIME DESTINATION bin)

25
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.cc Normal file
View File

@ -0,0 +1,25 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "hdfs-get.h"
namespace hdfs::tools {
Get::Get(const int argc, char **argv) : CopyToLocal(argc, argv) {}
std::string Get::GetToolName() const { return "get"; }
} // namespace hdfs::tools

47
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/hdfs-get.h Normal file
View File

@ -0,0 +1,47 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LIBHDFSPP_TOOLS_HDFS_GET
#define LIBHDFSPP_TOOLS_HDFS_GET
#include "hdfs-copy-to-local/hdfs-copy-to-local.h"
namespace hdfs::tools {
class Get : public CopyToLocal {
public:
/**
* {@inheritdoc}
*/
Get(int argc, char **argv);
// Abiding to the Rule of 5
Get(const Get &) = default;
Get(Get &&) = default;
Get &operator=(const Get &) = delete;
Get &operator=(Get &&) = delete;
~Get() override = default;
protected:
/**
* {@inheritdoc}
*/
[[nodiscard]] std::string GetToolName() const override;
};
} // namespace hdfs::tools
#endif

52
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs-get/main.cc Normal file
View File

@ -0,0 +1,52 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <cstdlib>
#include <exception>
#include <iostream>
#include <google/protobuf/stubs/common.h>
#include "hdfs-get.h"
int main(int argc, char *argv[]) {
const auto result = std::atexit([]() -> void {
// Clean up static data on exit and prevent valgrind memory leaks
google::protobuf::ShutdownProtobufLibrary();
});
if (result != 0) {
std::cerr << "Error: Unable to schedule clean-up tasks for HDFS "
"get tool, exiting"
<< std::endl;
std::exit(EXIT_FAILURE);
}
hdfs::tools::Get get(argc, argv);
auto success = false;
try {
success = get.Do();
} catch (const std::exception &e) {
std::cerr << "Error: " << e.what() << std::endl;
}
if (!success) {
std::exit(EXIT_FAILURE);
}
return 0;
}

88
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tools/hdfs_get.cc
View File

@ -1,88 +0,0 @@
/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/
#include <google/protobuf/stubs/common.h>
#include <unistd.h>
#include "tools_common.h"
void usage(){
std::cout << "Usage: hdfs_get [OPTION] SRC_FILE DST_FILE"
<< std::endl
<< std::endl << "Copy SRC_FILE from hdfs to DST_FILE on the local file system."
<< std::endl
<< std::endl << " -h display this help and exit"
<< std::endl
<< std::endl << "Examples:"
<< std::endl << "hdfs_get hdfs://localhost.localdomain:8020/dir/file /home/usr/myfile"
<< std::endl << "hdfs_get /dir/file /home/usr/dir/file"
<< std::endl;
}
int main(int argc, char *argv[]) {
if (argc > 4) {
usage();
exit(EXIT_FAILURE);
}
int input;
//Using GetOpt to read in the values
opterr = 0;
while ((input = getopt(argc, argv, "h")) != -1) {
switch (input)
{
case 'h':
usage();
exit(EXIT_SUCCESS);
case '?':
if (isprint(optopt))
std::cerr << "Unknown option `-" << (char) optopt << "'." << std::endl;
else
std::cerr << "Unknown option character `" << (char) optopt << "'." << std::endl;
usage();
exit(EXIT_FAILURE);
default:
exit(EXIT_FAILURE);
}
}
std::string uri_path = argv[optind];
std::string dest = argv[optind+1];
//Building a URI object from the given uri_path
hdfs::URI uri = hdfs::parse_path_or_exit(uri_path);
std::shared_ptr<hdfs::FileSystem> fs = hdfs::doConnect(uri, false);
if (!fs) {
std::cerr << "Could not connect the file system. " << std::endl;
exit(EXIT_FAILURE);
}
std::FILE* dst_file = std::fopen(dest.c_str(), "wb");
if(!dst_file){
std::cerr << "Unable to open the destination file: " << dest << std::endl;
exit(EXIT_FAILURE);
}
readFile(fs, uri.get_path(), 0, dst_file, false);
std::fclose(dst_file);
// Clean up static data and prevent valgrind memory leaks
google::protobuf::ShutdownProtobufLibrary();
return 0;
}