Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-15947. Replace deprecated protobuf APIs (#2856)

This commit is contained in:
Gautham B A 2021-04-05 22:22:48 +05:30 committed by GitHub
parent 9e2d5d6d05
commit 6e1df1c048
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc
View File

@ -56,7 +56,7 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL
std::string buf;
int size = msg->ByteSize();
const auto size = msg->ByteSizeLong();
buf.reserve(pbio::CodedOutputStream::VarintSize32(size) + size);
pbio::StringOutputStream ss(&buf);
pbio::CodedOutputStream os(&ss);
@ -68,9 +68,9 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL
return buf;
}
int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) {
size_t size = msg->ByteSize();
return ::google::protobuf::io::CodedOutputStream::VarintSize32(size) + size;
size_t DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) {
const auto size = msg->ByteSizeLong();
return ::google::protobuf::io::CodedOutputStream::VarintSize64(size) + size;
}
std::shared_ptr<std::string> GetRandomClientName() {

2
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h
View File

@ -47,7 +47,7 @@ Status ToStatus(const boost::system::error_code &ec);
// Determine size of buffer that needs to be allocated in order to serialize msg
// in delimited format
int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg);
size_t DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg);
// Construct msg from the input held in the CodedInputStream
// return false on failure, otherwise return true

8
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc
View File

@ -47,7 +47,7 @@ static const int kNoRetry = -1;
static void AddHeadersToPacket(std::string *res,
std::initializer_list<const pb::MessageLite *> headers,
const std::string *payload) {
int len = 0;
size_t len = 0;
std::for_each(
headers.begin(), headers.end(),
[&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
@ -68,7 +68,7 @@ static void AddHeadersToPacket(std::string *res,
std::for_each(
headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(v->ByteSizeLong(), buf);
buf = v->SerializeWithCachedSizesToArray(buf);
});
@ -78,13 +78,13 @@ static void AddHeadersToPacket(std::string *res,
}
static void ConstructPayload(std::string *res, const pb::MessageLite *header) {
int len = DelimitedPBMessageSize(header);
const auto len = DelimitedPBMessageSize(header);
res->reserve(len);
pbio::StringOutputStream ss(res);
pbio::CodedOutputStream os(&ss);
uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len);
assert(buf);
buf = pbio::CodedOutputStream::WriteVarint32ToArray(header->ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(header->ByteSizeLong(), buf);
buf = header->SerializeWithCachedSizesToArray(buf);
}

4
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc
View File

@ -36,7 +36,7 @@ using namespace ::std::placeholders;
static void AddHeadersToPacket(
std::string *res, std::initializer_list<const pb::MessageLite *> headers,
const std::string *payload) {
int len = 0;
size_t len = 0;
std::for_each(
headers.begin(), headers.end(),
[&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); });
@ -57,7 +57,7 @@ static void AddHeadersToPacket(
std::for_each(
headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) {
buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(v->ByteSizeLong(), buf);
buf = v->SerializeWithCachedSizesToArray(buf);
});

6
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc
View File

@ -119,7 +119,7 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) {
res.reserve(hdfs::DelimitedPBMessageSize(msg));
pbio::StringOutputStream os(&res);
pbio::CodedOutputStream out(&os);
out.WriteVarint32(msg->ByteSize());
out.WriteVarint64(msg->ByteSizeLong());
msg->SerializeToCodedStream(&out);
return res;
}
@ -141,9 +141,9 @@ static inline std::pair<error_code, string> ProducePacket(
*reinterpret_cast<unsigned *>(prefix) =
htonl(data.size() + checksum.size() + sizeof(int32_t));
*reinterpret_cast<short *>(prefix + sizeof(int32_t)) =
htons(proto.ByteSize());
htons(static_cast<uint16_t>(proto.ByteSizeLong()));
std::string payload(prefix, sizeof(prefix));
payload.reserve(payload.size() + proto.ByteSize() + checksum.size() +
payload.reserve(payload.size() + proto.ByteSizeLong() + checksum.size() +
data.size());
proto.AppendToString(&payload);
payload += checksum;

10
hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc
View File

@ -88,9 +88,9 @@ protected:
static inline std::pair<boost::system::error_code, string> RpcResponse(
const RpcResponseHeaderProto &h, const std::string &data,
const boost::system::error_code &ec = boost::system::error_code()) {
uint32_t payload_length =
pbio::CodedOutputStream::VarintSize32(h.ByteSize()) +
pbio::CodedOutputStream::VarintSize32(data.size()) + h.ByteSize() +
const auto payload_length =
pbio::CodedOutputStream::VarintSize64(h.ByteSizeLong()) +
pbio::CodedOutputStream::VarintSize64(data.size()) + h.ByteSizeLong() +
data.size();
std::string res;
@ -99,9 +99,9 @@ static inline std::pair<boost::system::error_code, string> RpcResponse(
buf = pbio::CodedOutputStream::WriteLittleEndian32ToArray(
htonl(payload_length), buf);
buf = pbio::CodedOutputStream::WriteVarint32ToArray(h.ByteSize(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(h.ByteSizeLong(), buf);
buf = h.SerializeWithCachedSizesToArray(buf);
buf = pbio::CodedOutputStream::WriteVarint32ToArray(data.size(), buf);
buf = pbio::CodedOutputStream::WriteVarint64ToArray(data.size(), buf);
buf = pbio::CodedOutputStream::WriteStringToArray(data, buf);
return std::make_pair(ec, std::move(res));