From 6e1df1c048a59b8ccf4655d32913c3d7ca00b977 Mon Sep 17 00:00:00 2001 From: Gautham B A Date: Mon, 5 Apr 2021 22:22:48 +0530 Subject: [PATCH] HDFS-15947. Replace deprecated protobuf APIs (#2856) --- .../src/main/native/libhdfspp/lib/common/util.cc | 8 ++++---- .../src/main/native/libhdfspp/lib/common/util.h | 2 +- .../src/main/native/libhdfspp/lib/rpc/request.cc | 8 ++++---- .../native/libhdfspp/lib/rpc/rpc_connection_impl.cc | 4 ++-- .../native/libhdfspp/tests/remote_block_reader_test.cc | 6 +++--- .../src/main/native/libhdfspp/tests/rpc_engine_test.cc | 10 +++++----- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc index 8f6e77a5395..c0e10183297 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc @@ -56,7 +56,7 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL std::string buf; - int size = msg->ByteSize(); + const auto size = msg->ByteSizeLong(); buf.reserve(pbio::CodedOutputStream::VarintSize32(size) + size); pbio::StringOutputStream ss(&buf); pbio::CodedOutputStream os(&ss); @@ -68,9 +68,9 @@ std::string SerializeDelimitedProtobufMessage(const ::google::protobuf::MessageL return buf; } -int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) { - size_t size = msg->ByteSize(); - return ::google::protobuf::io::CodedOutputStream::VarintSize32(size) + size; +size_t DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg) { + const auto size = msg->ByteSizeLong(); + return ::google::protobuf::io::CodedOutputStream::VarintSize64(size) + size; } std::shared_ptr GetRandomClientName() { diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h index 140f66e8482..45cd68a1f00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.h @@ -47,7 +47,7 @@ Status ToStatus(const boost::system::error_code &ec); // Determine size of buffer that needs to be allocated in order to serialize msg // in delimited format -int DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg); +size_t DelimitedPBMessageSize(const ::google::protobuf::MessageLite *msg); // Construct msg from the input held in the CodedInputStream // return false on failure, otherwise return true diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc index ecac2bcb00c..99762c89ee9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc @@ -47,7 +47,7 @@ static const int kNoRetry = -1; static void AddHeadersToPacket(std::string *res, std::initializer_list headers, const std::string *payload) { - int len = 0; + size_t len = 0; std::for_each( headers.begin(), headers.end(), [&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); }); @@ -68,7 +68,7 @@ static void AddHeadersToPacket(std::string *res, std::for_each( headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) { - buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf); + buf = pbio::CodedOutputStream::WriteVarint64ToArray(v->ByteSizeLong(), buf); buf = v->SerializeWithCachedSizesToArray(buf); }); @@ -78,13 +78,13 @@ static void AddHeadersToPacket(std::string *res, } static void ConstructPayload(std::string *res, const pb::MessageLite *header) { - int len = DelimitedPBMessageSize(header); + const auto len = DelimitedPBMessageSize(header); res->reserve(len); pbio::StringOutputStream ss(res); pbio::CodedOutputStream os(&ss); uint8_t *buf = os.GetDirectBufferForNBytesAndAdvance(len); assert(buf); - buf = pbio::CodedOutputStream::WriteVarint32ToArray(header->ByteSize(), buf); + buf = pbio::CodedOutputStream::WriteVarint64ToArray(header->ByteSizeLong(), buf); buf = header->SerializeWithCachedSizesToArray(buf); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc index a5de92e61bf..5d434ef370a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc @@ -36,7 +36,7 @@ using namespace ::std::placeholders; static void AddHeadersToPacket( std::string *res, std::initializer_list headers, const std::string *payload) { - int len = 0; + size_t len = 0; std::for_each( headers.begin(), headers.end(), [&len](const pb::MessageLite *v) { len += DelimitedPBMessageSize(v); }); @@ -57,7 +57,7 @@ static void AddHeadersToPacket( std::for_each( headers.begin(), headers.end(), [&buf](const pb::MessageLite *v) { - buf = pbio::CodedOutputStream::WriteVarint32ToArray(v->ByteSize(), buf); + buf = pbio::CodedOutputStream::WriteVarint64ToArray(v->ByteSizeLong(), buf); buf = v->SerializeWithCachedSizesToArray(buf); }); diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc index 4843da91786..ccec5812f61 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc @@ -119,7 +119,7 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) { res.reserve(hdfs::DelimitedPBMessageSize(msg)); pbio::StringOutputStream os(&res); pbio::CodedOutputStream out(&os); - out.WriteVarint32(msg->ByteSize()); + out.WriteVarint64(msg->ByteSizeLong()); msg->SerializeToCodedStream(&out); return res; } @@ -141,9 +141,9 @@ static inline std::pair ProducePacket( *reinterpret_cast(prefix) = htonl(data.size() + checksum.size() + sizeof(int32_t)); *reinterpret_cast(prefix + sizeof(int32_t)) = - htons(proto.ByteSize()); + htons(static_cast(proto.ByteSizeLong())); std::string payload(prefix, sizeof(prefix)); - payload.reserve(payload.size() + proto.ByteSize() + checksum.size() + + payload.reserve(payload.size() + proto.ByteSizeLong() + checksum.size() + data.size()); proto.AppendToString(&payload); payload += checksum; diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc index 931f873d6de..caf4842b298 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc @@ -88,9 +88,9 @@ protected: static inline std::pair RpcResponse( const RpcResponseHeaderProto &h, const std::string &data, const boost::system::error_code &ec = boost::system::error_code()) { - uint32_t payload_length = - pbio::CodedOutputStream::VarintSize32(h.ByteSize()) + - pbio::CodedOutputStream::VarintSize32(data.size()) + h.ByteSize() + + const auto payload_length = + pbio::CodedOutputStream::VarintSize64(h.ByteSizeLong()) + + pbio::CodedOutputStream::VarintSize64(data.size()) + h.ByteSizeLong() + data.size(); std::string res; @@ -99,9 +99,9 @@ static inline std::pair RpcResponse( buf = pbio::CodedOutputStream::WriteLittleEndian32ToArray( htonl(payload_length), buf); - buf = pbio::CodedOutputStream::WriteVarint32ToArray(h.ByteSize(), buf); + buf = pbio::CodedOutputStream::WriteVarint64ToArray(h.ByteSizeLong(), buf); buf = h.SerializeWithCachedSizesToArray(buf); - buf = pbio::CodedOutputStream::WriteVarint32ToArray(data.size(), buf); + buf = pbio::CodedOutputStream::WriteVarint64ToArray(data.size(), buf); buf = pbio::CodedOutputStream::WriteStringToArray(data, buf); return std::make_pair(ec, std::move(res));