HDFS-16467. Ensure Protobuf generated headers are included first (#4601)

* This PR ensures that the Protobuf generated headers
  are always included first, even when these headers
  are included transitively.
* This problem is specific to Windows only.
This commit is contained in:
Gautham B A 2022-07-23 23:20:15 +05:30 committed by GitHub
parent 63db1a85e3
commit 7de9b5ee27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 90 additions and 47 deletions

View File

@ -18,24 +18,27 @@
#include "common/util_c.h"
#include "expect.h"
#include "hdfs/hdfs.h"
#include "hdfspp/hdfs_ext.h"
#include "native_mini_dfs.h"
#include "os/thread.h"
#include "x-platform/c-api/syscall.h"
#include "hdfs/hdfs.h"
#include <errno.h>
#include <inttypes.h>
#include <pwd.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <unistd.h>
#ifndef WIN32
#include <sys/socket.h>
#include <sys/wait.h>
#include <pwd.h>
#endif
#define TO_STR_HELPER(X) #X
#define TO_STR(X) TO_STR_HELPER(X)
@ -197,7 +200,7 @@ static int fileEventCallback1(const char * event, const char * cluster, const ch
if (randomErrRatioStr) randomErrRatio = (int64_t)atoi(randomErrRatioStr);
if (randomErrRatio == 0) return DEBUG_SIMULATE_ERROR;
else if (randomErrRatio < 0) return LIBHDFSPP_EVENT_OK;
return random() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
return rand() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
}
static int fileEventCallback2(const char * event, const char * cluster, const char * file, int64_t value, int64_t cookie)
@ -235,7 +238,7 @@ static int doTestHdfsMiniStress(struct tlhThreadInfo *ti, int randomErr)
EXPECT_ZERO(hdfsCloseFile(ti->hdfs, file));
file = hdfsOpenFile(ti->hdfs, ti->fileNm, O_RDONLY, 0, 0, 0);
EXPECT_NONNULL(file);
seekPos = (((double)random()) / RAND_MAX) * (fileInfo->mSize - expected);
seekPos = (((double)rand()) / RAND_MAX) * (fileInfo->mSize - expected);
seekPos = (seekPos / expected) * expected;
ret = hdfsSeek(ti->hdfs, file, seekPos);
if (ret < 0) {

View File

@ -22,7 +22,7 @@
/* On Windows, unistd.h does not exist, so manually define what we need. */
#include <process.h> /* Declares getpid(). */
#include <windows.h>
#include <Windows.h>
/* Re-route sleep to Sleep, converting units from seconds to milliseconds. */
#define sleep(seconds) Sleep((seconds) * 1000)

View File

@ -16,6 +16,8 @@
* limitations under the License.
*/
#include "fs/filehandle.h"
#include "hdfspp/hdfspp.h"
#include "hdfspp/hdfs_ext.h"
@ -23,7 +25,6 @@
#include "common/configuration_loader.h"
#include "common/logging.h"
#include "fs/filesystem.h"
#include "fs/filehandle.h"
#include "x-platform/utils.h"
#include "x-platform/syscall.h"

View File

@ -21,6 +21,7 @@
#include <algorithm>
#include <sstream>
#include <iomanip>
#include <algorithm>
namespace hdfs {

View File

@ -19,10 +19,10 @@
#ifndef LIB_COMMON_LOGGING_H_
#define LIB_COMMON_LOGGING_H_
#include <boost/asio/ip/tcp.hpp>
#include "hdfspp/log.h"
#include <boost/asio/ip/tcp.hpp>
#include <sstream>
#include <mutex>
#include <memory>

View File

@ -26,6 +26,7 @@
#include <utility>
#include <future>
#include <boost/system/error_code.hpp>
namespace hdfs {

View File

@ -19,6 +19,7 @@
#include "common/util.h"
#include "common/util_c.h"
#include <boost/system/error_code.hpp>
#include <google/protobuf/message_lite.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>

View File

@ -18,9 +18,10 @@
#ifndef LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
#define LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
#include "ClientNamenodeProtocol.pb.h"
#include "hdfspp/ioservice.h"
#include "common/async_stream.h"
#include "ClientNamenodeProtocol.pb.h"
#include "common/libhdfs_events_impl.h"
#include "common/logging.h"
#include "common/util.h"

View File

@ -16,11 +16,12 @@
* limitations under the License.
*/
#include "reader/block_reader.h"
#include "filehandle.h"
#include "common/continuation/continuation.h"
#include "common/logging.h"
#include "connection/datanodeconnection.h"
#include "reader/block_reader.h"
#include "hdfspp/events.h"
#include "x-platform/types.h"

View File

@ -18,13 +18,14 @@
#ifndef LIBHDFSPP_LIB_FS_FILEHANDLE_H_
#define LIBHDFSPP_LIB_FS_FILEHANDLE_H_
#include "reader/readergroup.h"
#include "hdfspp/ioservice.h"
#include "common/async_stream.h"
#include "common/cancel_tracker.h"
#include "common/libhdfs_events_impl.h"
#include "common/new_delete.h"
#include "reader/fileinfo.h"
#include "reader/readergroup.h"
#include "bad_datanode_tracker.h"
#include "ClientNamenodeProtocol.pb.h"

View File

@ -16,21 +16,24 @@
* limitations under the License.
*/
#include "filesystem.h"
#include "filehandle.h"
#include "filesystem.h"
#include "common/namenode_info.h"
#include <functional>
#include <limits>
#include <future>
#include <tuple>
#include <pwd.h>
#include <iostream>
#include <boost/asio/ip/tcp.hpp>
#include "x-platform/syscall.h"
#ifndef WIN32
#include <pwd.h>
#endif
#define FMT_THIS_ADDR "this=" << (void*)this
namespace hdfs {

View File

@ -26,9 +26,12 @@
#include <future>
#include <tuple>
#include <iostream>
#include <pwd.h>
#include <utility>
#ifndef WIN32
#include <pwd.h>
#endif
#define FMT_THIS_ADDR "this=" << (void*)this
using boost::asio::ip::tcp;

View File

@ -18,17 +18,19 @@
#ifndef LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_
#define LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_
#include "ClientNamenodeProtocol.pb.h"
#include "rpc/rpc_engine.h"
#include "hdfspp/statinfo.h"
#include "hdfspp/fsinfo.h"
#include "hdfspp/content_summary.h"
#include "common/namenode_info.h"
#include "ClientNamenodeProtocol.pb.h"
#include "ClientNamenodeProtocol.hrpc.inl"
#include <memory>
#include <string>
#include "ClientNamenodeProtocol.hrpc.inl"
namespace hdfs {
/**

View File

@ -15,8 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "reader/block_reader.h"
#include "reader/datatransfer.h"
#include "common/continuation/continuation.h"
#include "common/continuation/asio.h"
#include "common/logging.h"

View File

@ -18,11 +18,12 @@
#ifndef BLOCK_READER_H_
#define BLOCK_READER_H_
#include "datatransfer.pb.h"
#include "hdfspp/status.h"
#include "common/async_stream.h"
#include "common/cancel_tracker.h"
#include "common/new_delete.h"
#include "datatransfer.pb.h"
#include "connection/datanodeconnection.h"
#include <memory>

View File

@ -18,6 +18,8 @@
#ifndef LIB_READER_DATA_TRANSFER_H_
#define LIB_READER_DATA_TRANSFER_H_
#include "datatransfer.pb.h"
#include "common/sasl_authenticator.h"
#include "common/async_stream.h"
#include "connection/datanodeconnection.h"

View File

@ -18,10 +18,13 @@
#ifndef LIB_READER_DATATRANFER_IMPL_H_
#define LIB_READER_DATATRANFER_IMPL_H_
#include "datatransfer.pb.h"
#include "common/continuation/continuation.h"
#include "common/continuation/asio.h"
#include "common/continuation/protobuf.h"
#include "common/sasl_authenticator.h"
#include <boost/asio/read.hpp>
#include <boost/asio/buffer.hpp>
#include <boost/asio/read.hpp>
#include <boost/asio/buffer.hpp>

View File

@ -15,14 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef READER_READER_GROUP_H_
#define READER_READER_GROUP_H_
#include "block_reader.h"
#include <memory>
#include <vector>
#include <mutex>
#include <vector>
namespace hdfs {

View File

@ -16,16 +16,17 @@
* limitations under the License.
*/
#include "RpcHeader.pb.h"
#include "ProtobufRpcEngine.pb.h"
#include "IpcConnectionContext.pb.h"
#include <functional>
#include "request.h"
#include "rpc_engine.h"
#include "sasl_protocol.h"
#include "hdfspp/ioservice.h"
#include "RpcHeader.pb.h"
#include "ProtobufRpcEngine.pb.h"
#include "IpcConnectionContext.pb.h"
#include <sstream>
namespace hdfs {

View File

@ -43,6 +43,7 @@
#include <unordered_map>
#include <boost/asio/ip/tcp.hpp>
#include <boost/system/error_code.hpp>
namespace hdfs {

View File

@ -15,15 +15,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "rpc_engine.h"
#include "rpc_connection_impl.h"
#include "sasl_protocol.h"
#include "RpcHeader.pb.h"
#include "ProtobufRpcEngine.pb.h"
#include "IpcConnectionContext.pb.h"
#include "rpc_engine.h"
#include "rpc_connection_impl.h"
#include "sasl_protocol.h"
#include <boost/asio/error.hpp>
#include <boost/system/error_code.hpp>
namespace hdfs {

View File

@ -35,6 +35,7 @@
#include <boost/date_time/posix_time/posix_time_duration.hpp>
#include <system_error>
#include <boost/system/error_code.hpp>
namespace hdfs {

View File

@ -27,6 +27,7 @@
#include <string>
#include <boost/date_time/posix_time/posix_time_duration.hpp>
#include <boost/system/error_code.hpp>
#include <openssl/rand.h>
#include <openssl/err.h>

View File

@ -16,6 +16,7 @@
* limitations under the License.
*/
#include "sasl_protocol.h"
#include "rpc_engine.h"
#include "rpc_connection.h"
#include "common/logging.h"
@ -23,7 +24,6 @@
#include "x-platform/syscall.h"
#include "sasl_engine.h"
#include "sasl_protocol.h"
#if defined USE_SASL
#if defined USE_CYRUS_SASL

View File

@ -19,12 +19,12 @@
#ifndef LIB_RPC_SASLPROTOCOL_H
#define LIB_RPC_SASLPROTOCOL_H
#include "RpcHeader.pb.h"
#include <memory>
#include <mutex>
#include <functional>
#include <RpcHeader.pb.h>
#include "hdfspp/status.h"
#include "common/auth_info.h"
#include "common/libhdfs_events_impl.h"

View File

@ -16,12 +16,13 @@
* limitations under the License.
*/
#include "reader/block_reader.h"
#include "fs/filehandle.h"
#include "common/libhdfs_events_impl.h"
#include "common/util.h"
#include "fs/filesystem.h"
#include "fs/filehandle.h"
#include "fs/bad_datanode_tracker.h"
#include "reader/block_reader.h"
#include <gtest/gtest.h>
#include <gmock/gmock.h>

View File

@ -24,6 +24,10 @@
#include <stdlib.h>
#include <string.h>
#ifdef WIN32
#define __PRETTY_FUNCTION__ __FUNCSIG__
#endif
/* Shim structs and functions that delegate to libhdfspp and libhdfs. */
struct hdfs_internal {
libhdfs_hdfsFS libhdfsRep;

View File

@ -16,8 +16,8 @@
* limitations under the License.
*/
#include <common/logging.h>
#include <bindings/c/hdfs.cc>
#include "bindings/c/hdfs.cc"
#include "common/logging.h"
#include <gmock/gmock.h>
#include <gtest/gtest.h>

View File

@ -18,6 +18,8 @@
#include "mock_connection.h"
#include <boost/asio/io_service.hpp>
namespace hdfs {
MockConnectionBase::MockConnectionBase(boost::asio::io_service *io_service)

View File

@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef LIBHDFSPP_TEST_MOCK_CONNECTION_H_
#define LIBHDFSPP_TEST_MOCK_CONNECTION_H_

View File

@ -16,13 +16,13 @@
* limitations under the License.
*/
#include "mock_connection.h"
#include "datatransfer.pb.h"
#include "common/util.h"
#include "common/cancel_tracker.h"
#include "reader/block_reader.h"
#include "reader/datatransfer.h"
#include "mock_connection.h"
#include "common/util.h"
#include "common/cancel_tracker.h"
#include "reader/fileinfo.h"
#include <google/protobuf/io/coded_stream.h>
@ -124,11 +124,11 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) {
return res;
}
static inline std::pair<error_code, string> Produce(const std::string &s) {
return make_pair(error_code(), s);
static inline std::pair<boost::system::error_code, string> Produce(const std::string &s) {
return make_pair(boost::system::error_code(), s);
}
static inline std::pair<error_code, string> ProducePacket(
static inline std::pair<boost::system::error_code, string> ProducePacket(
const std::string &data, const std::string &checksum, int offset_in_block,
int seqno, bool last_packet) {
PacketHeaderProto proto;
@ -148,7 +148,7 @@ static inline std::pair<error_code, string> ProducePacket(
proto.AppendToString(&payload);
payload += checksum;
payload += data;
return std::make_pair(error_code(), std::move(payload));
return std::make_pair(boost::system::error_code(), std::move(payload));
}
TEST(RemoteBlockReaderTest, TestReadSingleTrunk) {

View File

@ -16,11 +16,11 @@
* limitations under the License.
*/
#include "hdfspp/ioservice.h"
#include "RpcHeader.pb.h"
#include "hdfspp/ioservice.h"
#include "mock_connection.h"
#include "test.pb.h"
#include "RpcHeader.pb.h"
#include "rpc/rpc_connection_impl.h"
#include "common/namenode_info.h"
@ -41,6 +41,8 @@ using ::hadoop::common::EmptyResponseProto;
using ::hadoop::common::EchoRequestProto;
using ::hadoop::common::EchoResponseProto;
using boost::system::error_code;
using ::testing::Return;
using ::std::make_pair;