HDFS-16467. Ensure Protobuf generated headers are included first (#4601)

* This PR ensures that the Protobuf generated headers
  are always included first, even when these headers
  are included transitively.
* This problem is specific to Windows only.

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_mini_stress.c

@@ -18,24 +18,27 @@
 
 #include "common/util_c.h"
 #include "expect.h"
-#include "hdfs/hdfs.h"
 #include "hdfspp/hdfs_ext.h"
 #include "native_mini_dfs.h"
 #include "os/thread.h"
 #include "x-platform/c-api/syscall.h"
+#include "hdfs/hdfs.h"
 
 #include <errno.h>
 #include <inttypes.h>
-#include <pwd.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/socket.h>
 #include <sys/types.h>
-#include <sys/wait.h>
 #include <unistd.h>
 
+#ifndef WIN32
+#include <sys/socket.h>
+#include <sys/wait.h>
+#include <pwd.h>
+#endif
+
 #define TO_STR_HELPER(X) #X
 #define TO_STR(X) TO_STR_HELPER(X)
 
@@ -197,7 +200,7 @@ static int fileEventCallback1(const char * event, const char * cluster, const ch
   if (randomErrRatioStr) randomErrRatio = (int64_t)atoi(randomErrRatioStr);
   if (randomErrRatio == 0) return DEBUG_SIMULATE_ERROR;
   else if (randomErrRatio < 0) return LIBHDFSPP_EVENT_OK;
-  return random() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
+  return rand() % randomErrRatio == 0 ? DEBUG_SIMULATE_ERROR : LIBHDFSPP_EVENT_OK;
 }
 
 static int fileEventCallback2(const char * event, const char * cluster, const char * file, int64_t value, int64_t cookie)
@@ -235,7 +238,7 @@ static int doTestHdfsMiniStress(struct tlhThreadInfo *ti, int randomErr)
     EXPECT_ZERO(hdfsCloseFile(ti->hdfs, file));
     file = hdfsOpenFile(ti->hdfs, ti->fileNm, O_RDONLY, 0, 0, 0);
     EXPECT_NONNULL(file);
-    seekPos = (((double)random()) / RAND_MAX) * (fileInfo->mSize - expected);
+    seekPos = (((double)rand()) / RAND_MAX) * (fileInfo->mSize - expected);
     seekPos = (seekPos / expected) * expected;
     ret = hdfsSeek(ti->hdfs, file, seekPos);
     if (ret < 0) {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/os/windows/unistd.h

@@ -22,7 +22,7 @@
 /* On Windows, unistd.h does not exist, so manually define what we need. */
 
 #include <process.h> /* Declares getpid(). */
-#include <windows.h>
+#include <Windows.h>
 
 /* Re-route sleep to Sleep, converting units from seconds to milliseconds. */
 #define sleep(seconds) Sleep((seconds) * 1000)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/hdfs.cc

@@ -16,6 +16,8 @@
  * limitations under the License.
  */
 
+#include "fs/filehandle.h"
+
 #include "hdfspp/hdfspp.h"
 #include "hdfspp/hdfs_ext.h"
 
@@ -23,7 +25,6 @@
 #include "common/configuration_loader.h"
 #include "common/logging.h"
 #include "fs/filesystem.h"
-#include "fs/filehandle.h"
 #include "x-platform/utils.h"
 #include "x-platform/syscall.h"
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/fsinfo.cc

@@ -21,6 +21,7 @@
 #include <algorithm>
 #include <sstream>
 #include <iomanip>
+#include <algorithm>
 
 namespace hdfs {
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/logging.h

@@ -19,10 +19,10 @@
 #ifndef LIB_COMMON_LOGGING_H_
 #define LIB_COMMON_LOGGING_H_
 
-#include <boost/asio/ip/tcp.hpp>
-
 #include "hdfspp/log.h"
 
+#include <boost/asio/ip/tcp.hpp>
+
 #include <sstream>
 #include <mutex>
 #include <memory>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/namenode_info.cc

@@ -26,6 +26,7 @@
 #include <utility>
 #include <future>
 
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/common/util.cc

@@ -19,6 +19,7 @@
 #include "common/util.h"
 #include "common/util_c.h"
 
+#include <boost/system/error_code.hpp>
 #include <google/protobuf/message_lite.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/connection/datanodeconnection.h

@@ -18,9 +18,10 @@
 #ifndef LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
 #define LIBHDFSPP_LIB_CONNECTION_DATANODECONNECTION_H_
 
+#include "ClientNamenodeProtocol.pb.h"
+
 #include "hdfspp/ioservice.h"
 #include "common/async_stream.h"
-#include "ClientNamenodeProtocol.pb.h"
 #include "common/libhdfs_events_impl.h"
 #include "common/logging.h"
 #include "common/util.h"

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.cc

@@ -16,11 +16,12 @@
  * limitations under the License.
  */
 
+#include "reader/block_reader.h"
 #include "filehandle.h"
+
 #include "common/continuation/continuation.h"
 #include "common/logging.h"
 #include "connection/datanodeconnection.h"
-#include "reader/block_reader.h"
 #include "hdfspp/events.h"
 #include "x-platform/types.h"
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filehandle.h

@@ -18,13 +18,14 @@
 #ifndef LIBHDFSPP_LIB_FS_FILEHANDLE_H_
 #define LIBHDFSPP_LIB_FS_FILEHANDLE_H_
 
+#include "reader/readergroup.h"
+
 #include "hdfspp/ioservice.h"
 #include "common/async_stream.h"
 #include "common/cancel_tracker.h"
 #include "common/libhdfs_events_impl.h"
 #include "common/new_delete.h"
 #include "reader/fileinfo.h"
-#include "reader/readergroup.h"
 
 #include "bad_datanode_tracker.h"
 #include "ClientNamenodeProtocol.pb.h"

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc

@@ -16,21 +16,24 @@
  * limitations under the License.
  */
 
-#include "filesystem.h"
-
 #include "filehandle.h"
+#include "filesystem.h"
 #include "common/namenode_info.h"
 
 #include <functional>
 #include <limits>
 #include <future>
 #include <tuple>
-#include <pwd.h>
+#include <iostream>
 
 #include <boost/asio/ip/tcp.hpp>
 
 #include "x-platform/syscall.h"
 
+#ifndef WIN32
+#include <pwd.h>
+#endif
+
 #define FMT_THIS_ADDR "this=" << (void*)this
 
 namespace hdfs {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.cc

@@ -26,9 +26,12 @@
 #include <future>
 #include <tuple>
 #include <iostream>
-#include <pwd.h>
 #include <utility>
 
+#ifndef WIN32
+#include <pwd.h>
+#endif
+
 #define FMT_THIS_ADDR "this=" << (void*)this
 
 using boost::asio::ip::tcp;

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/namenode_operations.h

@@ -18,17 +18,19 @@
 #ifndef LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_
 #define LIBHDFSPP_LIB_FS_NAMENODEOPERATIONS_H_
 
+#include "ClientNamenodeProtocol.pb.h"
+
 #include "rpc/rpc_engine.h"
 #include "hdfspp/statinfo.h"
 #include "hdfspp/fsinfo.h"
 #include "hdfspp/content_summary.h"
 #include "common/namenode_info.h"
-#include "ClientNamenodeProtocol.pb.h"
-#include "ClientNamenodeProtocol.hrpc.inl"
 
 #include <memory>
 #include <string>
 
+#include "ClientNamenodeProtocol.hrpc.inl"
+
 namespace hdfs {
 
 /**

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.cc

@@ -15,8 +15,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #include "reader/block_reader.h"
 #include "reader/datatransfer.h"
+
 #include "common/continuation/continuation.h"
 #include "common/continuation/asio.h"
 #include "common/logging.h"

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/block_reader.h

@@ -18,11 +18,12 @@
 #ifndef BLOCK_READER_H_
 #define BLOCK_READER_H_
 
+#include "datatransfer.pb.h"
+
 #include "hdfspp/status.h"
 #include "common/async_stream.h"
 #include "common/cancel_tracker.h"
 #include "common/new_delete.h"
-#include "datatransfer.pb.h"
 #include "connection/datanodeconnection.h"
 
 #include <memory>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer.h

@@ -18,6 +18,8 @@
 #ifndef LIB_READER_DATA_TRANSFER_H_
 #define LIB_READER_DATA_TRANSFER_H_
 
+#include "datatransfer.pb.h"
+
 #include "common/sasl_authenticator.h"
 #include "common/async_stream.h"
 #include "connection/datanodeconnection.h"

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/datatransfer_impl.h

@@ -18,10 +18,13 @@
 #ifndef LIB_READER_DATATRANFER_IMPL_H_
 #define LIB_READER_DATATRANFER_IMPL_H_
 
-#include "datatransfer.pb.h"
 #include "common/continuation/continuation.h"
 #include "common/continuation/asio.h"
 #include "common/continuation/protobuf.h"
+#include "common/sasl_authenticator.h"
+
+#include <boost/asio/read.hpp>
+#include <boost/asio/buffer.hpp>
 
 #include <boost/asio/read.hpp>
 #include <boost/asio/buffer.hpp>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/reader/readergroup.h

@@ -15,14 +15,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #ifndef READER_READER_GROUP_H_
 #define READER_READER_GROUP_H_
 
 #include "block_reader.h"
 
 #include <memory>
-#include <vector>
 #include <mutex>
+#include <vector>
 
 namespace hdfs {
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/request.cc

@@ -16,16 +16,17 @@
  * limitations under the License.
  */
 
+#include "RpcHeader.pb.h"
+#include "ProtobufRpcEngine.pb.h"
+#include "IpcConnectionContext.pb.h"
+
 #include <functional>
+
 #include "request.h"
 #include "rpc_engine.h"
 #include "sasl_protocol.h"
 #include "hdfspp/ioservice.h"
 
-#include "RpcHeader.pb.h"
-#include "ProtobufRpcEngine.pb.h"
-#include "IpcConnectionContext.pb.h"
-
 #include <sstream>
 
 namespace hdfs {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection.h

@@ -43,6 +43,7 @@
 #include <unordered_map>
 
 #include <boost/asio/ip/tcp.hpp>
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.cc

@@ -15,15 +15,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-#include "rpc_engine.h"
-#include "rpc_connection_impl.h"
-#include "sasl_protocol.h"
 
 #include "RpcHeader.pb.h"
 #include "ProtobufRpcEngine.pb.h"
 #include "IpcConnectionContext.pb.h"
 
+#include "rpc_engine.h"
+#include "rpc_connection_impl.h"
+#include "sasl_protocol.h"
+
 #include <boost/asio/error.hpp>
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_connection_impl.h

@@ -35,6 +35,7 @@
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
 
 #include <system_error>
+#include <boost/system/error_code.hpp>
 
 namespace hdfs {
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/rpc_engine.cc

@@ -27,6 +27,7 @@
 #include <string>
 
 #include <boost/date_time/posix_time/posix_time_duration.hpp>
+#include <boost/system/error_code.hpp>
 #include <openssl/rand.h>
 #include <openssl/err.h>
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.cc

@@ -16,6 +16,7 @@
  * limitations under the License.
  */
 
+#include "sasl_protocol.h"
 #include "rpc_engine.h"
 #include "rpc_connection.h"
 #include "common/logging.h"
@@ -23,7 +24,6 @@
 #include "x-platform/syscall.h"
 
 #include "sasl_engine.h"
-#include "sasl_protocol.h"
 
 #if defined USE_SASL
   #if defined USE_CYRUS_SASL

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/rpc/sasl_protocol.h

@@ -19,12 +19,12 @@
 #ifndef LIB_RPC_SASLPROTOCOL_H
 #define LIB_RPC_SASLPROTOCOL_H
 
+#include "RpcHeader.pb.h"
+
 #include <memory>
 #include <mutex>
 #include <functional>
 
-#include <RpcHeader.pb.h>
-
 #include "hdfspp/status.h"
 #include "common/auth_info.h"
 #include "common/libhdfs_events_impl.h"

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/bad_datanode_test.cc

@@ -16,12 +16,13 @@
  * limitations under the License.
  */
 
+#include "reader/block_reader.h"
+#include "fs/filehandle.h"
+
 #include "common/libhdfs_events_impl.h"
 #include "common/util.h"
 #include "fs/filesystem.h"
-#include "fs/filehandle.h"
 #include "fs/bad_datanode_tracker.h"
-#include "reader/block_reader.h"
 
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c

@@ -24,6 +24,10 @@
 #include <stdlib.h>
 #include <string.h>
 
+#ifdef WIN32
+#define __PRETTY_FUNCTION__ __FUNCSIG__
+#endif
+
 /* Shim structs and functions that delegate to libhdfspp and libhdfs. */
 struct hdfs_internal {
   libhdfs_hdfsFS libhdfsRep;

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/logging_test.cc

@@ -16,8 +16,8 @@
  * limitations under the License.
  */
 
-#include <common/logging.h>
+#include "bindings/c/hdfs.cc"
-#include <bindings/c/hdfs.cc>
+#include "common/logging.h"
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.cc

@@ -18,6 +18,8 @@
 
 #include "mock_connection.h"
 
+#include <boost/asio/io_service.hpp>
+
 namespace hdfs {
 
 MockConnectionBase::MockConnectionBase(boost::asio::io_service *io_service)

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/mock_connection.h

@@ -15,6 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
 #ifndef LIBHDFSPP_TEST_MOCK_CONNECTION_H_
 #define LIBHDFSPP_TEST_MOCK_CONNECTION_H_
 

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/remote_block_reader_test.cc

@@ -16,13 +16,13 @@
  * limitations under the License.
  */
 
-#include "mock_connection.h"
-
-#include "datatransfer.pb.h"
-#include "common/util.h"
-#include "common/cancel_tracker.h"
 #include "reader/block_reader.h"
 #include "reader/datatransfer.h"
+
+#include "mock_connection.h"
+
+#include "common/util.h"
+#include "common/cancel_tracker.h"
 #include "reader/fileinfo.h"
 
 #include <google/protobuf/io/coded_stream.h>
@@ -124,11 +124,11 @@ static inline string ToDelimitedString(const pb::MessageLite *msg) {
   return res;
 }
 
-static inline std::pair<error_code, string> Produce(const std::string &s) {
+static inline std::pair<boost::system::error_code, string> Produce(const std::string &s) {
-  return make_pair(error_code(), s);
+  return make_pair(boost::system::error_code(), s);
 }
 
-static inline std::pair<error_code, string> ProducePacket(
+static inline std::pair<boost::system::error_code, string> ProducePacket(
     const std::string &data, const std::string &checksum, int offset_in_block,
     int seqno, bool last_packet) {
   PacketHeaderProto proto;
@@ -148,7 +148,7 @@ static inline std::pair<error_code, string> ProducePacket(
   proto.AppendToString(&payload);
   payload += checksum;
   payload += data;
-  return std::make_pair(error_code(), std::move(payload));
+  return std::make_pair(boost::system::error_code(), std::move(payload));
 }
 
 TEST(RemoteBlockReaderTest, TestReadSingleTrunk) {

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/rpc_engine_test.cc

@@ -16,11 +16,11 @@
  * limitations under the License.
  */
 
-#include "hdfspp/ioservice.h"
+#include "RpcHeader.pb.h"
 
+#include "hdfspp/ioservice.h"
 #include "mock_connection.h"
 #include "test.pb.h"
-#include "RpcHeader.pb.h"
 #include "rpc/rpc_connection_impl.h"
 #include "common/namenode_info.h"
 
@@ -41,6 +41,8 @@ using ::hadoop::common::EmptyResponseProto;
 using ::hadoop::common::EchoRequestProto;
 using ::hadoop::common::EchoResponseProto;
 
+using boost::system::error_code;
+
 using ::testing::Return;
 
 using ::std::make_pair;