HDFS-8764. Generate Hadoop RPC stubs from protobuf definitions. Contributed by Haohui Mai.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt

@@ -18,4 +18,48 @@ protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS
   ${COMMON_PROTO_DIR}/Security.proto
 )
 
-add_library(proto ${PROTO_SRCS} ${PROTO_HDRS})
+add_executable(protoc-gen-hrpc protoc_gen_hrpc.cc)
+target_link_libraries(protoc-gen-hrpc ${PROTOBUF_PROTOC_LIBRARY} ${PROTOBUF_LIBRARY})
+
+function(GEN_HRPC SRCS)
+  if(NOT ARGN)
+    message(SEND_ERROR "Error: GEN_HRPC() called without any proto files")
+    return()
+  endif()
+
+  if(DEFINED PROTOBUF_IMPORT_DIRS)
+    foreach(DIR ${PROTOBUF_IMPORT_DIRS})
+      get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
+      list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
+      if(${_contains_already} EQUAL -1)
+          list(APPEND _protobuf_include_path -I ${ABS_PATH})
+      endif()
+    endforeach()
+  endif()
+
+  set(${SRCS})
+
+  foreach(FIL ${ARGN})
+    get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.hrpc.inl"
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+      ARGS --plugin=protoc-gen-hrpc=${CMAKE_CURRENT_BINARY_DIR}/protoc-gen-hrpc --hrpc_out=${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+      DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE} protoc-gen-hrpc
+      COMMENT "Running HRPC protocol buffer compiler on ${FIL}"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+endfunction()
+
+gen_hrpc(HRPC_SRCS
+  ${CLIENT_PROTO_DIR}/ClientNamenodeProtocol.proto
+)
+
+add_library(proto ${PROTO_SRCS} ${PROTO_HDRS} ${HRPC_SRCS})

hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/cpp_helpers.h

@@ -0,0 +1,82 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Author: kenton@google.com (Kenton Varda)
+//  Based on original Protocol Buffers design by
+//  Sanjay Ghemawat, Jeff Dean, and others.
+
+#ifndef LIBHDFSPP_PROTO_CPP_HELPERS_H_
+#define LIBHDFSPP_PROTO_CPP_HELPERS_H_
+
+#include <string>
+
+/**
+ * The functions in this file are derived from the original implementation of
+ *the protobuf library from Google.
+ **/
+
+static inline std::string StripProto(const std::string &str) {
+  static const std::string kExtension = ".proto";
+  if (str.size() >= kExtension.size() &&
+      str.compare(str.size() - kExtension.size(), kExtension.size(),
+                  kExtension) == 0) {
+    return str.substr(0, str.size() - kExtension.size());
+  } else {
+    return str;
+  }
+}
+
+static inline std::string ToCamelCase(const std::string &input) {
+  bool cap_next_letter = true;
+  std::string result;
+  // Note:  I distrust ctype.h due to locales.
+  for (size_t i = 0; i < input.size(); i++) {
+    if ('a' <= input[i] && input[i] <= 'z') {
+      if (cap_next_letter) {
+        result += input[i] + ('A' - 'a');
+      } else {
+        result += input[i];
+      }
+      cap_next_letter = false;
+    } else if ('A' <= input[i] && input[i] <= 'Z') {
+      // Capital letters are left as-is.
+      result += input[i];
+      cap_next_letter = false;
+    } else if ('0' <= input[i] && input[i] <= '9') {
+      result += input[i];
+      cap_next_letter = true;
+    } else {
+      cap_next_letter = true;
+    }
+  }
+  return result;
+}
+
+#endif

hadoop-hdfs-project/hadoop-hdfs-client/src/main/native/libhdfspp/lib/proto/protoc_gen_hrpc.cc

@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "cpp_helpers.h"
+
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/compiler/plugin.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/io/printer.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/stubs/common.h>
+
+using ::google::protobuf::FileDescriptor;
+using ::google::protobuf::MethodDescriptor;
+using ::google::protobuf::ServiceDescriptor;
+using ::google::protobuf::compiler::CodeGenerator;
+using ::google::protobuf::compiler::GeneratorContext;
+using ::google::protobuf::io::Printer;
+using ::google::protobuf::io::ZeroCopyOutputStream;
+
+class StubGenerator : public CodeGenerator {
+public:
+  virtual bool Generate(const FileDescriptor *file, const std::string &,
+                        GeneratorContext *ctx,
+                        std::string *error) const override;
+
+private:
+  void EmitService(const ServiceDescriptor *service, Printer *out) const;
+  void EmitMethod(const MethodDescriptor *method, Printer *out) const;
+};
+
+bool StubGenerator::Generate(const FileDescriptor *file, const std::string &,
+                             GeneratorContext *ctx, std::string *) const {
+  namespace pb = ::google::protobuf;
+  std::unique_ptr<ZeroCopyOutputStream> os(
+      ctx->Open(StripProto(file->name()) + ".hrpc.inl"));
+  Printer out(os.get(), '$');
+  for (int i = 0; i < file->service_count(); ++i) {
+    const ServiceDescriptor *service = file->service(i);
+    EmitService(service, &out);
+  }
+  return true;
+}
+
+void StubGenerator::EmitService(const ServiceDescriptor *service,
+                                Printer *out) const {
+  out->Print("\n// GENERATED AUTOMATICALLY. DO NOT MODIFY.\n"
+             "class $service$ {\n"
+             "private:\n"
+             "  ::hdfs::RpcEngine *const engine_;\n"
+             "public:\n"
+             "  typedef std::function<void(const ::hdfs::Status &)> Callback;\n"
+             "  typedef ::google::protobuf::MessageLite Message;\n"
+             "  inline $service$(::hdfs::RpcEngine *engine)\n"
+             "    : engine_(engine) {}\n",
+             "service", service->name());
+  for (int i = 0; i < service->method_count(); ++i) {
+    const MethodDescriptor *method = service->method(i);
+    EmitMethod(method, out);
+  }
+  out->Print("};\n");
+}
+
+void StubGenerator::EmitMethod(const MethodDescriptor *method,
+                               Printer *out) const {
+  out->Print(
+      "\n  inline void $camel_method$(const Message *req, "
+      "const std::shared_ptr<Message> &resp, "
+      "Callback &&handler) {\n"
+      "    engine_->AsyncRpc(\"$method$\", req, resp, std::move(handler));\n"
+      "  }\n",
+      "camel_method", ToCamelCase(method->name()), "method", method->name());
+}
+
+int main(int argc, char *argv[]) {
+  StubGenerator generator;
+  return google::protobuf::compiler::PluginMain(argc, argv, &generator);
+}