experiment with protoc pluginhavardpe/protoc-gen-csi

10 files changed, 208 insertions, 7 deletions

diff --git a/.gitignore b/.gitignore
index c1d500c3c4f..f9d1b686b03 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,3 +53,7 @@ install_manifest.txt
 /vespaConfigVersion.cmake
 *.cbp
 !/.copr/Makefile
+*.pb.h
+*.pb.cc
+*.csi.h
+*.csi.cpp
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 33cf9914a07..6c094ff3f87 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -105,13 +105,13 @@ add_subdirectory(clustercontroller-reindexer)
 add_subdirectory(clustercontroller-utils)
 add_subdirectory(config)
 add_subdirectory(config-bundle)
-add_subdirectory(configd)
-add_subdirectory(configdefinitions)
 add_subdirectory(config-model)
 add_subdirectory(config-model-api)
 add_subdirectory(config-model-fat)
 add_subdirectory(config-provisioning)
 add_subdirectory(config-proxy)
+add_subdirectory(configd)
+add_subdirectory(configdefinitions)
 add_subdirectory(configserver)
 add_subdirectory(configserver-flags)
 add_subdirectory(configutil)
@@ -138,8 +138,8 @@ add_subdirectory(fsa)
 add_subdirectory(hosted-zone-api)
 add_subdirectory(http-client)
 add_subdirectory(jdisc-cloud-aws)
-add_subdirectory(jdisc_core)
 add_subdirectory(jdisc-security-filters)
+add_subdirectory(jdisc_core)
 add_subdirectory(jrt_test)
 add_subdirectory(linguistics)
 add_subdirectory(linguistics-components)
@@ -158,6 +158,7 @@ add_subdirectory(node-repository)
 add_subdirectory(orchestrator)
 add_subdirectory(persistence)
 add_subdirectory(predicate-search)
+add_subdirectory(protoc-gen-csi)
 add_subdirectory(searchcore)
 add_subdirectory(searchlib)
 add_subdirectory(searchsummary)
@@ -175,19 +176,19 @@ add_subdirectory(vdstestlib)
 add_subdirectory(vespa-3party-bundles)
 add_subdirectory(vespa-3party-jars)
 add_subdirectory(vespa-athenz)
+add_subdirectory(vespa-feed-client)
+add_subdirectory(vespa-feed-client-cli)
+add_subdirectory(vespa-osgi-testrunner)
+add_subdirectory(vespa-testrunner-components)
 add_subdirectory(vespabase)
 add_subdirectory(vespaclient)
 add_subdirectory(vespaclient-container-plugin)
 add_subdirectory(vespaclient-core)
 add_subdirectory(vespaclient-java)
-add_subdirectory(vespa-feed-client)
-add_subdirectory(vespa-feed-client-cli)
 add_subdirectory(vespajlib)
 add_subdirectory(vespalib)
 add_subdirectory(vespalog)
 add_subdirectory(vespamalloc)
-add_subdirectory(vespa-osgi-testrunner)
-add_subdirectory(vespa-testrunner-components)
 add_subdirectory(zkfacade)
 add_subdirectory(zookeeper-command-line-client)
 add_subdirectory(zookeeper-server)
diff --git a/functions.cmake b/functions.cmake
index 63effe580de..6d2620fb0ca 100644
--- a/functions.cmake
+++ b/functions.cmake
@@ -161,6 +161,45 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH)
     vespa_add_source_target("configgen_${TARGET}_${CONFIG_NAME}" DEPENDS ${CONFIG_H_PATH} ${CONFIG_CPP_PATH})
 endfunction()
 
+function(vespa_generate_protobuf_csi TARGET RELATIVE_PROTO_PATH)
+    # Generate from RELATIVE_PROTO_PATH
+    # Destination directory is always where generate is called (or, in the case of out-of-source builds, in the build-tree parallel)
+    # This may not be the same directory as where the .proto file is located.
+
+    get_filename_component(PROTO_FILENAME ${RELATIVE_PROTO_PATH} NAME)
+    get_filename_component(PROTO_NAME ${RELATIVE_PROTO_PATH} NAME_WE)
+    get_filename_component(PROTO_DIR ${RELATIVE_PROTO_PATH} DIRECTORY)
+
+    # Config destination is the current dir (in the build tree)
+    set(PROTO_DEST_DIR ${CMAKE_CURRENT_BINARY_DIR})
+
+    set(CSI_H_PATH ${PROTO_DEST_DIR}/${PROTO_NAME}.csi.h)
+    set(CSI_CPP_PATH ${PROTO_DEST_DIR}/${PROTO_NAME}.csi.cpp)
+
+    set(CSI_GEN_FILES ${CSI_H_PATH} ${CSI_CPP_PATH})
+
+    add_custom_command(
+        OUTPUT ${CSI_GEN_FILES}
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+        COMMAND protobuf::protoc -I ${PROTO_DIR} --plugin=${PROJECT_BINARY_DIR}/protoc-gen-csi/protoc-gen-csi --csi_out=${PROTO_DEST_DIR} ${PROTO_FILENAME}
+        COMMENT "Generating Coroutine Service Interface files for ${PROTO_NAME} in ${PROTO_DEST_DIR}"
+        MAIN_DEPENDENCY ${RELATIVE_PROTO_PATH}
+        DEPENDS protoc-gen-csi
+    )
+
+    # also generate basic protobuf files
+    protobuf_generate_cpp(CSI_PROTOBUF_SRCS CSI_PROTOBUF_HDRS ${PROTO_FILENAME})
+    vespa_suppress_warnings_for_protobuf_sources(SOURCES ${CSI_PROTOBUF_SRCS})
+
+    # Add generated to sources for target
+    target_sources(${TARGET} PRIVATE ${CSI_CPP_PATH})
+
+    # Needed to be able to do a #include for this target
+    target_include_directories(${TARGET} PUBLIC ${PROTO_DEST_DIR})
+
+    vespa_add_source_target("csi_gen_${TARGET}_${PROTO_NAME}" DEPENDS ${CSI_GEN_FILES} ${CSI_PROTOBUF_SRCS} ${CSI_PROTOBUF_HDRS})
+endfunction()
+
 macro(__split_sources_list)
     unset(SOURCE_FILES)
     unset(NON_TARGET_SOURCE_FILES)
diff --git a/protoc-gen-csi/.gitignore b/protoc-gen-csi/.gitignore
new file mode 100644
index 00000000000..6726f4e0fc3
--- /dev/null
+++ b/protoc-gen-csi/.gitignore
@@ -0,0 +1 @@
+/protoc-gen-csi
diff --git a/protoc-gen-csi/CMakeLists.txt b/protoc-gen-csi/CMakeLists.txt
new file mode 100644
index 00000000000..17977adb62e
--- /dev/null
+++ b/protoc-gen-csi/CMakeLists.txt
@@ -0,0 +1,7 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(protoc-gen-csi
+  SOURCES
+  protoc_gen_csi.cpp
+  EXTERNAL_DEPENDS
+  protoc
+)
diff --git a/protoc-gen-csi/protoc_gen_csi.cpp b/protoc-gen-csi/protoc_gen_csi.cpp
new file mode 100644
index 00000000000..fa9c3e089ca
--- /dev/null
+++ b/protoc-gen-csi/protoc_gen_csi.cpp
@@ -0,0 +1,114 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <google/protobuf/compiler/plugin.h>
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+#include <google/protobuf/descriptor.h>
+
+using namespace google::protobuf;
+using namespace google::protobuf::compiler;
+
+class MyGen : public CodeGenerator {
+public:
+    bool Generate(const FileDescriptor * file, const std::string & parameter,
+                  GeneratorContext * generator_context, std::string * error) const override;
+    ~MyGen();
+};
+
+MyGen::~MyGen() = default;
+
+int main(int argc, char* argv[]) {
+    MyGen generator;
+    return PluginMain(argc, argv, &generator);
+}
+
+void write_line(io::ZeroCopyOutputStream *target, const std::string &line) {
+    void *data = nullptr;
+    const char *src = line.c_str();
+    int left = line.size() + 1;
+    while (left > 0) {
+        int size = left;
+        if (target->Next(&data, &size)) {
+            if (size == 0) continue;
+            if (size >= left) {
+                memcpy(data, src, left);
+                char * buf = static_cast<char *>(data);
+                buf[left - 1] = '\n';
+                if (size > left) {
+                    target->BackUp(size - left);
+                }
+                return;
+            } else {
+                memcpy(data, src, size);
+                left -= size;
+                src += size;
+            }
+        } else {
+            perror("target->Next() returned false");
+            std::string message = "Error writing output: ";
+            message += strerror(errno);
+            throw message;
+        }
+    }
+}
+
+void my_generate(const std::string &name,
+                 const FileDescriptor &file,
+                 GeneratorContext &context)
+{
+    if (file.dependency_count() > 0
+        || file.public_dependency_count() > 0
+        || file.weak_dependency_count() > 0)
+    {
+        std::string message = "Importing dependencies not supported";
+        throw message;
+    }
+    if (file.extension_count() > 0) {
+        std::string message = "Extensions not supported";
+        throw message;
+    }
+    if (file.is_placeholder()) {
+        std::string message = "Unexpected placeholder file";
+        throw message;
+    }
+    auto filename_csi_h = name + ".csi.h";
+    auto csi_h = context.Open(filename_csi_h);
+    write_line(csi_h, "// DO NOT EDIT (generated by protoc-gen-csi)");
+    write_line(csi_h, "// Coroutine Service Interface: " + name);
+    write_line(csi_h, "#pragma once");
+
+    auto filename_csi_cpp = name + ".csi.cpp";
+    auto csi_cpp = context.Open(filename_csi_cpp);
+    write_line(csi_cpp, "// DO NOT EDIT (generated by protoc-gen-csi)");
+    write_line(csi_cpp, "// Coroutine Service Interface: " + name);
+    write_line(csi_cpp, "#include \"" + filename_csi_h + "\"");
+}
+
+bool MyGen::Generate(const FileDescriptor * file, const std::string & parameter, 
+                     GeneratorContext * generator_context, std::string * error) const
+{
+    std::string name = "[unknown]";
+    try {
+        if (file == nullptr) {
+            std::string m = "No FileDescriptor";
+            throw m;
+        }
+        name = file->name();
+        if (name.ends_with(".proto")) {
+            name = name.substr(0, name.size() - 6);
+        }
+        if (generator_context == nullptr) {
+            std::string m = "No GeneratorContext";
+            throw m;
+        }
+        if (! parameter.empty()) {
+            std::string m = "unknown command line parameter " + parameter;
+            throw m;
+        }
+        my_generate(name, *file, *generator_context);
+    } catch (const std::string &message) {
+        *error = name + ": " + message;
+        return false;
+    }
+    return true;
+}
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index 83711ab8200..f60f1463980 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -48,6 +48,7 @@ vespa_define_module(
     src/tests/compression
     src/tests/coro/active_work
     src/tests/coro/async_io
+    src/tests/coro/csi
     src/tests/coro/detached
     src/tests/coro/generator
     src/tests/coro/lazy
diff --git a/vespalib/src/tests/coro/csi/CMakeLists.txt b/vespalib/src/tests/coro/csi/CMakeLists.txt
new file mode 100644
index 00000000000..5987428fe65
--- /dev/null
+++ b/vespalib/src/tests/coro/csi/CMakeLists.txt
@@ -0,0 +1,11 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+vespa_add_executable(vespalib_csi_test_app TEST
+    SOURCES
+    csi_test.cpp
+    DEPENDS
+    vespalib
+    GTest::GTest
+)
+vespa_generate_protobuf_csi(vespalib_csi_test_app ./test.proto)
+vespa_add_test(NAME vespalib_csi_test_app COMMAND vespalib_csi_test_app)
diff --git a/vespalib/src/tests/coro/csi/csi_test.cpp b/vespalib/src/tests/coro/csi/csi_test.cpp
new file mode 100644
index 00000000000..301c12b2cec
--- /dev/null
+++ b/vespalib/src/tests/coro/csi/csi_test.cpp
@@ -0,0 +1,8 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <vespa/vespalib/gtest/gtest.h>
+
+TEST(CoroRpcTest, dummy_test) {
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/vespalib/src/tests/coro/csi/test.proto b/vespalib/src/tests/coro/csi/test.proto
new file mode 100644
index 00000000000..3e53ade7cfb
--- /dev/null
+++ b/vespalib/src/tests/coro/csi/test.proto
@@ -0,0 +1,15 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+syntax = "proto3";
+
+message TestRequest {
+    string name = 1;
+}
+
+message TestReply {
+    string name = 1;
+}
+
+service TestService {
+    rpc my_test (TestRequest) returns (TestReply) {}
+}