From 094838356b8e127fe708d6bb7940f540efb2107e Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Mon, 20 Mar 2023 12:53:31 +0000 Subject: experiment with protoc plugin --- .gitignore | 4 + CMakeLists.txt | 15 ++-- functions.cmake | 39 ++++++++++ protoc-gen-csi/.gitignore | 1 + protoc-gen-csi/CMakeLists.txt | 7 ++ protoc-gen-csi/protoc_gen_csi.cpp | 114 +++++++++++++++++++++++++++++ vespalib/CMakeLists.txt | 1 + vespalib/src/tests/coro/csi/CMakeLists.txt | 11 +++ vespalib/src/tests/coro/csi/csi_test.cpp | 8 ++ vespalib/src/tests/coro/csi/test.proto | 15 ++++ 10 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 protoc-gen-csi/.gitignore create mode 100644 protoc-gen-csi/CMakeLists.txt create mode 100644 protoc-gen-csi/protoc_gen_csi.cpp create mode 100644 vespalib/src/tests/coro/csi/CMakeLists.txt create mode 100644 vespalib/src/tests/coro/csi/csi_test.cpp create mode 100644 vespalib/src/tests/coro/csi/test.proto diff --git a/.gitignore b/.gitignore index c1d500c3c4f..f9d1b686b03 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ install_manifest.txt /vespaConfigVersion.cmake *.cbp !/.copr/Makefile +*.pb.h +*.pb.cc +*.csi.h +*.csi.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 33cf9914a07..6c094ff3f87 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -105,13 +105,13 @@ add_subdirectory(clustercontroller-reindexer) add_subdirectory(clustercontroller-utils) add_subdirectory(config) add_subdirectory(config-bundle) -add_subdirectory(configd) -add_subdirectory(configdefinitions) add_subdirectory(config-model) add_subdirectory(config-model-api) add_subdirectory(config-model-fat) add_subdirectory(config-provisioning) add_subdirectory(config-proxy) +add_subdirectory(configd) +add_subdirectory(configdefinitions) add_subdirectory(configserver) add_subdirectory(configserver-flags) add_subdirectory(configutil) @@ -138,8 +138,8 @@ add_subdirectory(fsa) add_subdirectory(hosted-zone-api) add_subdirectory(http-client) add_subdirectory(jdisc-cloud-aws) -add_subdirectory(jdisc_core) add_subdirectory(jdisc-security-filters) +add_subdirectory(jdisc_core) add_subdirectory(jrt_test) add_subdirectory(linguistics) add_subdirectory(linguistics-components) @@ -158,6 +158,7 @@ add_subdirectory(node-repository) add_subdirectory(orchestrator) add_subdirectory(persistence) add_subdirectory(predicate-search) +add_subdirectory(protoc-gen-csi) add_subdirectory(searchcore) add_subdirectory(searchlib) add_subdirectory(searchsummary) @@ -175,19 +176,19 @@ add_subdirectory(vdstestlib) add_subdirectory(vespa-3party-bundles) add_subdirectory(vespa-3party-jars) add_subdirectory(vespa-athenz) +add_subdirectory(vespa-feed-client) +add_subdirectory(vespa-feed-client-cli) +add_subdirectory(vespa-osgi-testrunner) +add_subdirectory(vespa-testrunner-components) add_subdirectory(vespabase) add_subdirectory(vespaclient) add_subdirectory(vespaclient-container-plugin) add_subdirectory(vespaclient-core) add_subdirectory(vespaclient-java) -add_subdirectory(vespa-feed-client) -add_subdirectory(vespa-feed-client-cli) add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) add_subdirectory(vespamalloc) -add_subdirectory(vespa-osgi-testrunner) -add_subdirectory(vespa-testrunner-components) add_subdirectory(zkfacade) add_subdirectory(zookeeper-command-line-client) add_subdirectory(zookeeper-server) diff --git a/functions.cmake b/functions.cmake index 63effe580de..6d2620fb0ca 100644 --- a/functions.cmake +++ b/functions.cmake @@ -161,6 +161,45 @@ function(vespa_generate_config TARGET RELATIVE_CONFIG_DEF_PATH) vespa_add_source_target("configgen_${TARGET}_${CONFIG_NAME}" DEPENDS ${CONFIG_H_PATH} ${CONFIG_CPP_PATH}) endfunction() +function(vespa_generate_protobuf_csi TARGET RELATIVE_PROTO_PATH) + # Generate from RELATIVE_PROTO_PATH + # Destination directory is always where generate is called (or, in the case of out-of-source builds, in the build-tree parallel) + # This may not be the same directory as where the .proto file is located. + + get_filename_component(PROTO_FILENAME ${RELATIVE_PROTO_PATH} NAME) + get_filename_component(PROTO_NAME ${RELATIVE_PROTO_PATH} NAME_WE) + get_filename_component(PROTO_DIR ${RELATIVE_PROTO_PATH} DIRECTORY) + + # Config destination is the current dir (in the build tree) + set(PROTO_DEST_DIR ${CMAKE_CURRENT_BINARY_DIR}) + + set(CSI_H_PATH ${PROTO_DEST_DIR}/${PROTO_NAME}.csi.h) + set(CSI_CPP_PATH ${PROTO_DEST_DIR}/${PROTO_NAME}.csi.cpp) + + set(CSI_GEN_FILES ${CSI_H_PATH} ${CSI_CPP_PATH}) + + add_custom_command( + OUTPUT ${CSI_GEN_FILES} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND protobuf::protoc -I ${PROTO_DIR} --plugin=${PROJECT_BINARY_DIR}/protoc-gen-csi/protoc-gen-csi --csi_out=${PROTO_DEST_DIR} ${PROTO_FILENAME} + COMMENT "Generating Coroutine Service Interface files for ${PROTO_NAME} in ${PROTO_DEST_DIR}" + MAIN_DEPENDENCY ${RELATIVE_PROTO_PATH} + DEPENDS protoc-gen-csi + ) + + # also generate basic protobuf files + protobuf_generate_cpp(CSI_PROTOBUF_SRCS CSI_PROTOBUF_HDRS ${PROTO_FILENAME}) + vespa_suppress_warnings_for_protobuf_sources(SOURCES ${CSI_PROTOBUF_SRCS}) + + # Add generated to sources for target + target_sources(${TARGET} PRIVATE ${CSI_CPP_PATH}) + + # Needed to be able to do a #include for this target + target_include_directories(${TARGET} PUBLIC ${PROTO_DEST_DIR}) + + vespa_add_source_target("csi_gen_${TARGET}_${PROTO_NAME}" DEPENDS ${CSI_GEN_FILES} ${CSI_PROTOBUF_SRCS} ${CSI_PROTOBUF_HDRS}) +endfunction() + macro(__split_sources_list) unset(SOURCE_FILES) unset(NON_TARGET_SOURCE_FILES) diff --git a/protoc-gen-csi/.gitignore b/protoc-gen-csi/.gitignore new file mode 100644 index 00000000000..6726f4e0fc3 --- /dev/null +++ b/protoc-gen-csi/.gitignore @@ -0,0 +1 @@ +/protoc-gen-csi diff --git a/protoc-gen-csi/CMakeLists.txt b/protoc-gen-csi/CMakeLists.txt new file mode 100644 index 00000000000..17977adb62e --- /dev/null +++ b/protoc-gen-csi/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(protoc-gen-csi + SOURCES + protoc_gen_csi.cpp + EXTERNAL_DEPENDS + protoc +) diff --git a/protoc-gen-csi/protoc_gen_csi.cpp b/protoc-gen-csi/protoc_gen_csi.cpp new file mode 100644 index 00000000000..fa9c3e089ca --- /dev/null +++ b/protoc-gen-csi/protoc_gen_csi.cpp @@ -0,0 +1,114 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include + +using namespace google::protobuf; +using namespace google::protobuf::compiler; + +class MyGen : public CodeGenerator { +public: + bool Generate(const FileDescriptor * file, const std::string & parameter, + GeneratorContext * generator_context, std::string * error) const override; + ~MyGen(); +}; + +MyGen::~MyGen() = default; + +int main(int argc, char* argv[]) { + MyGen generator; + return PluginMain(argc, argv, &generator); +} + +void write_line(io::ZeroCopyOutputStream *target, const std::string &line) { + void *data = nullptr; + const char *src = line.c_str(); + int left = line.size() + 1; + while (left > 0) { + int size = left; + if (target->Next(&data, &size)) { + if (size == 0) continue; + if (size >= left) { + memcpy(data, src, left); + char * buf = static_cast(data); + buf[left - 1] = '\n'; + if (size > left) { + target->BackUp(size - left); + } + return; + } else { + memcpy(data, src, size); + left -= size; + src += size; + } + } else { + perror("target->Next() returned false"); + std::string message = "Error writing output: "; + message += strerror(errno); + throw message; + } + } +} + +void my_generate(const std::string &name, + const FileDescriptor &file, + GeneratorContext &context) +{ + if (file.dependency_count() > 0 + || file.public_dependency_count() > 0 + || file.weak_dependency_count() > 0) + { + std::string message = "Importing dependencies not supported"; + throw message; + } + if (file.extension_count() > 0) { + std::string message = "Extensions not supported"; + throw message; + } + if (file.is_placeholder()) { + std::string message = "Unexpected placeholder file"; + throw message; + } + auto filename_csi_h = name + ".csi.h"; + auto csi_h = context.Open(filename_csi_h); + write_line(csi_h, "// DO NOT EDIT (generated by protoc-gen-csi)"); + write_line(csi_h, "// Coroutine Service Interface: " + name); + write_line(csi_h, "#pragma once"); + + auto filename_csi_cpp = name + ".csi.cpp"; + auto csi_cpp = context.Open(filename_csi_cpp); + write_line(csi_cpp, "// DO NOT EDIT (generated by protoc-gen-csi)"); + write_line(csi_cpp, "// Coroutine Service Interface: " + name); + write_line(csi_cpp, "#include \"" + filename_csi_h + "\""); +} + +bool MyGen::Generate(const FileDescriptor * file, const std::string & parameter, + GeneratorContext * generator_context, std::string * error) const +{ + std::string name = "[unknown]"; + try { + if (file == nullptr) { + std::string m = "No FileDescriptor"; + throw m; + } + name = file->name(); + if (name.ends_with(".proto")) { + name = name.substr(0, name.size() - 6); + } + if (generator_context == nullptr) { + std::string m = "No GeneratorContext"; + throw m; + } + if (! parameter.empty()) { + std::string m = "unknown command line parameter " + parameter; + throw m; + } + my_generate(name, *file, *generator_context); + } catch (const std::string &message) { + *error = name + ": " + message; + return false; + } + return true; +} diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 83711ab8200..f60f1463980 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -48,6 +48,7 @@ vespa_define_module( src/tests/compression src/tests/coro/active_work src/tests/coro/async_io + src/tests/coro/csi src/tests/coro/detached src/tests/coro/generator src/tests/coro/lazy diff --git a/vespalib/src/tests/coro/csi/CMakeLists.txt b/vespalib/src/tests/coro/csi/CMakeLists.txt new file mode 100644 index 00000000000..5987428fe65 --- /dev/null +++ b/vespalib/src/tests/coro/csi/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(vespalib_csi_test_app TEST + SOURCES + csi_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_generate_protobuf_csi(vespalib_csi_test_app ./test.proto) +vespa_add_test(NAME vespalib_csi_test_app COMMAND vespalib_csi_test_app) diff --git a/vespalib/src/tests/coro/csi/csi_test.cpp b/vespalib/src/tests/coro/csi/csi_test.cpp new file mode 100644 index 00000000000..301c12b2cec --- /dev/null +++ b/vespalib/src/tests/coro/csi/csi_test.cpp @@ -0,0 +1,8 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include + +TEST(CoroRpcTest, dummy_test) { +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/coro/csi/test.proto b/vespalib/src/tests/coro/csi/test.proto new file mode 100644 index 00000000000..3e53ade7cfb --- /dev/null +++ b/vespalib/src/tests/coro/csi/test.proto @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +syntax = "proto3"; + +message TestRequest { + string name = 1; +} + +message TestReply { + string name = 1; +} + +service TestService { + rpc my_test (TestRequest) returns (TestReply) {} +} -- cgit v1.2.3