Merge pull request #8907 from vespa-engine/geirst/logd-proto-converter

Implement functions for converting log messages to protobuf objects.

7 files changed, 198 insertions, 0 deletions

diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt
index 85aebac365b..9a5fdf32841 100644
--- a/logd/CMakeLists.txt
+++ b/logd/CMakeLists.txt
@@ -15,6 +15,7 @@ vespa_define_module(
 
     TESTS
     src/tests/legacy_forwarder
+    src/tests/proto_converter
     src/tests/rotate
 )
 
diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt
index a3ff813ad96..baf52f1d5d8 100644
--- a/logd/src/logd/CMakeLists.txt
+++ b/logd/src/logd/CMakeLists.txt
@@ -15,11 +15,15 @@ vespa_add_library(logd STATIC
     conn.cpp
     legacy_forwarder.cpp
     metrics.cpp
+    proto_converter.cpp
     state_reporter.cpp
     watcher.cpp
     ${logd_PROTOBUF_SRCS}
 
     DEPENDS
 )
+
 vespa_generate_config(logd ../main/resources/configdefinitions/logd.def)
 install_config_definition(../main/resources/configdefinitions/logd.def cloud.config.log.logd.def)
+
+vespa_add_target_package_dependency(logd Protobuf)
diff --git a/logd/src/logd/log_protocol_proto.h b/logd/src/logd/log_protocol_proto.h
new file mode 100644
index 00000000000..a8d5e4aa208
--- /dev/null
+++ b/logd/src/logd/log_protocol_proto.h
@@ -0,0 +1,11 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsuggest-override"
+
+#include "log_protocol.pb.h"
+
+#pragma GCC diagnostic pop
+
diff --git a/logd/src/logd/proto_converter.cpp b/logd/src/logd/proto_converter.cpp
new file mode 100644
index 00000000000..b3facd4ef4a
--- /dev/null
+++ b/logd/src/logd/proto_converter.cpp
@@ -0,0 +1,65 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "proto_converter.h"
+
+using ns_log::LogMessage;
+using ns_log::Logger;
+
+namespace logdemon {
+
+void
+ProtoConverter::log_messages_to_proto(const std::vector<LogMessage>& messages, ProtoLogRequest& proto)
+{
+    for (const auto& message : messages) {
+        auto* proto_message = proto.add_log_messages();
+        log_message_to_proto(message, *proto_message);
+    }
+}
+
+namespace {
+
+using ProtoLogLevel = ::logserver::protocol::protobuf::LogMessage_Level;
+
+ProtoLogLevel
+convert_level(const Logger::LogLevel& level)
+{
+    switch (level) {
+        case Logger::fatal:
+            return ProtoLogLevel::LogMessage_Level_FATAL;
+        case Logger::error:
+            return ProtoLogLevel::LogMessage_Level_ERROR;
+        case Logger::warning:
+            return ProtoLogLevel::LogMessage_Level_WARNING;
+        case Logger::config:
+            return ProtoLogLevel::LogMessage_Level_CONFIG;
+        case Logger::info:
+            return ProtoLogLevel::LogMessage_Level_INFO;
+        case Logger::event:
+            return ProtoLogLevel::LogMessage_Level_EVENT;
+        case Logger::debug:
+            return ProtoLogLevel::LogMessage_Level_DEBUG;
+        case Logger::spam:
+            return ProtoLogLevel::LogMessage_Level_SPAM;
+        case Logger::NUM_LOGLEVELS:
+            return ProtoLogLevel::LogMessage_Level_UNKNOWN;
+        default:
+            return ProtoLogLevel::LogMessage_Level_UNKNOWN;
+    }
+}
+
+}
+
+void
+ProtoConverter::log_message_to_proto(const LogMessage& message, ProtoLogMessage& proto)
+{
+    proto.set_time_nanos(message.time_nanos());
+    proto.set_hostname(message.hostname());
+    proto.set_process_id(message.process_id());
+    proto.set_thread_id(message.thread_id());
+    proto.set_service(message.service());
+    proto.set_component(message.component());
+    proto.set_level(convert_level(message.level()));
+    proto.set_payload(message.payload());
+}
+
+}
diff --git a/logd/src/logd/proto_converter.h b/logd/src/logd/proto_converter.h
new file mode 100644
index 00000000000..688648b99de
--- /dev/null
+++ b/logd/src/logd/proto_converter.h
@@ -0,0 +1,20 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "log_protocol_proto.h"
+#include <vespa/log/log_message.h>
+#include <vector>
+
+namespace logdemon {
+
+/**
+ * Contains functions to convert log messages to protobuf objects.
+ */
+struct ProtoConverter {
+    using ProtoLogRequest = logserver::protocol::protobuf::LogRequest;
+    using ProtoLogMessage = logserver::protocol::protobuf::LogMessage;
+
+    static void log_messages_to_proto(const std::vector<ns_log::LogMessage>& messages, ProtoLogRequest& proto);
+    static void log_message_to_proto(const ns_log::LogMessage& message, ProtoLogMessage& proto);
+};
+
+}
diff --git a/logd/src/tests/proto_converter/CMakeLists.txt b/logd/src/tests/proto_converter/CMakeLists.txt
new file mode 100644
index 00000000000..5ca048ecd4e
--- /dev/null
+++ b/logd/src/tests/proto_converter/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(logd_proto_converter_test_app TEST
+    SOURCES
+    proto_converter_test.cpp
+    DEPENDS
+    logd
+    gtest
+)
+vespa_add_test(NAME logd_proto_converter_test_app COMMAND logd_proto_converter_test_app)
diff --git a/logd/src/tests/proto_converter/proto_converter_test.cpp b/logd/src/tests/proto_converter/proto_converter_test.cpp
new file mode 100644
index 00000000000..aa0b00e34d6
--- /dev/null
+++ b/logd/src/tests/proto_converter/proto_converter_test.cpp
@@ -0,0 +1,88 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <logd/proto_converter.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using ns_log::Logger;
+using ns_log::LogMessage;
+
+using Converter = logdemon::ProtoConverter;
+using ProtoLogLevel = logserver::protocol::protobuf::LogMessage_Level;
+
+struct LogMessageTest : public ::testing::Test {
+    LogMessage message;
+    Converter::ProtoLogMessage proto;
+    void convert() {
+        Converter::log_message_to_proto(message, proto);
+    }
+    void expect_log_level_converted(ProtoLogLevel proto_level, Logger::LogLevel message_level) {
+       message = LogMessage(1, "", 1, 1, "", "", message_level, "");
+       convert();
+       EXPECT_EQ(proto_level, proto.level());
+    }
+};
+
+void
+expect_proto_log_message_equal(int64_t exp_time_nanos,
+                               const std::string& exp_hostname,
+                               int32_t exp_process_id,
+                               int32_t exp_thread_id,
+                               const std::string& exp_service,
+                               const std::string& exp_component,
+                               ProtoLogLevel exp_level,
+                               const std::string& exp_payload,
+                               const Converter::ProtoLogMessage& proto)
+{
+    EXPECT_EQ(exp_time_nanos, proto.time_nanos());
+    EXPECT_EQ(exp_hostname, proto.hostname());
+    EXPECT_EQ(exp_process_id, proto.process_id());
+    EXPECT_EQ(exp_thread_id, proto.thread_id());
+    EXPECT_EQ(exp_service, proto.service());
+    EXPECT_EQ(exp_component, proto.component());
+    EXPECT_EQ(exp_level, proto.level());
+    EXPECT_EQ(exp_payload, proto.payload());
+}
+
+TEST_F(LogMessageTest, log_message_is_converted)
+{
+    message = LogMessage(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info, "foo_payload");
+    convert();
+    expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component",
+                                   ProtoLogLevel::LogMessage_Level_INFO, "foo_payload", proto);
+}
+
+TEST_F(LogMessageTest, log_levels_are_converted)
+{
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_FATAL, Logger::fatal);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_ERROR, Logger::error);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_WARNING, Logger::warning);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_CONFIG, Logger::config);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_INFO, Logger::info);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_EVENT, Logger::event);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_DEBUG, Logger::debug);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_SPAM, Logger::spam);
+    expect_log_level_converted(ProtoLogLevel::LogMessage_Level_UNKNOWN, Logger::NUM_LOGLEVELS);
+}
+
+struct LogRequestTest : public ::testing::Test {
+    std::vector<LogMessage> messages;
+    Converter::ProtoLogRequest proto;
+    void convert() {
+        Converter::log_messages_to_proto(messages, proto);
+    }
+};
+
+TEST_F(LogRequestTest, log_messages_are_converted_to_request)
+{
+    messages.emplace_back(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info, "foo_payload");
+    messages.emplace_back(54321, "bar_host", 7, 9, "bar_service", "bar_component", Logger::event, "bar_payload");
+    convert();
+    EXPECT_EQ(2, proto.log_messages_size());
+    expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component",
+                                   ProtoLogLevel::LogMessage_Level_INFO, "foo_payload", proto.log_messages(0));
+    expect_proto_log_message_equal(54321, "bar_host", 7, 9, "bar_service", "bar_component",
+                                   ProtoLogLevel::LogMessage_Level_EVENT, "bar_payload", proto.log_messages(1));
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+