diff options
Diffstat (limited to 'logd')
-rw-r--r-- | logd/CMakeLists.txt | 1 | ||||
-rw-r--r-- | logd/src/logd/CMakeLists.txt | 4 | ||||
-rw-r--r-- | logd/src/logd/legacy_forwarder.cpp | 121 | ||||
-rw-r--r-- | logd/src/logd/legacy_forwarder.h | 11 | ||||
-rw-r--r-- | logd/src/logd/log_protocol_proto.h | 11 | ||||
-rw-r--r-- | logd/src/logd/proto_converter.cpp | 65 | ||||
-rw-r--r-- | logd/src/logd/proto_converter.h | 20 | ||||
-rw-r--r-- | logd/src/tests/proto_converter/CMakeLists.txt | 9 | ||||
-rw-r--r-- | logd/src/tests/proto_converter/proto_converter_test.cpp | 88 |
9 files changed, 216 insertions, 114 deletions
diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index 85aebac365b..9a5fdf32841 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -15,6 +15,7 @@ vespa_define_module( TESTS src/tests/legacy_forwarder + src/tests/proto_converter src/tests/rotate ) diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt index a3ff813ad96..baf52f1d5d8 100644 --- a/logd/src/logd/CMakeLists.txt +++ b/logd/src/logd/CMakeLists.txt @@ -15,11 +15,15 @@ vespa_add_library(logd STATIC conn.cpp legacy_forwarder.cpp metrics.cpp + proto_converter.cpp state_reporter.cpp watcher.cpp ${logd_PROTOBUF_SRCS} DEPENDS ) + vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) install_config_definition(../main/resources/configdefinitions/logd.def cloud.config.log.logd.def) + +vespa_add_target_package_dependency(logd Protobuf) diff --git a/logd/src/logd/legacy_forwarder.cpp b/logd/src/logd/legacy_forwarder.cpp index b512bab7fb6..b8b93a03530 100644 --- a/logd/src/logd/legacy_forwarder.cpp +++ b/logd/src/logd/legacy_forwarder.cpp @@ -3,6 +3,8 @@ #include "exceptions.h" #include "legacy_forwarder.h" #include "metrics.h" +#include <vespa/log/log_message.h> +#include <vespa/log/exceptions.h> #include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/locale/c.h> #include <unistd.h> @@ -11,6 +13,10 @@ LOG_SETUP(""); using LogLevel = ns_log::Logger::LogLevel; +using ns_log::BadLogLineException; +using ns_log::LogMessage; +using ns_log::Logger; +using LogLevel = Logger::LogLevel; namespace logdemon { @@ -18,7 +24,6 @@ LegacyForwarder::LegacyForwarder(Metrics &metrics) : _logserverfd(-1), _metrics(metrics), _forwardMap(), - _levelparser(), _badLines(0) {} LegacyForwarder::~LegacyForwarder() = default; @@ -70,120 +75,30 @@ LegacyForwarder::forwardLine(const char *line, const char *eol) bool LegacyForwarder::parseline(const char *linestart, const char *lineend) { - int llength = lineend - linestart; - - const char *fieldstart = linestart; - // time - const char *tab = strchr(fieldstart, '\t'); - if (tab == nullptr || tab == fieldstart) { - LOG(spam, "bad logline no 1. tab: %.*s", llength, linestart); - ++_badLines; - return false; - } - char *eod; - double logtime = vespalib::locale::c::strtod(fieldstart, &eod); - if (eod != tab) { - int fflen = tab - linestart; - LOG(spam, "bad logline first field not strtod parsable: %.*s", fflen, linestart); - ++_badLines; - return false; - } - time_t now = time(nullptr); - if (logtime - 864000 > now) { - int fflen = tab - linestart; - LOG(warning, "bad logline, time %.*s > 10 days in the future", fflen, linestart); - ++_badLines; - return false; - } - if (logtime + 8640000 < now) { - int fflen = tab - linestart; - LOG(warning, "bad logline, time %.*s > 100 days in the past", fflen, linestart); + LogMessage message; + try { + message.parse_log_line(std::string_view(linestart, lineend - linestart)); + } catch (BadLogLineException &e) { + LOG(spam, "bad logline: %s", e.what()); ++_badLines; return false; } - // hostname - fieldstart = tab + 1; - tab = strchr(fieldstart, '\t'); - if (tab == nullptr) { - LOG(spam, "bad logline no 2. tab: %.*s", llength, linestart); - ++_badLines; - return false; - } - - // pid - fieldstart = tab + 1; - tab = strchr(fieldstart, '\t'); - if (tab == nullptr || tab == fieldstart) { - LOG(spam, "bad logline no 3. tab: %.*s", llength, linestart); - return false; - } - - // service - fieldstart = tab + 1; - tab = strchr(fieldstart, '\t'); - if (tab == nullptr) { - LOG(spam, "bad logline no 4. tab: %.*s", llength, linestart); - ++_badLines; - return false; - } - if (tab == fieldstart) { - LOG(spam, "empty service in logline: %.*s", llength, linestart); - } - std::string service(fieldstart, tab-fieldstart); - - // component - fieldstart = tab + 1; - tab = strchr(fieldstart, '\t'); - if (tab == nullptr || tab == fieldstart) { - LOG(spam, "bad logline no 5. tab: %.*s", llength, linestart); - ++_badLines; - return false; - } - std::string component(fieldstart, tab-fieldstart); - - // level - fieldstart = tab + 1; - tab = strchr(fieldstart, '\t'); - if (tab == nullptr || tab == fieldstart) { - LOG(spam, "bad logline no 6. tab: %.*s", llength, linestart); - ++_badLines; - return false; - } - std::string level(fieldstart, tab-fieldstart); - LogLevel l = _levelparser.parseLevel(level.c_str()); - - // rest is freeform message, must be on this line: - if (tab > lineend) { - LOG(spam, "bad logline last tab after end: %.*s", llength, linestart); - ++_badLines; - return false; + std::string logLevelName; + if (message.level() >= LogLevel::NUM_LOGLEVELS) { + logLevelName = "unknown"; + } else { + logLevelName = Logger::logLevelNames[message.level()]; } - - _metrics.countLine(level, service); + _metrics.countLine(logLevelName, message.service()); // Check overrides - ForwardMap::iterator found = _forwardMap.find(l); + ForwardMap::iterator found = _forwardMap.find(message.level()); if (found != _forwardMap.end()) { return found->second; } return false; // Unknown log level } -LogLevel -LevelParser::parseLevel(const char *level) -{ - using ns_log::Logger; - - LogLevel l = Logger::parseLevel(level); - if (l >= 0 && l <= Logger::NUM_LOGLEVELS) { - return l; - } - if (_seenLevelMap.find(level) == _seenLevelMap.end()) { - LOG(warning, "unknown level '%s'", level); - _seenLevelMap.insert(level); - } - return Logger::fatal; -} } // namespace diff --git a/logd/src/logd/legacy_forwarder.h b/logd/src/logd/legacy_forwarder.h index da8dbcc82ab..81a93ce1d50 100644 --- a/logd/src/logd/legacy_forwarder.h +++ b/logd/src/logd/legacy_forwarder.h @@ -8,21 +8,11 @@ namespace logdemon { -using SeenMap = std::unordered_set<std::string>; // Mapping saying if a level should be forwarded or not using ForwardMap = std::map<ns_log::Logger::LogLevel, bool>; struct Metrics; -class LevelParser -{ -private: - SeenMap _seenLevelMap; -public: - ns_log::Logger::LogLevel parseLevel(const char *level); - LevelParser() : _seenLevelMap() {} -}; - /** * Class used to forward log lines to the logserver via a one-way text protocol. */ @@ -31,7 +21,6 @@ private: int _logserverfd; Metrics &_metrics; ForwardMap _forwardMap; - LevelParser _levelparser; int _badLines; const char *copystr(const char *b, const char *e) { int len = e - b; diff --git a/logd/src/logd/log_protocol_proto.h b/logd/src/logd/log_protocol_proto.h new file mode 100644 index 00000000000..a8d5e4aa208 --- /dev/null +++ b/logd/src/logd/log_protocol_proto.h @@ -0,0 +1,11 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsuggest-override" + +#include "log_protocol.pb.h" + +#pragma GCC diagnostic pop + diff --git a/logd/src/logd/proto_converter.cpp b/logd/src/logd/proto_converter.cpp new file mode 100644 index 00000000000..b3facd4ef4a --- /dev/null +++ b/logd/src/logd/proto_converter.cpp @@ -0,0 +1,65 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "proto_converter.h" + +using ns_log::LogMessage; +using ns_log::Logger; + +namespace logdemon { + +void +ProtoConverter::log_messages_to_proto(const std::vector<LogMessage>& messages, ProtoLogRequest& proto) +{ + for (const auto& message : messages) { + auto* proto_message = proto.add_log_messages(); + log_message_to_proto(message, *proto_message); + } +} + +namespace { + +using ProtoLogLevel = ::logserver::protocol::protobuf::LogMessage_Level; + +ProtoLogLevel +convert_level(const Logger::LogLevel& level) +{ + switch (level) { + case Logger::fatal: + return ProtoLogLevel::LogMessage_Level_FATAL; + case Logger::error: + return ProtoLogLevel::LogMessage_Level_ERROR; + case Logger::warning: + return ProtoLogLevel::LogMessage_Level_WARNING; + case Logger::config: + return ProtoLogLevel::LogMessage_Level_CONFIG; + case Logger::info: + return ProtoLogLevel::LogMessage_Level_INFO; + case Logger::event: + return ProtoLogLevel::LogMessage_Level_EVENT; + case Logger::debug: + return ProtoLogLevel::LogMessage_Level_DEBUG; + case Logger::spam: + return ProtoLogLevel::LogMessage_Level_SPAM; + case Logger::NUM_LOGLEVELS: + return ProtoLogLevel::LogMessage_Level_UNKNOWN; + default: + return ProtoLogLevel::LogMessage_Level_UNKNOWN; + } +} + +} + +void +ProtoConverter::log_message_to_proto(const LogMessage& message, ProtoLogMessage& proto) +{ + proto.set_time_nanos(message.time_nanos()); + proto.set_hostname(message.hostname()); + proto.set_process_id(message.process_id()); + proto.set_thread_id(message.thread_id()); + proto.set_service(message.service()); + proto.set_component(message.component()); + proto.set_level(convert_level(message.level())); + proto.set_payload(message.payload()); +} + +} diff --git a/logd/src/logd/proto_converter.h b/logd/src/logd/proto_converter.h new file mode 100644 index 00000000000..688648b99de --- /dev/null +++ b/logd/src/logd/proto_converter.h @@ -0,0 +1,20 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "log_protocol_proto.h" +#include <vespa/log/log_message.h> +#include <vector> + +namespace logdemon { + +/** + * Contains functions to convert log messages to protobuf objects. + */ +struct ProtoConverter { + using ProtoLogRequest = logserver::protocol::protobuf::LogRequest; + using ProtoLogMessage = logserver::protocol::protobuf::LogMessage; + + static void log_messages_to_proto(const std::vector<ns_log::LogMessage>& messages, ProtoLogRequest& proto); + static void log_message_to_proto(const ns_log::LogMessage& message, ProtoLogMessage& proto); +}; + +} diff --git a/logd/src/tests/proto_converter/CMakeLists.txt b/logd/src/tests/proto_converter/CMakeLists.txt new file mode 100644 index 00000000000..5ca048ecd4e --- /dev/null +++ b/logd/src/tests/proto_converter/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(logd_proto_converter_test_app TEST + SOURCES + proto_converter_test.cpp + DEPENDS + logd + gtest +) +vespa_add_test(NAME logd_proto_converter_test_app COMMAND logd_proto_converter_test_app) diff --git a/logd/src/tests/proto_converter/proto_converter_test.cpp b/logd/src/tests/proto_converter/proto_converter_test.cpp new file mode 100644 index 00000000000..aa0b00e34d6 --- /dev/null +++ b/logd/src/tests/proto_converter/proto_converter_test.cpp @@ -0,0 +1,88 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <logd/proto_converter.h> +#include <vespa/vespalib/gtest/gtest.h> + +using ns_log::Logger; +using ns_log::LogMessage; + +using Converter = logdemon::ProtoConverter; +using ProtoLogLevel = logserver::protocol::protobuf::LogMessage_Level; + +struct LogMessageTest : public ::testing::Test { + LogMessage message; + Converter::ProtoLogMessage proto; + void convert() { + Converter::log_message_to_proto(message, proto); + } + void expect_log_level_converted(ProtoLogLevel proto_level, Logger::LogLevel message_level) { + message = LogMessage(1, "", 1, 1, "", "", message_level, ""); + convert(); + EXPECT_EQ(proto_level, proto.level()); + } +}; + +void +expect_proto_log_message_equal(int64_t exp_time_nanos, + const std::string& exp_hostname, + int32_t exp_process_id, + int32_t exp_thread_id, + const std::string& exp_service, + const std::string& exp_component, + ProtoLogLevel exp_level, + const std::string& exp_payload, + const Converter::ProtoLogMessage& proto) +{ + EXPECT_EQ(exp_time_nanos, proto.time_nanos()); + EXPECT_EQ(exp_hostname, proto.hostname()); + EXPECT_EQ(exp_process_id, proto.process_id()); + EXPECT_EQ(exp_thread_id, proto.thread_id()); + EXPECT_EQ(exp_service, proto.service()); + EXPECT_EQ(exp_component, proto.component()); + EXPECT_EQ(exp_level, proto.level()); + EXPECT_EQ(exp_payload, proto.payload()); +} + +TEST_F(LogMessageTest, log_message_is_converted) +{ + message = LogMessage(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info, "foo_payload"); + convert(); + expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component", + ProtoLogLevel::LogMessage_Level_INFO, "foo_payload", proto); +} + +TEST_F(LogMessageTest, log_levels_are_converted) +{ + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_FATAL, Logger::fatal); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_ERROR, Logger::error); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_WARNING, Logger::warning); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_CONFIG, Logger::config); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_INFO, Logger::info); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_EVENT, Logger::event); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_DEBUG, Logger::debug); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_SPAM, Logger::spam); + expect_log_level_converted(ProtoLogLevel::LogMessage_Level_UNKNOWN, Logger::NUM_LOGLEVELS); +} + +struct LogRequestTest : public ::testing::Test { + std::vector<LogMessage> messages; + Converter::ProtoLogRequest proto; + void convert() { + Converter::log_messages_to_proto(messages, proto); + } +}; + +TEST_F(LogRequestTest, log_messages_are_converted_to_request) +{ + messages.emplace_back(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info, "foo_payload"); + messages.emplace_back(54321, "bar_host", 7, 9, "bar_service", "bar_component", Logger::event, "bar_payload"); + convert(); + EXPECT_EQ(2, proto.log_messages_size()); + expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component", + ProtoLogLevel::LogMessage_Level_INFO, "foo_payload", proto.log_messages(0)); + expect_proto_log_message_equal(54321, "bar_host", 7, 9, "bar_service", "bar_component", + ProtoLogLevel::LogMessage_Level_EVENT, "bar_payload", proto.log_messages(1)); +} + +GTEST_MAIN_RUN_ALL_TESTS() + |