summaryrefslogtreecommitdiffstats
path: root/logd
diff options
context:
space:
mode:
Diffstat (limited to 'logd')
-rw-r--r--logd/CMakeLists.txt1
-rw-r--r--logd/src/logd/CMakeLists.txt4
-rw-r--r--logd/src/logd/legacy_forwarder.cpp121
-rw-r--r--logd/src/logd/legacy_forwarder.h11
-rw-r--r--logd/src/logd/log_protocol_proto.h11
-rw-r--r--logd/src/logd/proto_converter.cpp65
-rw-r--r--logd/src/logd/proto_converter.h20
-rw-r--r--logd/src/tests/proto_converter/CMakeLists.txt9
-rw-r--r--logd/src/tests/proto_converter/proto_converter_test.cpp88
9 files changed, 216 insertions, 114 deletions
diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt
index 85aebac365b..9a5fdf32841 100644
--- a/logd/CMakeLists.txt
+++ b/logd/CMakeLists.txt
@@ -15,6 +15,7 @@ vespa_define_module(
TESTS
src/tests/legacy_forwarder
+ src/tests/proto_converter
src/tests/rotate
)
diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt
index a3ff813ad96..baf52f1d5d8 100644
--- a/logd/src/logd/CMakeLists.txt
+++ b/logd/src/logd/CMakeLists.txt
@@ -15,11 +15,15 @@ vespa_add_library(logd STATIC
conn.cpp
legacy_forwarder.cpp
metrics.cpp
+ proto_converter.cpp
state_reporter.cpp
watcher.cpp
${logd_PROTOBUF_SRCS}
DEPENDS
)
+
vespa_generate_config(logd ../main/resources/configdefinitions/logd.def)
install_config_definition(../main/resources/configdefinitions/logd.def cloud.config.log.logd.def)
+
+vespa_add_target_package_dependency(logd Protobuf)
diff --git a/logd/src/logd/legacy_forwarder.cpp b/logd/src/logd/legacy_forwarder.cpp
index b512bab7fb6..b8b93a03530 100644
--- a/logd/src/logd/legacy_forwarder.cpp
+++ b/logd/src/logd/legacy_forwarder.cpp
@@ -3,6 +3,8 @@
#include "exceptions.h"
#include "legacy_forwarder.h"
#include "metrics.h"
+#include <vespa/log/log_message.h>
+#include <vespa/log/exceptions.h>
#include <vespa/vespalib/component/vtag.h>
#include <vespa/vespalib/locale/c.h>
#include <unistd.h>
@@ -11,6 +13,10 @@
LOG_SETUP("");
using LogLevel = ns_log::Logger::LogLevel;
+using ns_log::BadLogLineException;
+using ns_log::LogMessage;
+using ns_log::Logger;
+using LogLevel = Logger::LogLevel;
namespace logdemon {
@@ -18,7 +24,6 @@ LegacyForwarder::LegacyForwarder(Metrics &metrics)
: _logserverfd(-1),
_metrics(metrics),
_forwardMap(),
- _levelparser(),
_badLines(0)
{}
LegacyForwarder::~LegacyForwarder() = default;
@@ -70,120 +75,30 @@ LegacyForwarder::forwardLine(const char *line, const char *eol)
bool
LegacyForwarder::parseline(const char *linestart, const char *lineend)
{
- int llength = lineend - linestart;
-
- const char *fieldstart = linestart;
- // time
- const char *tab = strchr(fieldstart, '\t');
- if (tab == nullptr || tab == fieldstart) {
- LOG(spam, "bad logline no 1. tab: %.*s", llength, linestart);
- ++_badLines;
- return false;
- }
- char *eod;
- double logtime = vespalib::locale::c::strtod(fieldstart, &eod);
- if (eod != tab) {
- int fflen = tab - linestart;
- LOG(spam, "bad logline first field not strtod parsable: %.*s", fflen, linestart);
- ++_badLines;
- return false;
- }
- time_t now = time(nullptr);
- if (logtime - 864000 > now) {
- int fflen = tab - linestart;
- LOG(warning, "bad logline, time %.*s > 10 days in the future", fflen, linestart);
- ++_badLines;
- return false;
- }
- if (logtime + 8640000 < now) {
- int fflen = tab - linestart;
- LOG(warning, "bad logline, time %.*s > 100 days in the past", fflen, linestart);
+ LogMessage message;
+ try {
+ message.parse_log_line(std::string_view(linestart, lineend - linestart));
+ } catch (BadLogLineException &e) {
+ LOG(spam, "bad logline: %s", e.what());
++_badLines;
return false;
}
- // hostname
- fieldstart = tab + 1;
- tab = strchr(fieldstart, '\t');
- if (tab == nullptr) {
- LOG(spam, "bad logline no 2. tab: %.*s", llength, linestart);
- ++_badLines;
- return false;
- }
-
- // pid
- fieldstart = tab + 1;
- tab = strchr(fieldstart, '\t');
- if (tab == nullptr || tab == fieldstart) {
- LOG(spam, "bad logline no 3. tab: %.*s", llength, linestart);
- return false;
- }
-
- // service
- fieldstart = tab + 1;
- tab = strchr(fieldstart, '\t');
- if (tab == nullptr) {
- LOG(spam, "bad logline no 4. tab: %.*s", llength, linestart);
- ++_badLines;
- return false;
- }
- if (tab == fieldstart) {
- LOG(spam, "empty service in logline: %.*s", llength, linestart);
- }
- std::string service(fieldstart, tab-fieldstart);
-
- // component
- fieldstart = tab + 1;
- tab = strchr(fieldstart, '\t');
- if (tab == nullptr || tab == fieldstart) {
- LOG(spam, "bad logline no 5. tab: %.*s", llength, linestart);
- ++_badLines;
- return false;
- }
- std::string component(fieldstart, tab-fieldstart);
-
- // level
- fieldstart = tab + 1;
- tab = strchr(fieldstart, '\t');
- if (tab == nullptr || tab == fieldstart) {
- LOG(spam, "bad logline no 6. tab: %.*s", llength, linestart);
- ++_badLines;
- return false;
- }
- std::string level(fieldstart, tab-fieldstart);
- LogLevel l = _levelparser.parseLevel(level.c_str());
-
- // rest is freeform message, must be on this line:
- if (tab > lineend) {
- LOG(spam, "bad logline last tab after end: %.*s", llength, linestart);
- ++_badLines;
- return false;
+ std::string logLevelName;
+ if (message.level() >= LogLevel::NUM_LOGLEVELS) {
+ logLevelName = "unknown";
+ } else {
+ logLevelName = Logger::logLevelNames[message.level()];
}
-
- _metrics.countLine(level, service);
+ _metrics.countLine(logLevelName, message.service());
// Check overrides
- ForwardMap::iterator found = _forwardMap.find(l);
+ ForwardMap::iterator found = _forwardMap.find(message.level());
if (found != _forwardMap.end()) {
return found->second;
}
return false; // Unknown log level
}
-LogLevel
-LevelParser::parseLevel(const char *level)
-{
- using ns_log::Logger;
-
- LogLevel l = Logger::parseLevel(level);
- if (l >= 0 && l <= Logger::NUM_LOGLEVELS) {
- return l;
- }
- if (_seenLevelMap.find(level) == _seenLevelMap.end()) {
- LOG(warning, "unknown level '%s'", level);
- _seenLevelMap.insert(level);
- }
- return Logger::fatal;
-}
} // namespace
diff --git a/logd/src/logd/legacy_forwarder.h b/logd/src/logd/legacy_forwarder.h
index da8dbcc82ab..81a93ce1d50 100644
--- a/logd/src/logd/legacy_forwarder.h
+++ b/logd/src/logd/legacy_forwarder.h
@@ -8,21 +8,11 @@
namespace logdemon {
-using SeenMap = std::unordered_set<std::string>;
// Mapping saying if a level should be forwarded or not
using ForwardMap = std::map<ns_log::Logger::LogLevel, bool>;
struct Metrics;
-class LevelParser
-{
-private:
- SeenMap _seenLevelMap;
-public:
- ns_log::Logger::LogLevel parseLevel(const char *level);
- LevelParser() : _seenLevelMap() {}
-};
-
/**
* Class used to forward log lines to the logserver via a one-way text protocol.
*/
@@ -31,7 +21,6 @@ private:
int _logserverfd;
Metrics &_metrics;
ForwardMap _forwardMap;
- LevelParser _levelparser;
int _badLines;
const char *copystr(const char *b, const char *e) {
int len = e - b;
diff --git a/logd/src/logd/log_protocol_proto.h b/logd/src/logd/log_protocol_proto.h
new file mode 100644
index 00000000000..a8d5e4aa208
--- /dev/null
+++ b/logd/src/logd/log_protocol_proto.h
@@ -0,0 +1,11 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsuggest-override"
+
+#include "log_protocol.pb.h"
+
+#pragma GCC diagnostic pop
+
diff --git a/logd/src/logd/proto_converter.cpp b/logd/src/logd/proto_converter.cpp
new file mode 100644
index 00000000000..b3facd4ef4a
--- /dev/null
+++ b/logd/src/logd/proto_converter.cpp
@@ -0,0 +1,65 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "proto_converter.h"
+
+using ns_log::LogMessage;
+using ns_log::Logger;
+
+namespace logdemon {
+
+void
+ProtoConverter::log_messages_to_proto(const std::vector<LogMessage>& messages, ProtoLogRequest& proto)
+{
+ for (const auto& message : messages) {
+ auto* proto_message = proto.add_log_messages();
+ log_message_to_proto(message, *proto_message);
+ }
+}
+
+namespace {
+
+using ProtoLogLevel = ::logserver::protocol::protobuf::LogMessage_Level;
+
+ProtoLogLevel
+convert_level(const Logger::LogLevel& level)
+{
+ switch (level) {
+ case Logger::fatal:
+ return ProtoLogLevel::LogMessage_Level_FATAL;
+ case Logger::error:
+ return ProtoLogLevel::LogMessage_Level_ERROR;
+ case Logger::warning:
+ return ProtoLogLevel::LogMessage_Level_WARNING;
+ case Logger::config:
+ return ProtoLogLevel::LogMessage_Level_CONFIG;
+ case Logger::info:
+ return ProtoLogLevel::LogMessage_Level_INFO;
+ case Logger::event:
+ return ProtoLogLevel::LogMessage_Level_EVENT;
+ case Logger::debug:
+ return ProtoLogLevel::LogMessage_Level_DEBUG;
+ case Logger::spam:
+ return ProtoLogLevel::LogMessage_Level_SPAM;
+ case Logger::NUM_LOGLEVELS:
+ return ProtoLogLevel::LogMessage_Level_UNKNOWN;
+ default:
+ return ProtoLogLevel::LogMessage_Level_UNKNOWN;
+ }
+}
+
+}
+
+void
+ProtoConverter::log_message_to_proto(const LogMessage& message, ProtoLogMessage& proto)
+{
+ proto.set_time_nanos(message.time_nanos());
+ proto.set_hostname(message.hostname());
+ proto.set_process_id(message.process_id());
+ proto.set_thread_id(message.thread_id());
+ proto.set_service(message.service());
+ proto.set_component(message.component());
+ proto.set_level(convert_level(message.level()));
+ proto.set_payload(message.payload());
+}
+
+}
diff --git a/logd/src/logd/proto_converter.h b/logd/src/logd/proto_converter.h
new file mode 100644
index 00000000000..688648b99de
--- /dev/null
+++ b/logd/src/logd/proto_converter.h
@@ -0,0 +1,20 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "log_protocol_proto.h"
+#include <vespa/log/log_message.h>
+#include <vector>
+
+namespace logdemon {
+
+/**
+ * Contains functions to convert log messages to protobuf objects.
+ */
+struct ProtoConverter {
+ using ProtoLogRequest = logserver::protocol::protobuf::LogRequest;
+ using ProtoLogMessage = logserver::protocol::protobuf::LogMessage;
+
+ static void log_messages_to_proto(const std::vector<ns_log::LogMessage>& messages, ProtoLogRequest& proto);
+ static void log_message_to_proto(const ns_log::LogMessage& message, ProtoLogMessage& proto);
+};
+
+}
diff --git a/logd/src/tests/proto_converter/CMakeLists.txt b/logd/src/tests/proto_converter/CMakeLists.txt
new file mode 100644
index 00000000000..5ca048ecd4e
--- /dev/null
+++ b/logd/src/tests/proto_converter/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(logd_proto_converter_test_app TEST
+ SOURCES
+ proto_converter_test.cpp
+ DEPENDS
+ logd
+ gtest
+)
+vespa_add_test(NAME logd_proto_converter_test_app COMMAND logd_proto_converter_test_app)
diff --git a/logd/src/tests/proto_converter/proto_converter_test.cpp b/logd/src/tests/proto_converter/proto_converter_test.cpp
new file mode 100644
index 00000000000..aa0b00e34d6
--- /dev/null
+++ b/logd/src/tests/proto_converter/proto_converter_test.cpp
@@ -0,0 +1,88 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include <logd/proto_converter.h>
+#include <vespa/vespalib/gtest/gtest.h>
+
+using ns_log::Logger;
+using ns_log::LogMessage;
+
+using Converter = logdemon::ProtoConverter;
+using ProtoLogLevel = logserver::protocol::protobuf::LogMessage_Level;
+
+struct LogMessageTest : public ::testing::Test {
+ LogMessage message;
+ Converter::ProtoLogMessage proto;
+ void convert() {
+ Converter::log_message_to_proto(message, proto);
+ }
+ void expect_log_level_converted(ProtoLogLevel proto_level, Logger::LogLevel message_level) {
+ message = LogMessage(1, "", 1, 1, "", "", message_level, "");
+ convert();
+ EXPECT_EQ(proto_level, proto.level());
+ }
+};
+
+void
+expect_proto_log_message_equal(int64_t exp_time_nanos,
+ const std::string& exp_hostname,
+ int32_t exp_process_id,
+ int32_t exp_thread_id,
+ const std::string& exp_service,
+ const std::string& exp_component,
+ ProtoLogLevel exp_level,
+ const std::string& exp_payload,
+ const Converter::ProtoLogMessage& proto)
+{
+ EXPECT_EQ(exp_time_nanos, proto.time_nanos());
+ EXPECT_EQ(exp_hostname, proto.hostname());
+ EXPECT_EQ(exp_process_id, proto.process_id());
+ EXPECT_EQ(exp_thread_id, proto.thread_id());
+ EXPECT_EQ(exp_service, proto.service());
+ EXPECT_EQ(exp_component, proto.component());
+ EXPECT_EQ(exp_level, proto.level());
+ EXPECT_EQ(exp_payload, proto.payload());
+}
+
+TEST_F(LogMessageTest, log_message_is_converted)
+{
+ message = LogMessage(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info, "foo_payload");
+ convert();
+ expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component",
+ ProtoLogLevel::LogMessage_Level_INFO, "foo_payload", proto);
+}
+
+TEST_F(LogMessageTest, log_levels_are_converted)
+{
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_FATAL, Logger::fatal);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_ERROR, Logger::error);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_WARNING, Logger::warning);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_CONFIG, Logger::config);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_INFO, Logger::info);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_EVENT, Logger::event);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_DEBUG, Logger::debug);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_SPAM, Logger::spam);
+ expect_log_level_converted(ProtoLogLevel::LogMessage_Level_UNKNOWN, Logger::NUM_LOGLEVELS);
+}
+
+struct LogRequestTest : public ::testing::Test {
+ std::vector<LogMessage> messages;
+ Converter::ProtoLogRequest proto;
+ void convert() {
+ Converter::log_messages_to_proto(messages, proto);
+ }
+};
+
+TEST_F(LogRequestTest, log_messages_are_converted_to_request)
+{
+ messages.emplace_back(12345, "foo_host", 3, 5, "foo_service", "foo_component", Logger::info, "foo_payload");
+ messages.emplace_back(54321, "bar_host", 7, 9, "bar_service", "bar_component", Logger::event, "bar_payload");
+ convert();
+ EXPECT_EQ(2, proto.log_messages_size());
+ expect_proto_log_message_equal(12345, "foo_host", 3, 5, "foo_service", "foo_component",
+ ProtoLogLevel::LogMessage_Level_INFO, "foo_payload", proto.log_messages(0));
+ expect_proto_log_message_equal(54321, "bar_host", 7, 9, "bar_service", "bar_component",
+ ProtoLogLevel::LogMessage_Level_EVENT, "bar_payload", proto.log_messages(1));
+}
+
+GTEST_MAIN_RUN_ALL_TESTS()
+