diff options
author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
---|---|---|
committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /logd |
Publish
Diffstat (limited to 'logd')
51 files changed, 2443 insertions, 0 deletions
diff --git a/logd/.gitignore b/logd/.gitignore new file mode 100644 index 00000000000..fd963ca589b --- /dev/null +++ b/logd/.gitignore @@ -0,0 +1,6 @@ +bin +conf +/target +/pom.xml.build +Makefile +Testing diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt new file mode 100644 index 00000000000..b69a56e7697 --- /dev/null +++ b/logd/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalog + vespalib + config_cloudconfig + + APPS + src/apps/logd + + LIBS + src/logd + + TESTS + src/tests/forward + src/tests/info + src/tests/rotate +) diff --git a/logd/OWNERS b/logd/OWNERS new file mode 100644 index 00000000000..67cd2820bb8 --- /dev/null +++ b/logd/OWNERS @@ -0,0 +1 @@ +arnej27959 diff --git a/logd/README b/logd/README new file mode 100644 index 00000000000..736ead1eb3b --- /dev/null +++ b/logd/README @@ -0,0 +1,3 @@ +This is the local log forwarding and rotation daemon. + +currently this is a simple "tail -f | netcat" equivalent. diff --git a/logd/README.server.protocol b/logd/README.server.protocol new file mode 100644 index 00000000000..256c6cfd7f7 --- /dev/null +++ b/logd/README.server.protocol @@ -0,0 +1,42 @@ +Protocol for talking to the log server: +(nothing implemented in logd yet). + +Pure line-based text protocol. Tcp to some port on the log server: + +From log daemon (or anything, really) to server: + +service <service> +component <service> <component> +state <service> <component> <levels> + + +This tells the log daemon that service <service> exists, and that it +has a component named <component>. + +<levels> syntax: + +fatal=forward,error=forward,warning=store,info=store,event=forward,debug=off,spam=off + +ie: level=<status>[,...]* + +Where <status> is one of: +forward : store and send to log server +noforward : do not send to logserver (only applicable to setdefaultstate) +store : store locally only (to file) +off : do not even generate the message in the program + + +From server to whoever is interested: + +setdefaultstate <levels> + +setstate <service> <component> <levels> + +Exactly same syntax. Levels that are not mentioned are not modified. + +list services +list components <service> +list states <service> <component> + +(list commands are replied to with service/component/state commands). + diff --git a/logd/pom.xml b/logd/pom.xml new file mode 100644 index 00000000000..8385960066e --- /dev/null +++ b/logd/pom.xml @@ -0,0 +1,41 @@ +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>logd</artifactId> + <version>6-SNAPSHOT</version> + <packaging>jar</packaging> + <name>${project.artifactId}</name> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-lib</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-class-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>config-gen</id> + <goals> + <goal>config-gen</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/logd/src/.gitignore b/logd/src/.gitignore new file mode 100644 index 00000000000..a39df0815b3 --- /dev/null +++ b/logd/src/.gitignore @@ -0,0 +1,3 @@ +Makefile.ini +config_command.sh +project.dsw diff --git a/logd/src/apps/logd/.gitignore b/logd/src/apps/logd/.gitignore new file mode 100644 index 00000000000..cad0e581304 --- /dev/null +++ b/logd/src/apps/logd/.gitignore @@ -0,0 +1,3 @@ +/.depend +/Makefile +/logd diff --git a/logd/src/apps/logd/CMakeLists.txt b/logd/src/apps/logd/CMakeLists.txt new file mode 100644 index 00000000000..7caf8300617 --- /dev/null +++ b/logd/src/apps/logd/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(logd_app + SOURCES + main.cpp + OUTPUT_NAME logd + INSTALL sbin + DEPENDS + logd +) diff --git a/logd/src/apps/logd/main.cpp b/logd/src/apps/logd/main.cpp new file mode 100644 index 00000000000..b76ec71bf8a --- /dev/null +++ b/logd/src/apps/logd/main.cpp @@ -0,0 +1,106 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <sys/stat.h> + +#include <vespa/log/log.h> +LOG_SETUP("logdemon"); +LOG_RCSID("$Id$"); + +#include <logd/errhandle.h> +#include <logd/sigterm.h> +#include <logd/service.h> +#include <logd/forward.h> +#include <logd/conf.h> +#include <logd/watch.h> +#include <logd/conn.h> +#include <vespa/config/common/exceptions.h> + +using namespace logdemon; +using config::FileSpec; + +int main(int, char**) +{ + Forwarder fwd; + + EV_STARTED("logdemon"); + + hook_signals(); + + const char *cfid = getenv("VESPA_CONFIG_ID"); + + try { + std::unique_ptr<ConfSub> subscriberP; + subscriberP.reset(new ConfSub(fwd, config::ConfigUri(cfid))); + ConfSub & subscriber(*subscriberP); + + int sleepcount = 0; + while (true) { + Watcher watcher(subscriber, fwd); + + try { + subscriber.latch(); + int fd = subscriber.getservfd(); + if (fd >= 0) { + sleepcount = 0 ; // connection OK, reset sleep time + watcher.watchfile(); + } else { + LOG(spam, "bad fd in subscriber"); + } + } catch (ConnectionException& ex) { + LOG(debug, "connection exception: %s", ex.what()); + subscriber.closeConn(); + } + if (gotSignaled()) { + throw SigTermException("caught signal"); + } + if (sleepcount < 60) { + ++sleepcount; + } else { + sleepcount = 60; + } + LOG(debug, "sleep %d...", sleepcount); + for (int i = 0; i < sleepcount; i++) { + sleep(1); + if (gotSignaled()) { + throw SigTermException("caught signal"); + } + } + } + } catch (config::ConfigRuntimeException & ex) { + LOG(error, "Configuration failed: %s", ex.what()); + EV_STOPPING("logdemon", "bad config"); + return 1; + } catch (config::InvalidConfigException & ex) { + LOG(error, "Configuration failed: %s", ex.what()); + EV_STOPPING("logdemon", "bad config"); + return 1; + } catch (SigTermException& ex) { + if (gotSignalNumber() == SIGTERM) { + LOG(debug, "stopping on SIGTERM"); + EV_STOPPING("logdemon", "done ok."); + } else { + LOG(warning, "stopping on signal %d", gotSignalNumber()); + char buf[100]; + snprintf(buf, sizeof buf, "got signal %d", gotSignalNumber()); + EV_STOPPING("logdemon", buf); + } + return 0; + } catch (MsgException& ex) { + LOG(error, "stopping on error: %s", ex.what()); + EV_STOPPING("logdemon", "fatal error"); + return 1; + } catch (...) { + LOG(error, "unknown exception"); + EV_STOPPING("logdemon", "unknown error"); + return 1; + } + LOG(error, "connecting to logserver failed"); + EV_STOPPING("logdemon", "giving up after endless retries"); + return 1; +} diff --git a/logd/src/logd/.gitignore b/logd/src/logd/.gitignore new file mode 100644 index 00000000000..af2c87d8c89 --- /dev/null +++ b/logd/src/logd/.gitignore @@ -0,0 +1,7 @@ +*.core +.depend +Makefile +config-logd.cpp +config-logd.h +core +logd diff --git a/logd/src/logd/CMakeLists.txt b/logd/src/logd/CMakeLists.txt new file mode 100644 index 00000000000..3c0f7bacd33 --- /dev/null +++ b/logd/src/logd/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(logd STATIC + SOURCES + watch.cpp + conn.cpp + conf.cpp + forward.cpp + service.cpp + cmdbuf.cpp + perform.cpp + sigterm.cpp + vtag.cpp + DEPENDS +) +vespa_generate_config(logd ../main/resources/configdefinitions/logd.def) +install(FILES ../main/resources/configdefinitions/logd.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/logd/src/logd/cmdbuf.cpp b/logd/src/logd/cmdbuf.cpp new file mode 100644 index 00000000000..9412a9cd588 --- /dev/null +++ b/logd/src/logd/cmdbuf.cpp @@ -0,0 +1,159 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/time.h> + +#include <vespa/log/log.h> +LOG_SETUP(""); +LOG_RCSID("$Id$"); + +#include "errhandle.h" +#include "service.h" +#include "forward.h" +#include "perform.h" +#include "cmdbuf.h" + +namespace logdemon { + +CmdBuf::CmdBuf() + : _size(1000), + _buf((char *)malloc(_size)), + _bp(_buf), + _left(_size) +{ } + + +CmdBuf::~CmdBuf() +{ + free(_buf); +} + + +bool +CmdBuf::hasCmd() +{ + char *p = _buf; + while (p < _bp) { + if (*p == '\n') return true; + p++; + } + return false; + +} + +void +CmdBuf::doCmd(Performer& via) +{ + char *p = _buf; + while (p < _bp) { + if (*p == '\n') { + *p = '\0'; + LOG(spam, "doing command: '%s'", _buf); + via.doCmd(_buf); + + ++p; + int len = p - _buf; + int movelen = _bp - p; + memmove(_buf, p, movelen); + _bp -= len; + _left += len; + + p = _buf; + continue; + } + p++; + } +} + + +void +CmdBuf::extend() +{ + _size *= 2; + int pos = _bp - _buf; + char *nbuf = (char *)realloc(_buf, _size); + if (nbuf == NULL) { + free(_buf); + LOG(error, "could not allocate %d bytes", _size); + throw SomethingBad("realloc failed"); + } + _buf = nbuf; + _bp = _buf + pos; + _left = _size - pos; +} + +#ifndef O_NONBLOCK +#define O_NONBLOCK O_NDELAY +#endif + +void +CmdBuf::maybeRead(int fd) +{ + struct timeval notime; + notime.tv_sec = 0; + notime.tv_usec = 0; + + fd_set fdset; + FD_ZERO(&fdset); + FD_SET(fd, &fdset); + + while (select(fd + 1, &fdset, NULL, NULL, ¬ime) > 0) { + // usually loops just once + int oflags = fcntl(fd, F_GETFL); + int nbflags = oflags | O_NONBLOCK; + + if (fcntl(fd, F_SETFL, nbflags) != 0) { + LOG(error, "could not fcntl logserver socket: %s", + strerror(errno)); + throw SomethingBad("fcntl failed"); + } + + ssize_t len = ::read(fd, _bp, _left); + if (len > 0) { + _bp += len; + _left -= len; + if (_left < 80) { + extend(); + } + } else if (len < 0) { + LOG(warning, "error reading from logserver socket: %s", + strerror(errno)); + throw ConnectionException("error reading"); + } + fcntl(fd, F_SETFL, oflags); + if (len == 0) { + LOG(warning, "read 0 bytes from logserver socket"); + throw ConnectionException("eof on socket"); + break; + } + } + return; +} + + +bool +CmdBuf::readFile(int fd) +{ + ssize_t len = ::read(fd, _bp, _left); + if (len > 0) { + _bp += len; + _left -= len; + if (_left < 80) { + extend(); + } + return true; + } + if (len < 0) { + LOG(error, "error reading file: %s", + strerror(errno)); + throw SomethingBad("read failed"); + } + return false; +} + +} // namespace diff --git a/logd/src/logd/cmdbuf.h b/logd/src/logd/cmdbuf.h new file mode 100644 index 00000000000..de471d50f85 --- /dev/null +++ b/logd/src/logd/cmdbuf.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +namespace logdemon { + +class CmdBuf +{ +private: + int _size; + char *_buf; + char *_bp; + int _left; + void extend(); + + CmdBuf(const CmdBuf& other); + CmdBuf& operator= (const CmdBuf& other); +public: + CmdBuf(); + ~CmdBuf(); + bool hasCmd(); + void doCmd(Performer &via); + void maybeRead(int fd); + bool readFile(int fd); +}; + +} // namespace diff --git a/logd/src/logd/conf.cpp b/logd/src/logd/conf.cpp new file mode 100644 index 00000000000..444a755170d --- /dev/null +++ b/logd/src/logd/conf.cpp @@ -0,0 +1,173 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <sys/stat.h> + +#include <vespa/log/log.h> +LOG_SETUP(""); +LOG_RCSID("$Id$"); + +#include "conn.h" +#include "service.h" +#include "forward.h" +#include "conf.h" +#include <vespa/config/common/exceptions.h> + +using cloud::config::log::LogdConfig; +using ns_log::Logger; + +namespace logdemon { + +void +ConfSub::configure(std::unique_ptr<LogdConfig> cfg) +{ + const LogdConfig &newconf(*cfg); + if (newconf.logserver.host != _logServer) + { + if (newconf.logserver.host.size() > 255) { + LOG(warning, "too long logserver hostname: %s", + newconf.logserver.host.c_str()); + return; + } + strcpy(_logServer, newconf.logserver.host.c_str()); + _newConf = true; + } + if (newconf.logserver.use != _use_logserver) + { + _use_logserver = newconf.logserver.use; + _newConf = true; + } + ForwardMap forwardMap; + forwardMap[Logger::fatal] = newconf.loglevel.fatal.forward; + forwardMap[Logger::error] = newconf.loglevel.error.forward; + forwardMap[Logger::warning] = newconf.loglevel.warning.forward; + forwardMap[Logger::config] = newconf.loglevel.config.forward; + forwardMap[Logger::info] = newconf.loglevel.info.forward; + forwardMap[Logger::event] = newconf.loglevel.event.forward; + forwardMap[Logger::debug] = newconf.loglevel.debug.forward; + forwardMap[Logger::spam] = newconf.loglevel.spam.forward; + _fw.setForwardMap(forwardMap); + + if (newconf.logserver.port != _logPort) { + _logPort = newconf.logserver.port; + _newConf = true; + } + if (newconf.rotate.size > 0) { + _rotate_size = newconf.rotate.size; + } else { + LOG(config, "bad rotate.size=%d must be positive", + newconf.rotate.size); + } + if (newconf.rotate.age > 0) { + _rotate_age = newconf.rotate.age; + } else { + LOG(config, "bad rotate.age=%d must be positive", + newconf.rotate.age); + } + if (newconf.remove.totalmegabytes > 0) { + _remove_meg = newconf.remove.totalmegabytes; + } else { + LOG(config, "bad remove.totalmegabytes=%d must be positive", + newconf.remove.totalmegabytes); + } + if (newconf.remove.age > 0) { + _remove_age = newconf.remove.age; + } else { + LOG(config, "bad remove.age=%d must be positive", + newconf.remove.age); + } +} + +void +ConfSub::latch() +{ + if (_subscriber.nextConfig(0)) + configure(_handle->getConfig()); + if (_newConf) { + if (_use_logserver) { + connectToLogserver(); + } else { + connectToDevNull(); + } + } +} + +void +ConfSub::connectToLogserver() +{ + int newfd = makeconn(_logServer, _logPort); + if (newfd >= 0) { + resetFileDescriptor(newfd); + LOG(debug, "connected to logserver at %s:%d", + _logServer, _logPort); + } else { + LOG(debug, "could not connect to %s:%d", + _logServer, _logPort); + } +} + +void +ConfSub::connectToDevNull() +{ + int newfd = open("/dev/null", O_RDWR); + if (newfd >= 0) { + resetFileDescriptor(newfd); + LOG(debug, "opened /dev/null for read/write"); + } else { + LOG(debug, "error opening /dev/null (%d): %s", newfd, strerror(newfd)); + } +} + +void +ConfSub::resetFileDescriptor(int newfd) +{ + if (_logserverfd >= 0) { + close(_logserverfd); + } + _logserverfd = newfd; + _fw.setLogserverFD(newfd); + _newConf = false; +} + +void +ConfSub::closeConn() +{ + close(_logserverfd); + _logserverfd = -1; + _newConf = true; +} + +ConfSub::ConfSub(Forwarder &fw, const config::ConfigUri & configUri) + : _logPort(0), + _logserverfd(-1), + _rotate_size(INT_MAX), + _rotate_age(INT_MAX), + _remove_meg(INT_MAX), + _remove_age(3650), + _use_logserver(true), + _fw(fw), + _subscriber(configUri.getContext()), + _handle(), + _newConf(false) +{ + _logServer[0] = '\0'; + _handle = _subscriber.subscribe<LogdConfig>(configUri.getConfigId()); + _subscriber.nextConfig(0); + configure(_handle->getConfig()); + + LOG(debug, "got logServer %s", _logServer); + LOG(debug, "got handle %p", _handle.get()); +} + +ConfSub::~ConfSub() +{ + LOG(debug, "forget logServer %s", _logServer); + LOG(debug, "done ~ConfSub()"); +} + +} // namespace diff --git a/logd/src/logd/conf.h b/logd/src/logd/conf.h new file mode 100644 index 00000000000..d16c6fbf09e --- /dev/null +++ b/logd/src/logd/conf.h @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <logd/config-logd.h> +#include <vespa/config/config.h> + +namespace logdemon { + +class ConfSub { +private: + char _logServer[256]; + int _logPort; + int _logserverfd; + int _rotate_size; + int _rotate_age; + int _remove_meg; + int _remove_age; + bool _use_logserver; + Forwarder& _fw; + config::ConfigSubscriber _subscriber; + config::ConfigHandle<cloud::config::log::LogdConfig>::UP _handle; + bool _newConf; + + void connectToLogserver(); + void connectToDevNull(); + void resetFileDescriptor(int newfd); + ConfSub(const ConfSub& other); + ConfSub& operator=(const ConfSub& other); +public: + void latch(); + void closeConn(); + ConfSub(Forwarder &fw, const config::ConfigUri & configUri); + ~ConfSub(); + + int getservfd() const { return _logserverfd; } + int getRotateSize() const { return _rotate_size; } + int getRotateAge() const { return _rotate_age; } + int getRemoveMegabytes() const { return _remove_meg; } + int getRemoveAge() const { return _remove_age; } + bool useLogserver() const { return _use_logserver; } + + void configure(std::unique_ptr<cloud::config::log::LogdConfig> cfg); +}; + +} // namespace + diff --git a/logd/src/logd/conn.cpp b/logd/src/logd/conn.cpp new file mode 100644 index 00000000000..eee1076278c --- /dev/null +++ b/logd/src/logd/conn.cpp @@ -0,0 +1,47 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <netdb.h> + +#include <vespa/log/log.h> +LOG_SETUP(""); +LOG_RCSID("$Id$"); + +#include "errhandle.h" +#include <logd/config-logd.h> +#include "conn.h" +#include <vespa/config/common/exceptions.h> +#include <vespa/vespalib/net/socket_address.h> + +namespace logdemon { + +static int retryBeforeWarningCount = 20; + +int makeconn(const char *logSrvHost, int logPort) +{ + auto handle = vespalib::SocketAddress::select_remote(logPort, logSrvHost).connect(); + if (!handle) { + const char *msgfmt = "Cannot connect to logserver on %s:%d: %s"; + if (retryBeforeWarningCount > 0) { + --retryBeforeWarningCount; + LOG(debug, msgfmt, logSrvHost, logPort, strerror(errno)); + } else { + LOG(warning, msgfmt, logSrvHost, logPort, strerror(errno)); + } + return -1; + } + LOG(debug, + "Made new connection to port %d. Connected to daemon.", + logPort); + return handle.release(); +} + +} // namespace diff --git a/logd/src/logd/conn.h b/logd/src/logd/conn.h new file mode 100644 index 00000000000..029db823547 --- /dev/null +++ b/logd/src/logd/conn.h @@ -0,0 +1,9 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +// simple utility function + +namespace logdemon { + +extern int makeconn(const char *logSrvHost, int logSrvPort); + +} diff --git a/logd/src/logd/defaults-logd.cfg b/logd/src/logd/defaults-logd.cfg new file mode 100644 index 00000000000..78dc9df2595 --- /dev/null +++ b/logd/src/logd/defaults-logd.cfg @@ -0,0 +1,6 @@ +logserver.host "dev-arnej" +logserver.port 7814 +rotate.size 1024000 +rotate.age 9000 +remove.totalmegabytes 100 +remove.age 2 diff --git a/logd/src/logd/errhandle.h b/logd/src/logd/errhandle.h new file mode 100644 index 00000000000..8c0dde2d00e --- /dev/null +++ b/logd/src/logd/errhandle.h @@ -0,0 +1,35 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <stdexcept> +#include <vespa/vespalib/stllike/string.h> + +namespace logdemon { + +class MsgException : public std::exception { +private: + vespalib::string _string; +public: + MsgException(const char *s) : _string(s) {} + virtual ~MsgException() throw() {} + const char *what() const throw() { return _string.c_str(); } +}; + +class ConnectionException : public MsgException +{ +public: + ConnectionException(const char *s) : MsgException(s) {} +}; + +class SigTermException : public MsgException +{ +public: + SigTermException(const char *s) : MsgException(s) {} +}; + +class SomethingBad : public MsgException +{ +public: + SomethingBad(const char *s) : MsgException(s) {} +}; + +} // namespace diff --git a/logd/src/logd/forward.cpp b/logd/src/logd/forward.cpp new file mode 100644 index 00000000000..373f95ab742 --- /dev/null +++ b/logd/src/logd/forward.cpp @@ -0,0 +1,205 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <assert.h> + +#include <vespa/log/log.h> +LOG_SETUP(""); +LOG_RCSID("$Id$"); + +#include "errhandle.h" +#include "service.h" +#include "forward.h" +#include "vtag.h" + +namespace logdemon { + +void +Forwarder::forwardText(const char *text, int len) +{ + int wsize = write(_logserverfd, text, len); + + if (wsize != len) { + if (wsize > 0) { + LOG(warning, "only wrote %d of %d bytes to logserver", wsize, len); + } else { + LOG(warning, "problem sending data to logserver: %s", strerror(errno)); + } + + throw ConnectionException("problem sending data"); + } +} + + +void +Forwarder::sendMode() +{ + char buf[1024]; + snprintf(buf, 1024, "mode logd %s\n", VersionTag); + int len = strlen(buf); + if (len < 100) { + forwardText(buf, len); + } else { + LOG(warning, "too long mode line: %s", buf); + } +} + + +void +Forwarder::forwardLine(const char *line, const char *eol) +{ + int linelen = eol - line; + + assert(_logserverfd >= 0); + assert (linelen > 0); + assert (linelen < 1024*1024); + assert (line[linelen - 1] == '\n'); + + if (parseline(line, eol)) { + forwardText(line, linelen); + } +} + +bool +Forwarder::parseline(const char *linestart, const char *lineend) +{ + using vespalib::string; + + int llength = lineend - linestart; + + const char *fieldstart = linestart; + // time + const char *tab = strchr(fieldstart, '\t'); + if (tab == NULL || tab == fieldstart) { + LOG(spam, "bad logline no 1. tab: %.*s", + llength, linestart); + ++_badLines; + return false; + } + char *eod; + double logtime = strtod(fieldstart, &eod); + if (eod != tab) { + int fflen = tab - linestart; + LOG(spam, "bad logline first field not strtod parsable: %.*s", + fflen, linestart); + ++_badLines; + return false; + } + time_t now = time(NULL); + if (logtime - 864000 > now) { + int fflen = tab - linestart; + LOG(warning, "bad logline, time %.*s > 10 days in the future", + fflen, linestart); + ++_badLines; + return false; + } + if (logtime + 8640000 < now) { + int fflen = tab - linestart; + LOG(warning, "bad logline, time %.*s > 100 days in the past", + fflen, linestart); + ++_badLines; + return false; + } + + // hostname + fieldstart = tab + 1; + tab = strchr(fieldstart, '\t'); + if (tab == NULL) { + LOG(spam, "bad logline no 2. tab: %.*s", + llength, linestart); + ++_badLines; + return false; + } + + // pid + fieldstart = tab + 1; + tab = strchr(fieldstart, '\t'); + if (tab == NULL || tab == fieldstart) { + LOG(spam, "bad logline no 3. tab: %.*s", + llength, linestart); + return false; + } + int pid = strtol(fieldstart, &eod, 10); + // not checked - pid may not be a number after all + + // service + fieldstart = tab + 1; + tab = strchr(fieldstart, '\t'); + if (tab == NULL) { + LOG(spam, "bad logline no 4. tab: %.*s", + llength, linestart); + ++_badLines; + return false; + } + if (tab == fieldstart) { + LOG(spam, "empty service in logline: %.*s", + llength, linestart); + } + string service(fieldstart, tab-fieldstart); + + // component + fieldstart = tab + 1; + tab = strchr(fieldstart, '\t'); + if (tab == NULL || tab == fieldstart) { + LOG(spam, "bad logline no 5. tab: %.*s", + llength, linestart); + ++_badLines; + return false; + } + string component(fieldstart, tab-fieldstart); + + // level + fieldstart = tab + 1; + tab = strchr(fieldstart, '\t'); + if (tab == NULL || tab == fieldstart) { + LOG(spam, "bad logline no 6. tab: %.*s", + llength, linestart); + ++_badLines; + return false; + } + string level(fieldstart, tab-fieldstart); + LogLevel l = _levelparser.parseLevel(level.c_str()); + + // rest is freeform message, must be on this line: + if (tab > lineend) { + LOG(spam, "bad logline last tab after end: %.*s", + llength, linestart); + ++_badLines; + return false; + } + + // Check overrides + ForwardMap::iterator found = _forwardMap.find(l); + if (found != _forwardMap.end()) { + return found->second; + } + + Service *svcp = knownServices.getService(service.c_str()); + Component *cp = svcp->getComponent(component.c_str()); + cp->remember(logtime, pid); + bool retval = cp->shouldForward(l); + return retval; +} + +LogLevel +LevelParser::parseLevel(const char *level) +{ + using ns_log::Logger; + + LogLevel l = Logger::parseLevel(level); + if (l >= 0 && l <= Logger::NUM_LOGLEVELS) { + return l; + } + if (! _seenLevelMap[level]) { + LOG(warning, "unknown level '%s'", level); + _seenLevelMap.set(level, true); + } + return Logger::fatal; +} + +} // namespace diff --git a/logd/src/logd/forward.h b/logd/src/logd/forward.h new file mode 100644 index 00000000000..1973d0994fe --- /dev/null +++ b/logd/src/logd/forward.h @@ -0,0 +1,52 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <logd/service.h> +#include <vespa/vespalib/util/hashmap.h> +#include <map> + +namespace logdemon { + +typedef vespalib::HashMap<bool> SeenMap; +// Mapping saying if a level should be forwarded or not +typedef std::map<ns_log::Logger::LogLevel, bool> ForwardMap; + +class LevelParser +{ +private: + SeenMap _seenLevelMap; +public: + ns_log::Logger::LogLevel parseLevel(const char *level); + LevelParser() : _seenLevelMap(false) {} +}; + +class Forwarder +{ +private: + int _logserverfd; + ForwardMap _forwardMap; + LevelParser _levelparser; + const char *copystr(const char *b, const char *e) { + int len = e - b; + char *ret = new char[len+1]; + strncpy(ret, b, len); + ret[len] = '\0'; + return ret; + } + bool parseline(const char *linestart, const char *lineend); +public: + Services knownServices; + int _badLines; + Forwarder() : _logserverfd(-1), + _forwardMap(), + _levelparser(), + knownServices(), + _badLines(0) {} + void forwardText(const char *text, int len); + void forwardLine(const char *line, const char *eol); + void setForwardMap(const ForwardMap & forwardMap) { _forwardMap = forwardMap; } + void setLogserverFD(int fd) { _logserverfd = fd; } + int getLogserverFD() { return _logserverfd; } + void sendMode(); +}; + +} // namespace diff --git a/logd/src/logd/perform.cpp b/logd/src/logd/perform.cpp new file mode 100644 index 00000000000..a112b42f598 --- /dev/null +++ b/logd/src/logd/perform.cpp @@ -0,0 +1,323 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <sys/types.h> +#include <sys/time.h> + +#include <vespa/log/log.h> +LOG_SETUP(""); +LOG_RCSID("$Id$"); + +#include "service.h" +#include "forward.h" +#include "perform.h" +#include "cmdbuf.h" + +namespace logdemon { + +Performer::~Performer() +{ +} + +static bool +isPrefix(const char *prefix, const char *line) +{ + while (*prefix) { + if (*prefix++ != *line++) return false; + } + return true; +} + +ExternalPerformer::~ExternalPerformer() +{ +} + +void +ExternalPerformer::listStates(const char *service, const char *component) +{ + Service *svc = _services.getService(service); + Component *cmp = svc->getComponent(component); + + using ns_log::Logger; + + char buf[1024]; + int pos = snprintf(buf, 1024, "state %s %s ", service, component); + for (int i = 0; pos < 1000 && i < Logger::NUM_LOGLEVELS; i++) { + LogLevel level = static_cast<LogLevel>(i); + const char *levstate = "off"; + if (cmp->shouldLogAtAll(level)) { + if (cmp->shouldForward(level)) { + levstate = "forward"; + } else { + levstate = "store"; + } + } + pos += snprintf(buf+pos, 1024-pos, "%s=%s,", + Logger::logLevelNames[i], + levstate); + } + if (pos < 1000) { + buf[pos-1]='\n'; + _forwarder.forwardText(buf, pos); + } else { + LOG(warning, "buffer to small to list states[%s, %s]", + service, component); + } +} + +void +ExternalPerformer::doCmd(char *line) +{ + if (isPrefix("list services", line)) { + ServIter it = _services._services.iterator(); + while (it.valid()) { + char buf[1024]; + snprintf(buf, 1024, "service %s\n", it.key()); + _forwarder.forwardText(buf, strlen(buf)); + it.next(); + } + return; + } + if (isPrefix("list components ", line)) { + const char *servstr = line+5+11; + Service *svc = _services.getService(servstr); + CompIter it = svc->_components.iterator(); + while (it.valid()) { + char buf[1024]; + snprintf(buf, 1024, "component %s %s\n", servstr, it.key()); + int len = strlen(buf); + if (len < 1000) { + _forwarder.forwardText(buf, len); + } else { + LOG(warning, "buffer too small to list component %s %s", + servstr, it.key()); + } + it.next(); + } + return; + } + if (isPrefix("list states ", line)) { + char *servstr = line+5+7; + char *compstr = strchr(servstr, ' '); + if (compstr == NULL) { + Service *svc = _services.getService(servstr); + CompIter it = svc->_components.iterator(); + while (it.valid()) { + listStates(servstr, it.key()); + it.next(); + } + return; + } + *compstr++ = '\0'; + listStates(servstr, compstr); + return; + } + if (isPrefix("setallstates", line)) { + char *levmods = strchr(line, ' '); + if (levmods == NULL) { + LOG(error, "bad command: %s", line); + } else { + char *orig = strdup(line); + *levmods++ = '\0'; + doSetAllStates(levmods, orig); + free(orig); + } + return; + } + if (isPrefix("setstate ", line)) { + char *servstr = line + 9; + char *compstr = strchr(servstr, ' '); + if (compstr == NULL) { + LOG(error, "bad command: %s", line); + return; + } + *compstr++ = '\0'; + char *levmods = strchr(compstr, ' '); + if (levmods == NULL) { + LOG(error, "bad command: %s %s", line, compstr); + return; + } + *levmods++ = '\0'; + + Service *svc = _services.getService(servstr); + Component *cmp = svc->getComponent(compstr); + if (doSetState(levmods, cmp, line) == NULL) return; + + // maybe ??? + listStates(servstr, compstr); + return; + } + if (isPrefix("setdefaultstate ", line)) { + char *levmods = strchr(line, ' '); + *levmods++ = '\0'; + while (levmods) { + char *newval = strchr(levmods, '='); + if (!newval) { + LOG(error, "bad command %s: expected level=value, got %s", + line, levmods); + return; + } + *newval++ = '\0'; + + LogLevel level = _levelparser.parseLevel(levmods); + char *nextlev = strchr(newval, ','); + if (nextlev) *nextlev++ = '\0'; + if (strcmp(newval, "forward") == 0) { + Component::defaultDoForward(level); + } else if (strcmp(newval, "noforward") == 0) { + Component::defaultDontForward(level); + } else { + LOG(error, "bad command %s %s=%s: want forward/noforward", + line, levmods, newval); + return; + } + levmods = nextlev; + } + return; + } + LOG(error, "unknown command: %s", line); +} + +void +ExternalPerformer::doSetAllStates(char *levmods, char *origline) { + while (levmods) { + char *newval = strchr(levmods, '='); + if (!newval) { + LOG(error, "bad command %s : expected level=value, got %s", + origline, levmods); + return; + } + *newval++ = '\0'; + + LogLevel level = _levelparser.parseLevel(levmods); + char *nextlev = strchr(newval, ','); + if (nextlev) *nextlev++ = '\0'; + ServIter it = _services._services.iterator(); + while (it.valid()) { + Service *svc = _services.getService(it.key()); + CompIter cit = svc->_components.iterator(); + while (cit.valid()) { + Component *cmp = svc->getComponent(cit.key()); + assert(cmp != 0); + + if (strcmp(newval, "forward") == 0) { + cmp->doForward(level); + cmp->doLogAtAll(level); + } else if (strcmp(newval, "noforward") == 0) { + cmp->dontForward(level); + } else if (strcmp(newval, "store") == 0) { + cmp->dontForward(level); + cmp->doLogAtAll(level); + } else if (strcmp(newval, "off") == 0) { + cmp->dontForward(level); + cmp->dontLogAtAll(level); + } else { + LOG(error, "bad command %s: want forward/store/off, got %s", + origline, newval); + return; + } + + cit.next(); + } + it.next(); + } + + levmods = nextlev; + } +} + +char * +ExternalPerformer::doSetState(char *levmods, Component *cmp, char * line) { + while (levmods) { + char *newval = strchr(levmods, '='); + if (!newval) { + LOG(error, "bad command %s : expected level=value, got %s", + line, levmods); + return NULL; + } + *newval++ = '\0'; + + LogLevel level = _levelparser.parseLevel(levmods); + char *nextlev = strchr(newval, ','); + if (nextlev) *nextlev++ = '\0'; + if (strcmp(newval, "forward") == 0) { + cmp->doForward(level); + cmp->doLogAtAll(level); + } else if (strcmp(newval, "noforward") == 0) { + cmp->dontForward(level); + } else if (strcmp(newval, "store") == 0) { + cmp->dontForward(level); + cmp->doLogAtAll(level); + } else if (strcmp(newval, "off") == 0) { + cmp->dontForward(level); + cmp->dontLogAtAll(level); + } else { + LOG(error, "bad command %s %s=%s: want forward/store/off", + line, levmods, newval); + return NULL; + } + levmods = nextlev; + } + return levmods; +} + +void +InternalPerformer::doCmd(char *line) +{ + if (isPrefix("setstate ", line)){ + char *servstr = line + 9; + char *compstr = strchr(servstr, ' '); + if (compstr == NULL) { + LOG(error, "bad internal command: %s", line); + return; + } + *compstr++ = '\0'; + char *levmods = strchr(compstr, ' '); + if (levmods == NULL) { + LOG(error, "bad internal command: %s %s", line, compstr); + return; + } + *levmods++ = '\0'; + + // ignore services with slash in the name, invalid + if (strchr(servstr, '/') != NULL) + return; + + Service *svc = _services.getService(servstr); + Component *cmp = svc->getComponent(compstr); + + while (levmods) { + char *newval = strchr(levmods, '='); + if (!newval) { + LOG(error, "bad internal %s %s: expected level=value, got %s", + line, compstr, levmods); + return; + } + *newval++ = '\0'; + + LogLevel level = _levelparser.parseLevel(levmods); + char *nextlev = strchr(newval, ','); + if (nextlev) *nextlev++ = '\0'; + if (strcmp(newval, "forward") == 0) { + cmp->doForward(level); + } else if (strcmp(newval, "store") == 0) { + cmp->dontForward(level); + } else if (strcmp(newval, "off") == 0) { + cmp->dontForward(level); + } else { + LOG(error, "bad internal %s %s %s=%s: want forward/store/off", + line, compstr, levmods, newval); + return; + } + levmods = nextlev; + } + return; + } + LOG(error, "unknown command: %s", line); +} + +} // namespace diff --git a/logd/src/logd/perform.h b/logd/src/logd/perform.h new file mode 100644 index 00000000000..57a1fd6fd66 --- /dev/null +++ b/logd/src/logd/perform.h @@ -0,0 +1,39 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +namespace logdemon { + +class Performer +{ +protected: + LevelParser _levelparser; +public: + virtual void doCmd(char *line) = 0; + Performer() : _levelparser() {} + virtual ~Performer(); +}; + +class ExternalPerformer: public Performer +{ +private: + Forwarder& _forwarder; + Services& _services; + void listStates(const char *service, const char *component); +public: + void doCmd(char *line); + void doSetAllStates(char *levmods, char * line); + char *doSetState(char *levmods, Component *cmp, char *line); + ExternalPerformer(Forwarder& fw, Services& s) + : _forwarder(fw), _services(s) {} + ~ExternalPerformer(); +}; + +class InternalPerformer: public Performer +{ + Services& _services; +public: + void doCmd(char *line); + InternalPerformer(Services& s) : _services(s) {} + ~InternalPerformer() {} +}; + +} // namespace diff --git a/logd/src/logd/service.cpp b/logd/src/logd/service.cpp new file mode 100644 index 00000000000..0cc2d414862 --- /dev/null +++ b/logd/src/logd/service.cpp @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <sys/stat.h> +#include <assert.h> + +#include <vespa/log/log.h> +#include <vespa/log/control-file.h> +LOG_SETUP("logdemon"); +LOG_RCSID("$Id$"); + +#include "service.h" + +namespace logdemon { + +unsigned long Component::defFwd = (unsigned long)-1; + + +void +Component::doLogAtAll(LogLevel level) +{ + using ns_log::ControlFile; + + char lcfn[FILENAME_MAX]; + if (! ControlFile::makeName(_myservice, lcfn, FILENAME_MAX)) { + LOG(debug, "no logcontrol file for service '%s'", _myservice); + return; + } + try { + ControlFile foo(lcfn, ControlFile::READWRITE); + unsigned int *lstring = foo.getLevels(_logctlname); + lstring[level] = CHARS_TO_UINT(' ', ' ', 'O', 'N'); + } catch (...) { + LOG(debug, "exception changing logcontrol for %s", _myservice); + } +} + +void +Component::dontLogAtAll(LogLevel level) +{ + using ns_log::ControlFile; + + char lcfn[FILENAME_MAX]; + if (! ControlFile::makeName(_myservice, lcfn, FILENAME_MAX)) { + LOG(debug, "no logcontrol file for service '%s'", _myservice); + return; + } + try { + ControlFile foo(lcfn, ControlFile::READWRITE); + unsigned int *lstring = foo.getLevels(_logctlname); + lstring[level] = CHARS_TO_UINT(' ', 'O', 'F', 'F'); + } catch (...) { + LOG(debug, "exception changing logcontrol for %s", _myservice); + } +} + +bool +Component::shouldLogAtAll(LogLevel level) +{ + using ns_log::ControlFile; + + char lcfn[FILENAME_MAX]; + if (! ControlFile::makeName(_myservice, lcfn, FILENAME_MAX)) { + LOG(spam, "no logcontrol file for service '%s'", _myservice); + return true; + } + try { + ControlFile foo(lcfn, ControlFile::READWRITE); + unsigned int *lstring = foo.getLevels(_logctlname); + if (lstring[level] == CHARS_TO_UINT(' ', ' ', 'O', 'N')) + return true; + else + return false; + } catch (...) { + LOG(debug, "exception checking logcontrol for %s", _myservice); + } + return true; +} + + +Service::~Service() +{ + CompIter it = _components.iterator(); + while (it.valid()) { + delete it.value(); + it.next(); + } + free(_myname); +} + +Services::~Services() +{ + ServIter it = _services.iterator(); + while (it.valid()) { + delete it.value(); + it.next(); + } +} + +void +Services::dumpState(int fildesc) +{ + using ns_log::Logger; + + ServIter sit = _services.iterator(); + while (sit.valid()) { + Service *svc = sit.value(); + CompIter it = svc->_components.iterator(); + while (it.valid()) { + Component *cmp = it.value(); + char buf[1024]; + int pos = snprintf(buf, 1024, "setstate %s %s ", + sit.key(), it.key()); + for (int i = 0; pos < 1000 && i < Logger::NUM_LOGLEVELS; i++) { + LogLevel l = static_cast<LogLevel>(i); + pos += snprintf(buf+pos, 1024-pos, "%s=%s,", + Logger::logLevelNames[i], + cmp->shouldForward(l) ? "forward" : "store"); + } + if (pos < 1000) { + buf[pos-1]='\n'; + write(fildesc, buf, pos); + } else { + LOG(warning, "buffer to small to dumpstate[%s, %s]", + sit.key(), it.key()); + } + it.next(); + } + sit.next(); + } +} + + +} // namespace diff --git a/logd/src/logd/service.h b/logd/src/logd/service.h new file mode 100644 index 00000000000..a6d6ca4ff60 --- /dev/null +++ b/logd/src/logd/service.h @@ -0,0 +1,96 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +#include <assert.h> +#include <logd/config-logd.h> +#include <vespa/vespalib/util/hashmap.h> + +namespace logdemon { + +typedef ns_log::Logger::LogLevel LogLevel; + +class Component +{ + unsigned long _isforwarding; + double _lastseen; + int _lastpid; + const char *_myservice; + char *_myname; + char *_logctlname; + + Component(const Component& other); + Component& operator= (const Component& other); + + static unsigned long defFwd; +public: + static void defaultDoForward(LogLevel level) { defFwd |= (1 << level); } + static void defaultDontForward(LogLevel level) { defFwd &= ~(1 << level); } + + void doForward(LogLevel level) { _isforwarding |= (1 << level); } + void dontForward(LogLevel level) { _isforwarding &= ~(1 << level); } + bool shouldForward(LogLevel level) { + return ((_isforwarding & (1 << level)) != 0); + } + void doLogAtAll(LogLevel level); + void dontLogAtAll(LogLevel level); + bool shouldLogAtAll(LogLevel level); + Component(const char *servicename, const char *name) + : _isforwarding(defFwd), _lastseen(0.0), _lastpid(0), + _myservice(servicename), _myname(strdup(name)), + _logctlname(strdup(name)) + { + assert(ns_log::Logger::NUM_LOGLEVELS < 32); + const char *withoutprefix = strchr(name, '.'); + if (withoutprefix != NULL) { + strcpy(_logctlname, withoutprefix); + } else { + strcpy(_logctlname, ""); + } + } + ~Component() { free(_myname); free(_logctlname); } + void remember(double t, int p) { _lastseen = t; _lastpid = p; } + double lastSeen() const { return _lastseen; } + double lastPid() const { return _lastpid; } +}; + +typedef vespalib::HashMap<Component *> CompMap; +typedef vespalib::HashMap<Component *>::Iterator CompIter; + +class Service +{ +private: + char *_myname; + Service(const Service& other); + Service& operator= (const Service& other); +public: + CompMap _components; + Component *getComponent(const char *comp) { + if (! _components.isSet(comp)) { + _components.set(comp, new Component(_myname, comp)); + } + return _components[comp]; + } + Service(const char *name) + : _myname(strdup(name)), _components(NULL) {} + ~Service(); +}; + +typedef vespalib::HashMap<Service *> ServMap; +typedef vespalib::HashMap<Service *>::Iterator ServIter; + +class Services +{ +public: + ServMap _services; + Service *getService(const char *serv) { + if (! _services.isSet(serv)) { + _services.set(serv, new Service(serv)); + } + return _services[serv]; + } + Services() : _services(NULL) {} + ~Services(); + void dumpState(int fildesc); +}; + +} // namespace + diff --git a/logd/src/logd/sigterm.cpp b/logd/src/logd/sigterm.cpp new file mode 100644 index 00000000000..2cdfcae75af --- /dev/null +++ b/logd/src/logd/sigterm.cpp @@ -0,0 +1,27 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "sigterm.h" +#include <vespa/vespalib/util/signalhandler.h> + +void hook_signals() +{ + vespalib::SignalHandler::INT.hook(); + vespalib::SignalHandler::TERM.hook(); + vespalib::SignalHandler::PIPE.ignore(); +} + +bool gotSignaled() +{ + return (vespalib::SignalHandler::INT.check() || + vespalib::SignalHandler::TERM.check()); +} + +int gotSignalNumber() +{ + if (vespalib::SignalHandler::TERM.check()) { + return SIGTERM; + } + if (vespalib::SignalHandler::INT.check()) { + return SIGINT; + } + return 0; +} diff --git a/logd/src/logd/sigterm.h b/logd/src/logd/sigterm.h new file mode 100644 index 00000000000..fc86d25dbb7 --- /dev/null +++ b/logd/src/logd/sigterm.h @@ -0,0 +1,6 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +// setup for catching signals +extern void hook_signals(); +extern bool gotSignaled(); +extern int gotSignalNumber(); diff --git a/logd/src/logd/vtag.cpp b/logd/src/logd/vtag.cpp new file mode 100644 index 00000000000..17b677d4662 --- /dev/null +++ b/logd/src/logd/vtag.cpp @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace logdemon { + +#ifndef V_TAG +#define V_TAG "NOTAG" +#define V_TAG_DATE "NOTAG" +#define V_TAG_SYSTEM "NOTAG" +#define V_TAG_SYSTEM_REV "NOTAG" +#define V_TAG_BUILDER "NOTAG" +#endif + +char VersionTag[] = V_TAG; +char VersionTagDate[] = V_TAG_DATE; +char VersionTagSystem[] = V_TAG_SYSTEM; +char VersionTagSystemRev[] = V_TAG_SYSTEM_REV; +char VersionTagBuilder[] = V_TAG_BUILDER; + +} // namespace logdemon diff --git a/logd/src/logd/vtag.h b/logd/src/logd/vtag.h new file mode 100644 index 00000000000..7c78914829e --- /dev/null +++ b/logd/src/logd/vtag.h @@ -0,0 +1,13 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace logdemon { + +extern char VersionTag[]; +extern char VersionTagDate[]; +extern char VersionTagSystem[]; +extern char VersionTagSystemRev[]; +extern char VersionTagBuilder[]; + +} + diff --git a/logd/src/logd/watch.cpp b/logd/src/logd/watch.cpp new file mode 100644 index 00000000000..19dc97163e8 --- /dev/null +++ b/logd/src/logd/watch.cpp @@ -0,0 +1,444 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <glob.h> +#include <sys/stat.h> + +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +LOG_SETUP(""); +LOG_RCSID("$Id$"); + +#include "errhandle.h" +#include "sigterm.h" +#include "service.h" +#include "forward.h" +#include "conf.h" +#include "watch.h" +#include "perform.h" +#include "cmdbuf.h" + +namespace logdemon { +namespace { + +// wait until 1 second has passed since "start" +void snooze(const struct timeval &start) +{ + struct timeval sincestart; + gettimeofday(&sincestart, 0); + // compute time elapsed since start: + sincestart.tv_sec -= start.tv_sec; + sincestart.tv_usec -= start.tv_usec; + + // how many microseconds to wait: + long wait_usecs = (1000000 - sincestart.tv_usec); + wait_usecs -= (1000000 * sincestart.tv_sec); + + if (wait_usecs <= 0) { + // already used enough time, no sleep + return; + } + + struct timespec tsp; + tsp.tv_sec = (wait_usecs / 1000000); + tsp.tv_nsec = (wait_usecs % 1000000) * 1000; + + if (nanosleep(&tsp, NULL) != 0 && errno != EINTR) { + LOG(error, "nanosleep %ld s %ld ns failed: %s", + (long)tsp.tv_sec, (long)tsp.tv_nsec, strerror(errno)); + throw SomethingBad("nanosleep failed"); + } +} + +int elapsed(struct timeval &start) { + struct timeval now; + gettimeofday(&now, 0); + int diffsecs = now.tv_sec - start.tv_sec; + if (now.tv_usec < start.tv_usec) { + --diffsecs; + } + return diffsecs; +} + +} // namespace logdemon::<unnamed> + +static const int bufsiz = 1024*1024; + +Watcher::Watcher(ConfSub &cfs, Forwarder &fw) + : _buffer(new char[bufsiz]), + _confsubscriber(cfs), + _forwarder(fw), + _wfd(-1) +{ + if (_buffer == NULL) { + LOG(error, "could not allocate 1MB memory"); + throw SomethingBad("out of memory"); + } +} + +Watcher::~Watcher() +{ + delete[] _buffer; + if (_wfd >= 0) { + LOG(debug, "~Watcher closing %d", _wfd); + close(_wfd); + } +} + + +struct donecache { + dev_t st_dev; /* device */ + ino_t st_ino; /* inode number */ + off_t offset; + bool valid; +}; + +class StateSaver { +public: + StateSaver(); + ~StateSaver(); + void saveState(const donecache&, Services&); + bool loadState(donecache&, Forwarder&); + void doFullsave() { _cachecounter = 300; } +private: + int _savefd; + int _cachecounter; +}; + +void StateSaver::saveState(const donecache& already, + Services& currentserv) +{ + if (_savefd < 0) { + // cannot save state + return; + } + off_t origin = 0; + lseek(_savefd, origin, SEEK_SET); + if (write(_savefd, &already, sizeof(already)) != sizeof(already)) { + LOG(error, "error writing to donecachefile: %s", strerror(errno)); + close(_savefd); + _savefd = -1; + } else if (++_cachecounter > 300) { + currentserv.dumpState(_savefd); + off_t here = lseek(_savefd, (off_t)0, SEEK_CUR); + LOG(debug, "cached already %d/%d %d, trunc at %d", + (int)already.st_dev, (int)already.st_ino, + (int)already.offset, (int)here); + if (here == (off_t)-1) { + LOG(error, "lseek failed: %s", strerror(errno)); + } else if (ftruncate(_savefd, here) < 0) { + LOG(error, "ftruncate %d=%d failed: %s", + _savefd, (int)here, strerror(errno)); + } + _cachecounter = 0; + } +} + +bool +StateSaver::loadState(donecache& already, Forwarder& fwd) +{ + if (_savefd >= 0 && + read(_savefd, &already, sizeof(already)) == sizeof(already)) + { + InternalPerformer iperf(fwd.knownServices); + CmdBuf filebuf; + while (filebuf.readFile(_savefd)) { + while (filebuf.hasCmd()) { + filebuf.doCmd(iperf); + } + } + return true; + } else { + return false; + } +} + +StateSaver::StateSaver() : _savefd(-1), _cachecounter(300) +{ + _savefd = open("var/db/vespa/logd.donestate", O_RDWR|O_CREAT, 0664); + if (_savefd < 0) { + LOG(warning, "could not open var/db/vespa/logd.donestate: %s", + strerror(errno)); + } +} + +StateSaver::~StateSaver() +{ + LOG(debug, "~StateSaver closing %d", _savefd); + if (_savefd >= 0) { + close(_savefd); + } +} + +void +Watcher::watchfile() +{ + struct donecache already; + + char *target = getenv("VESPA_LOG_TARGET"); + if (target == NULL || strncmp(target, "file:", 5) != 0) { + LOG(error, "expected VESPA_LOG_TARGET (%s) to be a file: target", + target); + throw SomethingBad("bad log target"); + } + const char *filename = target+5; + + if (strlen(filename) + 50 > FILENAME_MAX) { + LOG(error, "too long filename '%s'", filename); + throw SomethingBad("too long filename in watchfile"); + } + + ExternalPerformer performer(_forwarder, _forwarder.knownServices); + CmdBuf cmdbuf; + + StateSaver dcf; + if (dcf.loadState(already, _forwarder)) { + already.valid = true; + } + + _forwarder.sendMode(); + + int sleepcount = 0; + time_t created = 0; + + again: + // XXX should close and/or check _wfd first ? + _wfd = open(filename, O_RDONLY|O_CREAT, 0664); + if (_wfd < 0) { + LOG(error, "open(%s) failed: %s", filename, + strerror(errno)); + throw SomethingBad("could not create or open logfile"); + } + + bool rotate = false; + struct timeval rotStart; + off_t offset = 0; + + while (1) { + struct stat sb; + if (fstat(_wfd, &sb) != 0) { + LOG(error, "fstat(%s) failed: %s", filename, + strerror(errno)); + throw SomethingBad("fstat failed"); + } + if (created == 0) { + created = sb.st_ctime; + } + if (already.valid) { + if (sb.st_dev == already.st_dev && + sb.st_ino == already.st_ino && + sb.st_size >= already.offset) + { + offset = already.offset; + } + // only update offset from cache once: + already.valid = false; + } + + if (sb.st_size < offset) { + // this is bad, maybe somebody else truncated the file + LOG(error, "file mysteriously shrunk %d -> %d", + (int)offset, (int)sb.st_size); + return; + } + + struct timeval tickStart; + gettimeofday(&tickStart, 0); + + if (sb.st_size > offset) { + lseek(_wfd, offset, SEEK_SET); + ssize_t rsize = read(_wfd, _buffer, bufsiz - 1); + if (rsize > 0) { + _buffer[rsize] = '\0'; + char *l = _buffer; + char *nnl = (char *)memchr(_buffer, '\n', rsize); + if (nnl == NULL && rsize == bufsiz - 1) { + // incredibly long block without any newline ? + LOG(error, "no newline in %ld bytes, skipping", + static_cast<long>(rsize)); + offset += rsize; + } + while (nnl != NULL && elapsed(tickStart) < 1) { + ++nnl; + _forwarder.forwardLine(l, nnl); + ssize_t wsize = nnl - l; + offset += wsize; + l = nnl; + nnl = strchr(l, '\n'); + } + } else { + LOG(error, "could not read from %s: %s", + filename, strerror(errno)); + throw SomethingBad("read failed"); + } + } + + already.offset = offset; + already.st_dev = sb.st_dev; + already.st_ino = sb.st_ino; + + time_t now = time(NULL); + bool wantrotate = (now > created + _confsubscriber.getRotateAge()) + || (sb.st_size > _confsubscriber.getRotateSize()); + + if (rotate) { + int rotTime = elapsed(rotStart); + if (rotTime > 59 || + (sb.st_size == offset && rotTime > 4)) + { + removeOldLogs(filename); + if (sb.st_size != offset) { + LOG(warning, "logfile rotation incomplete after %d s (dropping %" PRIu64 " bytes)", + rotTime, sb.st_size - offset); + } else { + LOG(debug, "logfile rotation complete after %d s", rotTime); + } + created = now; + rotate = false; + close(_wfd); + goto again; + } + } else if (stat(filename, &sb) != 0 + || sb.st_dev != already.st_dev + || sb.st_ino != already.st_ino) + { + LOG(warning, "logfile rotated away underneath"); + created = now; + close(_wfd); + goto again; + } else if (wantrotate) { + rotate = true; + gettimeofday(&rotStart, 0); + LOG(debug, "preparing to rotate logfile, old logfile size %d, age %d seconds", + (int)offset, (int)(now-created)); + char newfn[FILENAME_MAX]; + int l = strlen(filename); + strcpy(newfn, filename); + struct tm *nowtm = gmtime(&now); + if (strftime(newfn+l, FILENAME_MAX-l-1, + "-%Y-%m-%d.%H-%M-%S", nowtm) < 10) + { + LOG(error, "could not strftime"); + throw SomethingBad("strftime failed"); + } + + if (rename(filename, newfn) != 0) { + LOG(error, "could not rename logfile %s -> %s: %s", + filename, newfn, strerror(errno)); + throw SomethingBad("rename failed"); + } else { + LOG(debug, "old logfile name: %s", newfn); + } + } + + dcf.saveState(already, _forwarder.knownServices); + + _confsubscriber.latch(); + + if (_confsubscriber.useLogserver()) { + cmdbuf.maybeRead(_forwarder.getLogserverFD()); + while (cmdbuf.hasCmd()) { + cmdbuf.doCmd(performer); + // in case forwarding changes + dcf.doFullsave(); + } + } + + if (gotSignaled()) { + throw SigTermException("caught signal"); + } + snooze(tickStart); + if (gotSignaled()) { + throw SigTermException("caught signal"); + } + if (++sleepcount > 99) { + if (_forwarder._badLines) { + LOG(info, "seen %d bad loglines in %d iterations", + _forwarder._badLines, sleepcount); + _forwarder._badLines = 0; + sleepcount=0; + } + } + } +} + +static int globerrfunc(const char *path, int errno_was) +{ + LOG(warning, "glob %s: %s", path, strerror(errno_was)); + return 0; +} + +void +Watcher::removeOldLogs(const char *prefix) +{ + const char suffix[] = "-*-*-*.*-*-*"; + char pattern[FILENAME_MAX]; + int l = strlen(prefix) + sizeof(suffix) + 20; + if (l > FILENAME_MAX) { + LOG(error, "too long filename prefix in removeOldLog()"); + return; + } + + strcpy(pattern, prefix); + strcat(pattern, suffix); + + glob_t myglob; + myglob.gl_pathc = 0; + myglob.gl_offs = 0; + myglob.gl_flags = 0; + myglob.gl_pathv = NULL; + + off_t totalsize = 0; + + int globresult = glob(pattern, 0, &globerrfunc, &myglob); + if (globresult == 0) { + for (int i = 0; i < (int)myglob.gl_pathc; i++) { + const char *fname = myglob.gl_pathv[myglob.gl_pathc-i-1]; + + struct stat sb; + if (stat(fname, &sb) != 0) { + LOG(warning, "cannot stat %s: %s", fname, strerror(errno)); + continue; + } + if (S_ISREG(sb.st_mode)) { + if (sb.st_mtime + + _confsubscriber.getRemoveAge() * 86400 < time(NULL)) + { + LOG(info, "removing %s, too old (%f days)", fname, + (double)(time(NULL)-sb.st_mtime)/86400.0); + + if (unlink(fname) != 0) { + LOG(warning, "cannot remove %s: %s", + fname, strerror(errno)); + } + continue; + } + totalsize += sb.st_size; + if (totalsize > (_confsubscriber.getRemoveMegabytes() + * 1048576LL)) + { + LOG(info, "removing %s, total size (%" PRId64 ") too big", + fname, static_cast<int64_t>(totalsize)); + if (unlink(fname) != 0) { + LOG(warning, "cannot remove %s: %s", + fname, strerror(errno)); + } + } + } else { + LOG(warning, "not a regular file: %s", fname); + } + } + } else if (globresult == GLOB_NOMATCH) { + LOG(info, "no old logfiles matching %s", pattern); + } else { + LOG(warning, "glob %s failed: %d", pattern, globresult); + } + globfree(&myglob); +} + +} // namespace diff --git a/logd/src/logd/watch.h b/logd/src/logd/watch.h new file mode 100644 index 00000000000..6385e725104 --- /dev/null +++ b/logd/src/logd/watch.h @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once +namespace logdemon { + +class Watcher +{ +private: + char *_buffer; + ConfSub& _confsubscriber; + Forwarder& _forwarder; + int _wfd; + + Watcher(const Watcher& other); + Watcher& operator=(const Watcher& other); +public: + Watcher(ConfSub &cfs, Forwarder &fw); + ~Watcher(); + + void watchfile(); + void removeOldLogs(const char *prefix); +}; + +} // namespace diff --git a/logd/src/main/resources/configdefinitions/logd.def b/logd/src/main/resources/configdefinitions/logd.def new file mode 100644 index 00000000000..664c949fdc2 --- /dev/null +++ b/logd/src/main/resources/configdefinitions/logd.def @@ -0,0 +1,33 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=cloud.config.log + +## Host to contact the logserver on. +logserver.host string default="localhost" + +## Port to contact the logserver on. +logserver.port int default=5821 + +## Forward to a logserver. If false, logserver.host and logserver.port are irrelevant +logserver.use bool default=true + +## Loglevel config whether they should be stored and/or forwarded +loglevel.fatal.forward bool default=true +loglevel.error.forward bool default=true +loglevel.warning.forward bool default=true +loglevel.config.forward bool default=true +loglevel.info.forward bool default=true +loglevel.event.forward bool default=false +loglevel.debug.forward bool default=false +loglevel.spam.forward bool default=false + +## when the logfile reaches this size, rotate it +rotate.size int default=10000000 + +## when the logfile reaches this age (in seconds), rotate it +rotate.age int default=86400 + +## remove old logfiles so total size does not exceed this (in megabytes) +remove.totalmegabytes int default=1000 + +## remove old logfiles older than this (in days) +remove.age int default=31 diff --git a/logd/src/testlist.txt b/logd/src/testlist.txt new file mode 100644 index 00000000000..7b16a4e21f2 --- /dev/null +++ b/logd/src/testlist.txt @@ -0,0 +1,3 @@ +tests/forward +tests/info +tests/rotate diff --git a/logd/src/tests/forward/.gitignore b/logd/src/tests/forward/.gitignore new file mode 100644 index 00000000000..a6c6ba525b6 --- /dev/null +++ b/logd/src/tests/forward/.gitignore @@ -0,0 +1,5 @@ +/.depend +/Makefile +/forward.txt +/forward_test +logd_forward_test_app diff --git a/logd/src/tests/forward/CMakeLists.txt b/logd/src/tests/forward/CMakeLists.txt new file mode 100644 index 00000000000..c71b8d89c6e --- /dev/null +++ b/logd/src/tests/forward/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(logd_forward_test_app + SOURCES + forward.cpp + DEPENDS + logd +) +vespa_add_test(NAME logd_forward_test_app COMMAND logd_forward_test_app) diff --git a/logd/src/tests/forward/forward.cpp b/logd/src/tests/forward/forward.cpp new file mode 100644 index 00000000000..332b4661edc --- /dev/null +++ b/logd/src/tests/forward/forward.cpp @@ -0,0 +1,64 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/log.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <logd/forward.h> +#include <sstream> + +using ns_log::Logger; +using namespace logdemon; + +struct ForwardFixture { + Forwarder & forwarder; + int fd; + const std::string fname; + const std::string logLine; + ForwardFixture(Forwarder & fw, const std::string & fileName) + : forwarder(fw), + fd(-1), + fname(fileName), + logLine(createLogLine()) + { + fd = open(fileName.c_str(), O_CREAT | O_TRUNC | O_WRONLY, 0777); + forwarder.setLogserverFD(fd); + } + ~ForwardFixture() { + close(fd); + } + + const std::string createLogLine() { + FastOS_Time timer; + timer.SetNow(); + std::stringstream ss; + ss << std::fixed << timer.Secs(); + ss << "\texample.yahoo.com\t7518/34779\tlogd\tlogdemon\tevent\tstarted/1 name=\"logdemon\"\n"; + return ss.str(); + } + + void verifyForward(bool doForward) { + const std::string & line(logLine); + forwarder.forwardLine(line.c_str(), line.c_str() + line.length()); + fsync(fd); + int rfd = open(fname.c_str(), O_RDONLY); + char *buffer[2048]; + ssize_t bytes = read(rfd, buffer, 2048); + ssize_t expected = doForward ? line.length() : 0; + EXPECT_EQUAL(expected, bytes); + close(rfd); + } +}; + +TEST_FF("require that forwarder forwards if set", Forwarder(), ForwardFixture(f1, "forward.txt")) { + ForwardMap forwardMap; + forwardMap[Logger::event] = true; + f1.setForwardMap(forwardMap); + f2.verifyForward(true); +} + +TEST_FF("require that forwarder does not forward if not set", Forwarder(), ForwardFixture(f1, "forward.txt")) { + ForwardMap forwardMap; + forwardMap[Logger::event] = false; + f1.setForwardMap(forwardMap); + f2.verifyForward(false); +} + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/logd/src/tests/info/.gitignore b/logd/src/tests/info/.gitignore new file mode 100644 index 00000000000..9ca863402a5 --- /dev/null +++ b/logd/src/tests/info/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +info_test +logd_info_test_app diff --git a/logd/src/tests/info/CMakeLists.txt b/logd/src/tests/info/CMakeLists.txt new file mode 100644 index 00000000000..3be1ab3cb49 --- /dev/null +++ b/logd/src/tests/info/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(logd_info_test_app + SOURCES + info.cpp + DEPENDS +) +vespa_add_test(NAME logd_info_test_app NO_VALGRIND COMMAND logd_info_test_app) diff --git a/logd/src/tests/info/DESC b/logd/src/tests/info/DESC new file mode 100644 index 00000000000..cb24f291863 --- /dev/null +++ b/logd/src/tests/info/DESC @@ -0,0 +1 @@ +Print some info about this module. diff --git a/logd/src/tests/info/FILES b/logd/src/tests/info/FILES new file mode 100644 index 00000000000..62e5403ecb5 --- /dev/null +++ b/logd/src/tests/info/FILES @@ -0,0 +1 @@ +info.cpp diff --git a/logd/src/tests/info/info.cpp b/logd/src/tests/info/info.cpp new file mode 100644 index 00000000000..c674bd74d88 --- /dev/null +++ b/logd/src/tests/info/info.cpp @@ -0,0 +1,14 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/log.h> +LOG_SETUP("info_test"); +#include <vespa/fastos/fastos.h> +#include <vespa/vespalib/testkit/testapp.h> + +TEST_SETUP(Test); + +int +Test::Main() +{ + TEST_INIT("info_test"); + TEST_DONE(); +} diff --git a/logd/src/tests/rotate/.gitignore b/logd/src/tests/rotate/.gitignore new file mode 100644 index 00000000000..5e5dafe459c --- /dev/null +++ b/logd/src/tests/rotate/.gitignore @@ -0,0 +1,11 @@ +*.pid +.depend +Makefile +logd.cfg +dummylogger +dummyserver +logserver.port +rotate_test +vespa.log* +logd_dummylogger_app +logd_dummyserver_app diff --git a/logd/src/tests/rotate/CMakeLists.txt b/logd/src/tests/rotate/CMakeLists.txt new file mode 100644 index 00000000000..0398b14bcf4 --- /dev/null +++ b/logd/src/tests/rotate/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(logd_dummyserver_app + SOURCES + dummyserver.cpp + DEPENDS +) +vespa_add_executable(logd_dummylogger_app + SOURCES + dummylogger.cpp + DEPENDS +) +vespa_add_test(NAME logd_rotate_test NO_VALGRIND COMMAND sh rotate_test.sh BENCHMARK) diff --git a/logd/src/tests/rotate/DESC b/logd/src/tests/rotate/DESC new file mode 100644 index 00000000000..202911eb96e --- /dev/null +++ b/logd/src/tests/rotate/DESC @@ -0,0 +1,2 @@ +Test that logd will drop data in favor of rotation if log server is +too slow. diff --git a/logd/src/tests/rotate/FILES b/logd/src/tests/rotate/FILES new file mode 100644 index 00000000000..8a7b941417d --- /dev/null +++ b/logd/src/tests/rotate/FILES @@ -0,0 +1,4 @@ +rotate_test.sh +dummyserver.cpp +dummylogger.cpp +defaults-logd.cfg diff --git a/logd/src/tests/rotate/create_configfile.sh b/logd/src/tests/rotate/create_configfile.sh new file mode 100755 index 00000000000..3211218c369 --- /dev/null +++ b/logd/src/tests/rotate/create_configfile.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +read port < logserver.port + +cat > logd.cfg << EOF +logserver.host "localhost" +logserver.port $port +rotate.size 1024 +rotate.age 1000 +remove.totalmegabytes 100 +remove.age 30 +EOF diff --git a/logd/src/tests/rotate/dummylogger.cpp b/logd/src/tests/rotate/dummylogger.cpp new file mode 100644 index 00000000000..375ce4026b3 --- /dev/null +++ b/logd/src/tests/rotate/dummylogger.cpp @@ -0,0 +1,29 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdio.h> +#include <stdlib.h> +#include <time.h> +#include <vespa/log/log.h> +LOG_SETUP("dummylogger"); + +int +main(int argc, char **argv) +{ + (void) argc; + (void) argv; + int cnt = 100000; + for (int i = 0; i < cnt; ++i) { + LOG(info, "This is log message %d/%d", i + 1, cnt); + if ((i % 100) == 0) { + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = 250000000; + nanosleep(&t, 0); + } + if ((i % 1000) == 999) { + int percent = (i + 1) / 1000; + fprintf(stdout, "log progress: %d%%\n", percent); + fflush(stdout); + } + } + return 0; +} diff --git a/logd/src/tests/rotate/dummyserver.cpp b/logd/src/tests/rotate/dummyserver.cpp new file mode 100644 index 00000000000..c36bf4597cb --- /dev/null +++ b/logd/src/tests/rotate/dummyserver.cpp @@ -0,0 +1,50 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <time.h> +#include <fcntl.h> +#include <vespa/vespalib/net/socket_address.h> + +void error(const char *msg) +{ + perror(msg); + exit(1); +} + +int main(int /*argc*/, char ** /*argv*/) +{ + auto handle = vespalib::SocketAddress::select_local(0).listen(); + if (!handle) { + error("ERROR: could not listen to server port"); + } + int portno = vespalib::SocketAddress::address_of(handle.get()).port(); + printf("Got port %d", portno); + int fd = open("logserver.port", O_CREAT | O_WRONLY, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + char out[6]; + sprintf(out, "%d\n", portno); + write(fd, out, sizeof(out)); + close(fd); + sockaddr_storage cli_addr; + socklen_t clilen = sizeof(cli_addr); + int newsockfd = accept(handle.get(), + (struct sockaddr *) &cli_addr, + &clilen); + if (newsockfd < 0) + error("ERROR on accept"); + char buffer[1024]; + while (true) { + ssize_t n = read(newsockfd, buffer, sizeof(buffer)); + if (n < 0) error("ERROR reading from socket"); + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = 200000000; + nanosleep(&t, 0); + } + return 0; +} diff --git a/logd/src/tests/rotate/rotate_test.sh b/logd/src/tests/rotate/rotate_test.sh new file mode 100755 index 00000000000..97962e0485c --- /dev/null +++ b/logd/src/tests/rotate/rotate_test.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +export VESPA_LOG_TARGET=file:vespa.log + +rm -rf vespa.log* + +./logd_dummyserver_app & +echo $! > dummyserver.pid + +sleep 5 + +./create_configfile.sh + + +export VESPA_CONFIG_ID=file:logd.cfg +../../apps/logd/logd & +echo $! > logd.pid + +./logd_dummylogger_app + +echo "stopping servers..." +cat *.pid | xargs kill + +echo "looking for incomplete log forwarding due to slow log server..." +if grep incomplete vespa.log*; then + echo "OK" + exit 0 +else + echo "FAIL" + exit 1 +fi diff --git a/logd/testrun/.gitignore b/logd/testrun/.gitignore new file mode 100644 index 00000000000..faed45bc94a --- /dev/null +++ b/logd/testrun/.gitignore @@ -0,0 +1,10 @@ +test-report.html +test-report.html.* +test.*.*.desc +test.*.*.file.* +test.*.*.files.html +test.*.*.log +tmp.* +xsync.log +/test.*.*.result +Makefile |