diff options
| author | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
|---|---|---|
| committer | Jon Bratseth <bratseth@yahoo-inc.com> | 2016-06-15 23:09:44 +0200 |
| commit | 72231250ed81e10d66bfe70701e64fa5fe50f712 (patch) | |
| tree | 2728bba1131a6f6e5bdf95afec7d7ff9358dac50 /vespalog | |
Publish
Diffstat (limited to 'vespalog')
138 files changed, 11668 insertions, 0 deletions
diff --git a/vespalog/.gitignore b/vespalog/.gitignore new file mode 100644 index 00000000000..ad5e60bcc0f --- /dev/null +++ b/vespalog/.gitignore @@ -0,0 +1,15 @@ +.classpath +.project +archive +bin +build +include +javadoc +javamake.pdb +staging +target +testLogs +vespalog.iml +/pom.xml.build +Makefile +Testing diff --git a/vespalog/CMakeLists.txt b/vespalog/CMakeLists.txt new file mode 100644 index 00000000000..17a91b8bbee --- /dev/null +++ b/vespalog/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + vespadefaults + fastos + + LIBS + src/vespa/log + + APPS + src/logctl + src/logger + + TESTS + src/test + src/test/simple + src/test/threads +) + +vespa_install_script(src/logfmt/logfmt.pl logfmt bin) +install(FILES src/logfmt/logfmt.1 DESTINATION man/man1) diff --git a/vespalog/INSTALL b/vespalog/INSTALL new file mode 100644 index 00000000000..f148a0c18a4 --- /dev/null +++ b/vespalog/INSTALL @@ -0,0 +1,17 @@ +1. Make and install fastos with the correct options. + +2. cd src + +3. ./configure --fastos-dir <fastos-install-dir> [options] + Additional optons: + --install-dir <val> [..] + +4. make + +5. make install + +This will install the following: + +bin/logctl +include/log/log.h +lib/liblog.a diff --git a/vespalog/OWNERS b/vespalog/OWNERS new file mode 100644 index 00000000000..67cd2820bb8 --- /dev/null +++ b/vespalog/OWNERS @@ -0,0 +1 @@ +arnej27959 diff --git a/vespalog/README b/vespalog/README new file mode 100644 index 00000000000..697e02c6dbc --- /dev/null +++ b/vespalog/README @@ -0,0 +1,183 @@ +This is the unified logging library for Vespa. + +To use in a C++ program, make and install this module, and do the +following in your program(s): + +1. In each .cpp file, add the following near the top, preferably +before any other non-system include: + +#include "log/log.h" +LOG_SETUP("name"); + +Where "name" should be replaced with the proper COMPONENT NAME (see +separate section about component names). + +NOTE: Naming convention: In LOG_SETUP, only use lower-case letters, +and use "-" to separate words if necessary, ie: + LOG_SETUP(".my-lib.bla-bla") +Do not use spaces. + +This will define a macro named LOG that performs logging. The reason +for doing this before any other non-system includes, is that you will +be able to use the LOG macro in your included .h files as well +then. If you do not LOG from any of your included .h files, it doesn't +matter. + +2. Optionally, add a LOG_RCSID("$Id$") line after LOG_SETUP. This will +cause debug messages to be tagged with the rcs version of the file +as well (in addition to the normal filename and line number). + +3. At each location where you want to make a log message, use the LOG + macro like this: + +LOG(<level>, "printf-format" [, optional printf arguments]); + +<level> can be one of : + fatal - deprecated, use "error" + error - An error condition + warning - A problem that is not severe enough to warrant an error message. + + event - Information in machine-readable (fixed) format. + event should be used instead of info where possible + + info - Information in non-machine-readable format (ie for humans only) + info should not be used for error conditions, but is fine for + progress reports and similar (unless an event is defined for it!) + + debug - Debug messages, intented for developers (ie you!) + only. Debug messages are normally suppressed, and will use + very little resources unless they are enabled. Arguments + to suppressed log messages are _not_ evaluated. + + Debug messages should contain whatever information you need + while debugging your code to make sure it behaves like it should, + and to track down any suspected problems. + + spam - Spammy debug messages, intented for developers only. This is intended + for VERY spammy debug output, such as printing 100 lines per + document processed. + +4. When linking the program, link with -llog. + + +COMPONENT NAMES + +Component names are hierarchical, a "." separates each level. The root +component name must be IDENTICAL in all files for a particular +binary. The root component name should be identical to the name of the +final binary. + +There is some support in LOG_SETUP for simplifying this: + +1. An empty argument to LOG_SETUP will use whatever root component name + is defined in any other .cpp file. + +2. If the component name used in LOG_SETUP starts with a ".", it will be + appended to the root component name that is defined in any other + .cpp file. + + +The normal way to use this is then: + +1. In applications + + a) Find the .cpp file that defines the main() function, and use + LOG_SETUP("binary-name"); + in that file. + + b) For all other .cpp files in the application that only want to use + the root name, the following will then be equivalent, and the short + form is preferable if you are ever going to rename the program: + + LOG_SETUP(""); + or + LOG_SETUP("binary-name"); + + +2. In libraries or sub-modules + + Decide on a name for the library or sub-module, and use this: + LOG_SETUP(".name") (where "name" is the name of the library + or sub-module) + + If you want more levels you can also do + LOG_SETUP(".level1.level2"); + + And so on. + +The reason it is useful to have more than one component name in an +application is that it is possible to change at _runtime_ which log +levels are suppressed in each individual component. This is especially +useful for enabling debug output only in parts of a program. + +Some hypothetical examples: + +In the application "fdispatch" there could be some files that have +code primarily for query parsing. It also links with the library "fnet" +for communicating. + +in main.cpp the following is done: +LOG_SETUP("fdispatch"); +LOG_RCSID("$Id$"); + +In the .cpp files that worry about query parsing, the following is done: +LOG_SETUP(".query-parse"); +LOG_RCSID("$Id$"); + +In fnet, all files do at least this: +LOG_SETUP(".fnet"); +LOG_RCSID("$Id$"); + +It is also possible to sub-divide fnet into more components, and do +something like this in some of the fnet files: +LOG_SETUP(".fnet.network"); +LOG_RCSID("$Id$"); + + +OUTPUT FORMAT + +The output log format is tab-separated fields, and the following +fields exist, always in this order: + +<time> <host> <process/thread id> <service> <component> <level> <message> + +time: Seconds since 1970-01-01 UTC, optionally with fractional seconds. + Examples: "1088516763", "1088516763.808240" + Time is always output with a "neutral" time zone (UTC). + +host: Hostname as given by "uname -n", or "-" for no hostname. + +process/thread-id: pid[/tid] of the process/thread that procuded the log. + For systems where pid uniquely defines the logging thread, or where + there are no threads, only the pid will be logged. If a tid is logged + in addition, the format will be "pid/tid". + +service: Service name as given by Vespa configuration setup, or "-" for + no service name + +component: Application decided logging component name. Hierarchical with + "." between each level, top level should always be the name of the + binary that produced the log. + +level (see above) + +message: The log message itself. Any content is possible, but the following + quoting rules are enforced by the library: + + +QUOTING + +The library will forcibly quote all characters that have an ASCII +value < 32 (and also 92) according to the following table, + +9 -> \t +10 -> \n +13 -> \r +92 -> \\ (ie \ is escaped as \\) + +All other values that must be quoted will be quoted as \xXX where X is +a hexadecimal number, eg 1 -> \x01, 2 -> \x02, 17 -> \x11 and so on. + +Any code reading logs should be prepared to see \xXX style quoting also +for the characters that have defined alphabetic quoting shortcuts (ie +9, 10, 13 and 92)! diff --git a/vespalog/README.events b/vespalog/README.events new file mode 100644 index 00000000000..7d9e70eae97 --- /dev/null +++ b/vespalog/README.events @@ -0,0 +1,158 @@ +Quick Specification for VESPA Log Events +======================================== + +Events are a machine-readable form of log messages, intended for +automated monitoring. + +Each event has an event type, a version and an optional payload. + +In the log format, event types are expressed as a single word, +versions as a simple integer, and the payload as a set of key=value +pairs. + +The event payload is backslash-quoted just like log messages are in +general. This means that events may be double-quoted during +transport. Double-quote delimiters are not supported. + +These are the supported event types and their usage. You can find +more details about each of the event types further below. + + Event Type Description +============================================================================ + starting Sent by a process starting another process + started Sent by a process when it has started + stopping Sent by a process that will exit + stopped Sent by the calling process when a sub-process has exited + reloading Sent when some form of reloading starts + reloaded Sent when the reload is complete + crash Sent by the calling process when a sub-process crashes + progress For tracking any type of progress + count For tracking any type of counter (queries served etc) + value For tracking any type of value (qps, queue-length, etc..) + collection For tracking events related to collections (collection out of sync etc) + +Event: starting +--------------- +Payload: name=<name> + +This event is sent by processes when they are about to start another +process. Typical for, but not limited to, shell scripts. This event +is not required to track processes, but is useful in cases where a +sub-process may fail during startup. + +Example: +starting name=fdispatch + + +Event: started +-------------- +Payload: name=<name> +The "started" event is sent by a process that just started up. + +Example: +started name=fdispatch + + +Event: stopping +--------------- +Payload: name=<name> why=<why> +The "stopping" event is sent by a process that is about to exit. + +Example: +stopping name=fdispatch why="clean exit" + + +Event: stopped +-------------- +Payload: name=<name> pid=<pid> exitcode=<exitcode> + +This event is sent by a process monitoring when a sub-process exits. + +Example: +stopped name=fdispatch pid=76512 + + +Event: reloading +---------------- +Payload: name=<name> + +This event is used to signal that a process is about to start some +kind of reloading operation. + +The "name" payload attribute says what kind of reload operation that +is being done, and is specific for each library/application. The +"reloading" event is optional for reloads, for a simple reload that +doesn't fail, you can use "reloaded" directly. + +Example: +reloading name=partitions + + +Event: reloaded +--------------- +The "reloaded" event is sent after the reload is complete (and +successful). Any "reloading" event must have a matching "reloaded" event + +Example: + +reloaded name=crawler.configfile:$VESPA_HOME/var/mms/docproccfg.xml + + +Event: crash +------------ +Payload: name=<appname> pid=<pid> signal=<signal> + +Submitted by a process monitoring a sub-process when the sub-process +crashes (dumps core etc.) + +Example: + +crash name=fastcrawler pid=12345 signal=11 + + +Event: progress +--------------- +Payload: name=<name> value=<value> [total=<total>] + +Somewhat general event for tracking progress. The <name> is specific +to each library/application. + +Example: +progress name=indexing.autoindex value=17 total=90 + + +Event: count +------------ +Payload: name=<name> value=<value> + +Somewhat general event for counting stuff. As with "progress", the +<name> is specific to each library/application. Counters are asssumed +to increase with time, counting the number of events since the program +was started for example. + + +Event: value +------------ +Payload: name=<name> value=<value> + +General event for values. "Value is for values that cannot be counts" +Typical values are queue lengths, transaction frequencies and so on. + + +Examples: +count name=queue-length value=2346235 + +Event: collection +----------------- + +Payload: collectionid=<id> name=<name> <params> + +Represents events related to collections. The name describes what +happened to the collection ("syncrows", "created", etc). params +is a string on the form param1=... param2=... that holds various +parameters for the event. For "syncrows", the parameters are: +column, badrow, goodrow. + +Examples: +collection collectionid=1111111111 name=syncrows column=1 badrow=0 goodrow=1 + diff --git a/vespalog/README.server.protocol b/vespalog/README.server.protocol new file mode 100644 index 00000000000..7d89bf625c0 --- /dev/null +++ b/vespalog/README.server.protocol @@ -0,0 +1,46 @@ +Protocol for talking to the log server: + +Pure line-based text protocol. Tcp to some port on the log server: + +From log daemon (or anything, really) to server: + +service <service> +component <service> <component> +state <service> <component> <levels> + + +This tells the log daemon that service <service> exists, and that it +has a component named <component>. + +<levels> syntax: + +fatal=forward,error=forward,warning=store,info=store,event=forward,debug=off,spam=off + +ie: level=<status>[,...]* + +Where <status> is one of: +forward : store and send to log server +store : store locally only (to file) +off : do not even generate the message in the program + + + +From server to whoever is interested: + +setstate <service> <component> <levels> + +Exactly same syntax. Levels that are not mentioned are not modified. + +list services +list components <service> +list states <service> <component> + +(list commands are replied to with service/component/state commands). + + + + + + + + diff --git a/vespalog/pom.xml b/vespalog/pom.xml new file mode 100644 index 00000000000..5a39f43b1c3 --- /dev/null +++ b/vespalog/pom.xml @@ -0,0 +1,71 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>parent</artifactId> + <version>6-SNAPSHOT</version> + <relativePath>../parent/pom.xml</relativePath> + </parent> + <artifactId>vespalog</artifactId> + <version>6-SNAPSHOT</version> + <name>Vespa Log</name> + <packaging>container-plugin</packaging> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>vespajlib</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>annotations</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + <arg>-Xlint:-serial</arg> + <arg>-Werror</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/vespalog/src/.gitignore b/vespalog/src/.gitignore new file mode 100644 index 00000000000..41357d5435e --- /dev/null +++ b/vespalog/src/.gitignore @@ -0,0 +1,5 @@ +Makefile.ini +config.h +config_command.sh +vespalog.mak +project.dsw diff --git a/vespalog/src/Log.pm b/vespalog/src/Log.pm new file mode 100644 index 00000000000..06eda129238 --- /dev/null +++ b/vespalog/src/Log.pm @@ -0,0 +1,93 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package Log; + +require 5.006_001; +use strict; +use warnings; +use Sys::Hostname; + + +#initialize +my $VESPA_LOG_TARGET = $ENV{VESPA_LOG_TARGET} || "fd:2"; +my $VESPA_LOG_LEVELS = $ENV{VESPA_LOG_LEVELS} || "all -debug -spam"; + +my $SERVICE = $ENV{VESPA_SERVICE_NAME} || "-"; +my $HOST = hostname; + +my %LEVEL = ( error => 0x01, + warning => 0x02, + info => 0x04, + config => 0x08, + debug => 0x10, + spam => 0x20, + all => 0x3f); + +my %instance = (); + +my $VESPA_LOG_FILTER; +foreach (split(/\s+/, $VESPA_LOG_LEVELS)) { + /(-?)(\S+)/ || die "Log level parse error: $_"; + my ($inv, $value) = ($1, $LEVEL{$2}); + die "Unknown level: $2" unless $value; + if ($inv) { + $VESPA_LOG_FILTER &= ~$value; + } else { + $VESPA_LOG_FILTER |= $value; + } +} + +if ($VESPA_LOG_TARGET =~ /fd:(\d+)/) { + open(TARGET, ">&=$1") || die $!; +} elsif ($VESPA_LOG_TARGET =~ /file:(.+)/) { + open(TARGET, ">>$1") || die $!; +} else { + die "Illegal target $VESPA_LOG_TARGET"; +} +select(TARGET); $| = 1; + +sub new { + my ($self, $name) = @_; + my $type = ref($self) || $self; + $instance{$name} = bless { component => $name } unless $instance{$name}; + return $instance{$name}; +} + +sub open_target { + return unless $VESPA_LOG_TARGET =~ /file:(.+)/; + close(TARGET); + open(TARGET, ">>$1") || die $!; + select(TARGET); $| = 1; +} + +sub escape_message { + $_ = shift; + s/\n/\\n/g; + s/\r/\\r/g; + s/\t/\\t/g; + s/([\x80-\xFF])/sprintf("\\x%x",ord($1))/eg; + return $_; +} + +sub log { + my ($this, $level, $msg) = @_; + my $component = ref($this) ? $this->{component} : "-"; + die "Unknown logging level: '$level'" unless $LEVEL{$level}; + return unless $VESPA_LOG_FILTER & $LEVEL{$level}; + + open_target; + $msg = escape_message($msg); + + # format: time host pid service component level message + print TARGET (time()."\t$HOST\t$$\t$SERVICE\t$component\t$level\t$msg\n"); +} + +sub error { &log(shift, "error", @_); } +sub warning { &log(shift, "warning", @_); } +sub info { &log(shift, "info", @_); } +sub config { &log(shift, "config", @_); } +sub debug { &log(shift, "debug", @_); } +sub spam { &log(shift, "spam", @_); } + + +$SIG{__DIE__} = sub { Log->log("error", @_); }; +$SIG{__WARN__} = sub { Log->log("warning", @_); }; diff --git a/vespalog/src/logctl/.gitignore b/vespalog/src/logctl/.gitignore new file mode 100644 index 00000000000..6089704ed9a --- /dev/null +++ b/vespalog/src/logctl/.gitignore @@ -0,0 +1,3 @@ +.depend +Makefile +logctl diff --git a/vespalog/src/logctl/CMakeLists.txt b/vespalog/src/logctl/CMakeLists.txt new file mode 100644 index 00000000000..666e840fe60 --- /dev/null +++ b/vespalog/src/logctl/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalog_logctl_app + SOURCES + logctl.cpp + OUTPUT_NAME logctl + INSTALL bin + DEPENDS + vespalog +) diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp new file mode 100644 index 00000000000..8457fb944ce --- /dev/null +++ b/vespalog/src/logctl/logctl.cpp @@ -0,0 +1,273 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <cstdio> +#include <cstdlib> +#include <unistd.h> +#include <cstring> +#include <limits.h> + +#include <memory> +#include <vector> +#include <string> + +#include <vespa/defaults.h> +#include <vespa/log/control-file.h> +#include <vespa/log/internal.h> +#include <vespa/log/component.h> +LOG_SETUP("logctl", "$Id$"); + +using namespace ns_log; + +static void modifyLevels(const char *file, char *component, char *levels, + bool shouldCreateFile, bool shouldCreateEntry); +static void readLevels(const char *file, char *component); + + +static void +usage(const char *name) +{ + fprintf(stderr, "Usage: %s [OPTION] <service>[:component-specification]\n" + " or: %s [OPTION] <service>[:component-spec] <level-mods>\n" + "Print or modify log levels for a VESPA service.\n\n" + " -c Create the control file if it does not exist (implies -n)\n" + " -a Update all .logcontrol files in <dir>\n" + " -r Reset to default levels\n" + " -n Create the component entry if it does not exist\n" + " -f <file> Use <file> as the log control file\n" + " -d <dir> Look in <dir> for log control files\n\n" + "<level-mods> are defined as follows:\n" + " <level>=<on|off>[,<level>=<on|off>]...\n" + "<level> is one of:\n" + " all, fatal, error, warning, info, event, config, debug or spam\n\n" + "component-specification specicies which sub-components of the\n" + "service should be controlled. If it is empty, all components\n" + "are controlled:\n" + " x. : Matches only component x\n" + " x : Matches component x and all its sub-components\n\n" + "Example: %s topleveldispatch:log all=on,spam=off,debug=off : For service\n" + "topleveldispatch, set log and all sub-components of log to enable all\n" + "except spam and debug.\n\n", name, name, name); +} + +static std::vector<std::string> +findAllFiles(const char *dir) +{ + std::vector<std::string> rv; + DIR *d = opendir(dir); + if (d == NULL) { + perror(dir); + return rv; + } + LOG(spam, "scanning %s", dir); + + struct dirent *entry; + while ((entry = readdir(d)) != NULL) { + if (strcmp(entry->d_name, ".") == 0) continue; + if (strcmp(entry->d_name, "..") == 0) continue; + + const char *suffix = ".logcontrol"; + + LOG(spam, "check %s", entry->d_name); + + if (strlen(entry->d_name) > strlen(suffix)) { + char *cmp = entry->d_name + strlen(entry->d_name) - strlen(suffix); + if (strcmp(suffix, cmp) == 0) { + std::string fn = dir; + fn.append("/"); + fn.append(entry->d_name); + + struct stat sb; + if (stat(fn.c_str(), &sb) == 0) { + if (S_ISREG(sb.st_mode)) { + *cmp = 0; + rv.push_back(entry->d_name); + } + } else { + perror(fn.c_str()); + } + } + } + } + closedir(d); + return rv; +} + + +int +main(int argc, char **argv) +{ + vespa::Defaults::bootstrap(argv[0]); + + const char *dir = getenv("VESPA_LOG_CONTROL_DIR"); + const char *file = getenv("VESPA_LOG_CONTROL_FILE"); + const char *root = getenv("ROOT"); + if (!root) { + root = vespa::Defaults::vespaHome(); + } + bool shouldCreateFile = false; + bool shouldCreateEntry = false; + bool doAllFiles = false; + bool doOnlyFile = false; + bool doResetLevels = false; + + while (1) { + int c = getopt(argc, argv, "acnrf:d:h"); + if (c == -1) + break; + switch (c) { + case 'a': + doAllFiles = true; + break; + case 'r': + doResetLevels = true; + case 'c': + shouldCreateFile = true; + // fallthrough + case 'n': + shouldCreateEntry = true; + break; + case 'f': + file = strdup(optarg); + doOnlyFile = true; + break; + case 'd': + dir = strdup(optarg); + break; + case 'h': + usage(argv[0]); + exit(EXIT_SUCCESS); + } + } + + + char buf[PATH_MAX]; + if (!dir && !file) { + snprintf(buf, sizeof buf, "%s/var/db/vespa/logcontrol", root); + dir = buf; + } + + typedef std::vector<std::string> strlist_t; + + strlist_t services; + + char nullComponent[] = "default"; + char *component = nullComponent; + + if (doAllFiles) { + services = findAllFiles(dir); + if (doOnlyFile) { + fprintf(stderr, "-f and -a options cannot be used at the same time\n"); + exit(EXIT_FAILURE); + } + // No log control files exist + if (services.empty()) { + exit(EXIT_SUCCESS); + } + } else { + if (optind >= argc) { + usage(argv[0]); + fprintf(stderr, "ERROR: Missing service argument!\n"); + exit(EXIT_FAILURE); + } + char *service = strdup(argv[optind]); + ++optind; + + char *delim = strchr(service, ':'); + if (delim) { + *delim = 0; + services.push_back(service); + *delim = '.'; + component = delim; + } else { + services.push_back(service); + } + } + + char defLevels[] = "all=on,debug=off,spam=off"; + char *levels = NULL; + + if (doResetLevels) { + levels = defLevels; + } else { + if (argc > optind) { + levels = strdup(argv[optind]); + ++optind; + } + } + + if (argc > optind) { + usage(argv[0]); + fprintf(stderr, "ERROR: Too many arguments!\n\n"); + exit(EXIT_FAILURE); + } + + bool hadFailure = false; + bool hadSuccess = false; + + for (strlist_t::iterator it = services.begin(); + it != services.end(); + ++it) + { + const char *service = (*it).c_str(); + + char serviceFile[PATH_MAX]; + if (! doOnlyFile) { + snprintf(serviceFile, sizeof serviceFile, "%s/%s.logcontrol", dir, service); + file = serviceFile; + } + // fprintf(stderr, "Log control file %s:\n", file); + + try { + if (levels) { + modifyLevels(file, component, levels, shouldCreateFile, shouldCreateEntry); + } else { + readLevels(file, component); + } + hadSuccess = true; + } catch (InvalidLogException& x) { + fprintf(stderr, "Failed: %s\n", x.what()); + hadFailure = true; + } + } + if (hadFailure) exit(EXIT_FAILURE); + if (! hadSuccess) { + fprintf(stderr, "no logcontrol files updates\n"); + exit(EXIT_FAILURE); + } + exit(EXIT_SUCCESS); +} + +static void +modifyLevels(const char *file, char *componentPattern, char *levels, + bool shouldCreateFile, bool shouldCreateEntry) +{ + ControlFile cf(file, shouldCreateFile + ? ControlFile::CREATE : ControlFile::READWRITE); + Component *c; + if (shouldCreateEntry) { + cf.ensureComponent(componentPattern); + } + ComponentIterator iter(cf.getComponentIterator()); + while ((c = iter.next()) != NULL) { + std::unique_ptr<Component> component(c); + if (component->matches(componentPattern)) { + component->modifyLevels(levels); + } + } + cf.flush(); +} + +static void +readLevels(const char *file, char *componentPattern) +{ + ControlFile cf(file, ControlFile::READONLY); + Component *c; + ComponentIterator iter(cf.getComponentIterator()); + while ((c = iter.next()) != NULL) { + std::unique_ptr<Component> component(c); + if (c->matches(componentPattern)) { + c->display(); + } + } +} + diff --git a/vespalog/src/logfmt/.gitignore b/vespalog/src/logfmt/.gitignore new file mode 100644 index 00000000000..568d69eee3e --- /dev/null +++ b/vespalog/src/logfmt/.gitignore @@ -0,0 +1 @@ +logfilter diff --git a/vespalog/src/logfmt/logfilter.c b/vespalog/src/logfmt/logfilter.c new file mode 100644 index 00000000000..709ce2c016e --- /dev/null +++ b/vespalog/src/logfmt/logfilter.c @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdio.h> +#include <stdlib.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> +#include <signal.h> + +int main(int argc, char *argv[]) +{ + int lfpipe[2]; + pid_t lfmtpid; + pid_t progpid; + int wstat; + + if (argc < 3) { + fprintf(stderr, "Usage: %s logfmt prog [...]\n", argv[0]); + exit(1); + } + + pipe(lfpipe); + lfmtpid = fork(); + if (lfmtpid == 0) { + close(lfpipe[1]); + if (lfpipe[0] != 0) { + close(0); + dup(lfpipe[0]); + close(lfpipe[0]); + } + execlp(argv[1], argv[1], "-", (const char *)NULL); + perror("exec logfmt failed"); + exit(1); + } + if (lfmtpid < 0) { + perror("fork failed"); + exit(1); + } + close(lfpipe[0]); + + progpid = fork(); + if (progpid == 0) { + char buf[20]; + int i=2; + + sprintf(buf, "fd:%d", lfpipe[1]); + setenv("VESPA_LOG_TARGET", buf, 1); + for (; i<argc; i++) { + argv[i-2] = argv[i]; + } + argv[i-2] = NULL; + execvp(argv[0], argv); + perror("exec program failed"); + exit(1); + } + if (progpid < 0) { + perror("fork failed"); + exit(1); + } + close(lfpipe[1]); + + if (waitpid(lfmtpid, &wstat, 0) != lfmtpid) { + perror("bad waitpid for logfmt"); + } + if (waitpid(progpid, &wstat, 0) != progpid) { + perror("bad waitpid for program"); + } + + if (WIFEXITED(wstat)) { + return WEXITSTATUS(wstat); + } + if (WIFSIGNALED(wstat)) { + int sig = WTERMSIG(wstat); + signal(sig, SIG_DFL); + kill(getpid(), sig); + } + /* should not get here */ + exit(1); +} diff --git a/vespalog/src/logfmt/logfmt.1 b/vespalog/src/logfmt/logfmt.1 new file mode 100644 index 00000000000..a16b1ea2eb3 --- /dev/null +++ b/vespalog/src/logfmt/logfmt.1 @@ -0,0 +1,218 @@ +.\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $ +.\" +.Dd October 29, 2004 +.Dt LOGFMT \&1 "Vespa documentation" +.Os "Yahoo! Vespa" "2.0" +.Os +.Sh NAME +.Nm logfmt +.Nd select and format messages from Vespa log files +.Sh SYNOPSIS +.Nm +.Op Fl l Ar levellist +.Op Fl s Ar fieldlist +.Op Fl p Ar pid +.Op Fl S Ar service +.Op Fl H Ar host +.Op Fl c Ar regex +.Op Fl m Ar regex +.Op Fl t +.Op Fl f +.Op Fl L +.Op Fl N +.Op Fl ts +.Op Ar +.Sh DESCRIPTION +The +.Nm +utility reads Vespa log files, select messages and writes a formatted +version of selected messages to the standard output. +.Pp +The options are as follows: +.Bl -tag -width ".It Fl l Ar levellist" +.It Fl l Ar levellist +Select which log levels to select. +The default is to select "fatal", "error", "warning" and "info" +levels, and suppress "config", "event", "debug" and "spam" levels; +but when using this option, only the named levels will be selected. +The +.Ar levellist +is a comma separated list of level names. +The name +.Em all +may be used to add all known levels. +Prepending a minus sign will deselect the level named. +Starting the list with a plus sign will add and remove levels +from the current (or default) list of levels instead +of replacing it. +.It Fl s Ar fieldlist +Select which fields of log messages to show. +The order of the actual output fields is fixed. +When using this option, only the named fields will be shown. The +fieldlist is a comma separated list of field names. The name +.Em all +may be used to add all possible fields. +Prepending a minus sign will turn off display of the named field. +Starting the list with a plus sign will add and remove fields +from the current (or default) list of fields instead +of replacing it. +.Pp +The fields which may be named are: +.Bl -tag -width component +.It time +Print the time in seconds since the epoch. +Ignored if +.Em fmttime +is shown. +.It fmttime +Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format. +Note that the time is printed in the local timezone; to get GMT +output use +.Nm "\*[q]env TZ=GMT logfmt\*[q]" +as your command. +.It msecs +Add milliseconds after the seconds in +.Em time +and +.Em fmttime +output. Ignored if +.Em usecs +is in effect. +.It usecs +Add microseconds after the seconds in +.Em time +and +.Em fmttime +output. +.It host +Print the hostname field. +.It level +Print the level field (uppercased). +.It pid +Print the pid field. +.It service +Print the service field. +.It component +Print the component field. +.It message +Print the message text field. +You probably always want to add this. +.El +.Pp +Using this option several times works as if the given +.Ar fieldlist +arguments had been concatenated into one comma-separated list. +The default fields to show are as if +.Bk +.Op Fl s Ar fmttime,msecs,level,service,component,message +.Ek +had been given. +.It Fl p Ar pid +Select only messages where the pid field matches the +.Ar pid +string exactly. +.It Fl S Ar service +Select only messages where the service field matches the +.Ar service +string exactly. +.It Fl H Ar host +Select only messages where the hostname field matches the +.Ar host +string exactly. +.It Fl c Ar regex +Select only messages where the component field matches the +.Ar regex +given, using +.Xr perlre +regular expression matching. +.It Fl m Ar regex +Select only messages where the message text field matches the +.Ar regex +given, using +.Xr perlre +regular expression matching. +.It Fl f +Invoke tail -F to follow input file +.It Fl L +.It Fl -livestream +Only useful for multi-machine setups: +Invoke vespa-replicate-log-stream to follow live stream from log server, +instead of reading any input file. Also implies +.Bk +.Fl "s +host" +.Ek +as default. +.It Fl N +Dequote quoted newlines in the message text field to an actual newline plus tab. +.It Fl t +Format the component field (if shown) as a fixed-with string, +truncating if necessary. +.It Fl ts +Format the service field (if shown) as a fixed-with string, +truncating if necessary. +.El +.Sh EXAMPLES +The command: +.Pp +.Bd -literal -offset indent +logfmt -l event -s service,message,fmttime,message +.Ed +.Pp +will display only messages with log level "event", +printing a human-readable time (without any fractional seconds), +the service generating the event and the event message, like this: +.Bd -literal -offset indent +[2004-12-07 18:43:01] config-sentinel starting/1 name="logd" +[2004-12-07 18:43:01] logd started/1 name="logdemon" +[2004-12-07 18:45:51] rtc starting/1 name="rtc.index0" +[2004-12-07 18:45:51] rtc.index0 started/1 name="flexindexer.index" +[2004-12-07 18:45:51] rtc.index0 stopping/1 name="flexindexer.index" why="done" +[2004-12-07 18:45:53] rtc stopped/1 name="rtc.index0" pid=50600 exitcode=0 +[2004-12-07 18:46:13] logd stopping/1 name="logdemon" why="done ok." +[2004-12-07 18:46:13] config-sentinel stopped/1 name="logd" pid=49633 exitcode=0 +.Ed +.Pp +Note that the second "message" item in the fieldlist is redundant, +and that order of printed field is fixed no matter what the fieldlist +order is. +.Pp +The command: +.Pp +.Bd -literal -offset indent +logfmt -l all-info,-debug -s level \e + -s time,usecs,component,message -t -l -event +.Ed +.Pp +will display messages with log levels that are +.Em not +any of +.Em info, debug, +or +.Em event, +printing the time in seconds and microseconds, the log level, the +component name, and the message text, possibly somewhat like this: +.Bd -literal -offset indent +1102441382.530423 CONFIG : nc Config handle: 'pandora.0-rtx' +1102441551.471568 CONFIG : flexindexer.doc Adding document type typetest-0 +1102441573.148211 WARNING : logdemon stopping on signal 15 +1102441887.158000 WARNING : com.yahoo.fs4.m read exception +1102441935.569567 WARNING : rtc Dispatch inherited job failed for dir dispatch0 +1102442115.746001 WARNING : fdispatch Search node 172.24.94.75:10124 down +1102442474.205920 WARNING : rtx RTC (tcp/172.24.94.75:10161) : DOWN +1102442474.515877 WARNING : fdispatch Search node localhost:10128 down +1102442983.075669 ERROR : flexindexer.std Unable to find cluster map defaultcluster +.Ed +.Sh FILES +If no file argument is given, +.Nm +will read the last vespa log file $VESPA_HOME/logs/vespa/vespa.log (this also works with the +.Fl f +option). +Otherwise, reads only the files given as arguments. +To read standard input, supply a single dash '-' as a file argument. +.Sh SEE ALSO +Documentation in the "log" module for input file format. +.Sh HISTORY +Developed as part of Vespa 1.1. The default output format +reflects the old "fastlib" log formatting, with minor differences +and is intended to be human-readable, not parsed. diff --git a/vespalog/src/logfmt/logfmt.pl b/vespalog/src/logfmt/logfmt.pl new file mode 100644 index 00000000000..9152d1fcfec --- /dev/null +++ b/vespalog/src/logfmt/logfmt.pl @@ -0,0 +1,327 @@ +#!/usr/local/bin/perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# BEGIN perl environment bootstrap section +# Do not edit between here and END as this section should stay identical in all scripts + +use File::Basename; +use File::Path; + +sub findpath { + my $myfullname = ${0}; + my($myname, $mypath) = fileparse($myfullname); + + return $mypath if ( $mypath && -d $mypath ); + $mypath=`pwd`; + + my $pwdfullname = $mypath . "/" . $myname; + return $mypath if ( -f $pwdfullname ); + return 0; +} + +# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise +sub is_vespa_home { + my($VESPA_HOME) = shift; + my $COMMON_ENV="libexec/vespa/common-env.sh"; + if ( $VESPA_HOME && -d $VESPA_HOME ) { + my $common_env = $VESPA_HOME . "/" . $COMMON_ENV; + return $VESPA_HOME if -f $common_env; + } + return 0; +} + +# Returns the home of Vespa, or dies if it cannot +sub findhome { + # Try the VESPA_HOME env variable + return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'}); + if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly + die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n"; + } + + # Try the ROOT env variable + $ROOT = $ENV{'ROOT'}; + return $ROOT if is_vespa_home($ROOT); + + # Try the script location or current dir + my $mypath = findpath(); + if ($mypath) { + while ( $mypath =~ s|/[^/]*$|| ) { + return $mypath if is_vespa_home($mypath); + } + } + die "FATAL: Missing VESPA_HOME environment variable\n"; +} + +BEGIN { + my $tmp = findhome(); + if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; } + $ENV{'VESPA_HOME'} = $tmp; +} +my $VESPA_HOME = $ENV{'VESPA_HOME'}; + +# END perl environment bootstrap section + +use 5.006_001; +use strict; +use warnings; + +use Getopt::Long qw(:config no_ignore_case); + +my %showflags = ( + time => 1, + fmttime => 1, + msecs => 1, + usecs => 0, + host => 0, + level => 1, + pid => 0, + service => 1, + component => 1, + message => 1 +); + +my %levelflags = ( + fatal => 1, + error => 1, + warning => 1, + info => 1, + config => 0, + event => 0, + debug => 0, + spam => 0 +); + +# Do not buffer the output +$| = 1; + +my $compore; +my $msgtxre; +my $onlypid; +my $onlysvc; +my $onlyhst; + +my $shortsvc; +my $shortcmp; + +my @optlevels; +my @optshow; +my $optlevels; +my $optfollow; +my $optnldequote; +my $optreplication; +my $opthelp = ''; + +my $bad = 0; + +GetOptions ('level|l=s' => \@optlevels, + 'service|S=s' => \$onlysvc, + 'show|s=s' => \@optshow, + 'pid|p=s' => \$onlypid, + 'component|c=s' => \$compore, + 'message|m=s' => \$msgtxre, + 'help|h' => \$opthelp, + 'follow|f' => \$optfollow, + 'livestream|L' => \$optreplication, + 'nldequote|N' => \$optnldequote, + 'host|H=s' => \$onlyhst, + 'truncateservice|ts' => \$shortsvc, + 'truncatecomponent|tc|t' => \$shortcmp, +) or $bad=1; + +if ( $optreplication ) { + open(STDIN, "vespa-replicate-log-stream |"); + $showflags{'host'} = 1; +} elsif ( @ARGV == 0 and ! -p STDIN) { + push(@ARGV, "$VESPA_HOME/logs/vespa/vespa.log"); +} + +if ( $optfollow ) { + my $filearg = ""; + if ( @ARGV > 1 ) { + print STDERR "ERROR: Cannot follow more than one file\n\n"; + $bad=1; + } else { + $filearg = shift @ARGV if (@ARGV > 0); + open(STDIN, "tail -F $filearg |") + or die "cannot open 'tail -F $filearg' as input pipe\n"; + } +} + +if ( $opthelp || $bad ) { + print STDERR "Usage: $0 [options] [inputfile ...]\n", + "Options:\n", + " -l LEVELLIST\t--level=LEVELLIST\tselect levels to include\n", + " -s FIELDLIST\t--show=FIELDLIST\tselect fields to print\n", + " -p PID\t--pid=PID\t\tselect messages from given PID\n", + " -S SERVICE\t--service=SERVICE\tselect messages from given SERVICE\n", + " -H HOST\t--host=HOST\t\tselect messages from given HOST\n", + " -c REGEX\t--component=REGEX\tselect components matching REGEX\n", + " -m REGEX\t--message=REGEX\t\tselect message text matching REGEX\n", + " -f\t\t--follow\t\tinvoke tail -F to follow input file\n", + " -L\t\t--livestream\t\tfollow log stream from logserver\n", + " -N\t\t--nldequote\t\tdequote newlines in message text field\n", + " -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n", + " --ts\t\t--truncateservice\tchop service to 9 chars\n", + "\n", + "FIELDLIST is comma separated, available fields:\n", + "\t time fmttime msecs usecs host level pid service component message\n", + "Available levels for LEVELLIST:\n", + "\t fatal error warning info event debug spam\n", + "for both lists, use 'all' for all possible values, and -xxx to disable xxx.\n"; + exit $bad; +} + + +$optlevels = join(",", @optlevels ); +if ( $optlevels ) { + my $k; + unless ( $optlevels =~ s/^\+// or $optlevels =~ m/^-/ ) { + $levelflags{$_} = 0 foreach ( keys %levelflags ); + } + my @l = split(/,|(?=-)/, $optlevels); + my $l; + foreach $l ( @l ) { + my $v = 1; + my $minus = ""; + if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } + if ( $l eq "all" ) { + foreach $k ( keys %levelflags ) { + $levelflags{$k} = $v; + } + } elsif ( defined $levelflags{$l} ) { + $levelflags{$l} = $v; + } else { + print STDERR "bad level option '$minus$l'\n"; + exit 1; + } + } +# print STDERR "select level $_ => $levelflags{$_}\n" +# foreach ( keys %levelflags ); +} + +my $optshow; +$optshow = join(",", @optshow ); +if ( $optshow ) { + my $k; + unless ( $optshow =~ s/^\+// or $optshow =~ m/^-/ ) { + $showflags{$_} = 0 foreach ( keys %showflags ); + } + my @l = split(/,|(?=-)/, $optshow); + my $l; + foreach $l ( @l ) { + my $v = 1; + my $minus = ""; + if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } + if ( $l eq "all" ) { + foreach $k ( keys %showflags ) { + $showflags{$k} = $v; + } + } elsif ( defined $showflags{$l} ) { + $showflags{$l} = $v; + } else { + print STDERR "bad show option '$minus$l'\n"; + exit 1; + } + } +# print STDERR "show field $_ => $showflags{$_}\n" +# foreach ( keys %showflags ) ; +} + +while (<>) { + chomp; + if ( /^ + (\d+)\.?(\d*) # seconds, optional fractional seconds + \t + ([^\t]*) # host + \t + (\d+\/?\d*|\-\/\d+) # pid, optional tid + \t + ([^\t]*) # servicename + \t + ([^\t]*) # componentname + \t + (\w+) # level + \t + (.*) # message text + $/x ) + { + my $secs = $1; + my $usec = $2 . "000000"; # make sure we have atleast 6 digits + my $host = $3; + my $pidn = $4; + my $svcn = $5; + my $comp = $6; + my $levl = $7; + my $msgt = $8; + + if ( ! defined $levelflags{$levl} ) { + print STDERR "Warning: unknown level '$levl' in input\n"; + $levelflags{$levl} = 1; + } + next unless ( $levelflags{$levl} ); + + if ($compore && $comp !~ m/$compore/o) { next; } + if ($msgtxre && $msgt !~ m/$msgtxre/o) { next; } + if ($onlypid && $pidn ne $onlypid) { next; } + if ($onlysvc && $svcn ne $onlysvc) { next; } + if ($onlyhst && $host ne $onlyhst) { next; } + + $levl = "\U$levl"; + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday); + ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($secs); + my $datestr = sprintf("%04d-%02d-%02d", + 1900+$year, 1+$mon, $mday); + my $timestr = sprintf("%02d:%02d:%02d", + $hour, $min, $sec); + + if ( $showflags{"time"} || $showflags{"fmttime"} ) { + if ($showflags{"fmttime"} ) { + print "[$datestr $timestr"; + if ( $showflags{"usecs"} ) { + printf ".%.6s", $usec; + } elsif ( $showflags{"msecs"} ) { + printf ".%.3s", $usec; + } + print "] "; + } else { + printf "%s.%.6s ", $secs, $usec; + } + } + if ( $showflags{"host"} ) { + printf "%-8s ", $host; + } + if ( $showflags{"level"} ) { + printf "%-7s : ", $levl; + } + if ( $showflags{"pid"} ) { + printf "%5s ", $pidn; + } + if ( $showflags{"service"} ) { + if ( $shortsvc ) { + printf "%-9.9s ", $svcn; + } else { + printf "%-16s ", $svcn; + } + } + if ( $showflags{"component"} ) { + if ( $shortcmp ) { + printf "%-15.15s ", $comp; + } else { + printf "%s\t", $comp; + } + } + if ( $showflags{"message"} ) { + if ( $optnldequote ) { + my $did_dequote_1 = ( $msgt =~ s/\\n\\t/\n\t/g ); + my $did_dequote_2 = ( $msgt =~ s/\\n/\n\t/g ); + $msgt = "\n\t${msgt}" if ( $did_dequote_1 || $did_dequote_2 ); + $msgt =~ s/\\t/\t/; + } + print $msgt; + } + print "\n"; + } else { + print STDERR "bad log line: '$_'\n"; + } +} diff --git a/vespalog/src/logger/.gitignore b/vespalog/src/logger/.gitignore new file mode 100644 index 00000000000..7cd5c615ee3 --- /dev/null +++ b/vespalog/src/logger/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +vespa-runserver +vlogconv +vlogreplay diff --git a/vespalog/src/logger/CMakeLists.txt b/vespalog/src/logger/CMakeLists.txt new file mode 100644 index 00000000000..b2327a1327a --- /dev/null +++ b/vespalog/src/logger/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalog_vlogconv_app + SOURCES + logger.cpp + llreader.cpp + OUTPUT_NAME vlogconv + INSTALL bin + DEPENDS + vespalog +) +vespa_add_executable(vespalog_vlogreplay_app + SOURCES + logreplay.c + OUTPUT_NAME vlogreplay + INSTALL bin + DEPENDS +) +vespa_add_executable(vespalog_vespa-runserver_app + SOURCES + runserver.cpp + llreader.cpp + OUTPUT_NAME vespa-runserver + INSTALL bin + DEPENDS + vespalog +) diff --git a/vespalog/src/logger/llreader.cpp b/vespalog/src/logger/llreader.cpp new file mode 100644 index 00000000000..82fa48777b0 --- /dev/null +++ b/vespalog/src/logger/llreader.cpp @@ -0,0 +1,113 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <sys/time.h> + +#include "llreader.h" + +namespace ns_log { + + +InputBuf::InputBuf(int fd) + : _inputfd(fd), + _size(1000), + _buf((char *)malloc(_size)), + _bp(_buf), + _left(_size) +{ } + + +InputBuf::~InputBuf() +{ + free(_buf); +} + + +bool +InputBuf::hasInput() +{ + char *p = _buf; + while (p < _bp) { + if (*p == '\n') return true; + p++; + } + return false; + +} + +void +InputBuf::doInput(LLParser& via) +{ + char *p = _buf; + while (p < _bp) { + if (*p == '\n') { + *p = '\0'; + via.doInput(_buf); + ++p; + int len = p - _buf; + int movelen = _bp - p; + memmove(_buf, p, movelen); + _bp -= len; + _left += len; + p = _buf; + continue; + } + p++; + } +} + +void +InputBuf::extend() +{ + _size *= 2; + int pos = _bp - _buf; + char *nbuf = (char *)realloc(_buf, _size); + if (nbuf == NULL) { + free(_buf); + throw MsgException("realloc failed"); + } + _buf = nbuf; + _bp = _buf + pos; + _left = _size - pos; +} + +bool +InputBuf::blockRead() +{ + if (_left < 80) { + extend(); + } + ssize_t len = ::read(_inputfd, _bp, _left); + if (len > 0) { + _bp += len; + _left -= len; + // printf("read %d bytes: '%.*s'\n", len, len, _buf); + } else if (len < 0) { + throw MsgException("error reading"); + } else { + // EOF on input + return false; + } + return true; +} + +void +InputBuf::doAllInput(LLParser &outputvia) +{ + while (blockRead()) { + while (hasInput()) { + doInput(outputvia); + } + } + if (_bp != _buf) { + if (_left < 1) extend(); + *_bp++ = '\n'; + doInput(outputvia); + } +} + +} // namespace diff --git a/vespalog/src/logger/llreader.h b/vespalog/src/logger/llreader.h new file mode 100644 index 00000000000..7519f99959f --- /dev/null +++ b/vespalog/src/logger/llreader.h @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdexcept> + +#include <vespa/log/log.h> +#include <vespa/log/log-target.h> +#include <vespa/log/llparser.h> + +namespace ns_log { + +class MsgException : public std::exception { +private: + const char *_string; + MsgException& operator = (const MsgException &); +public: + MsgException(const MsgException &x) : std::exception(), _string(x._string) {} + MsgException(const char *s) : _string(s) {} + ~MsgException() throw() {} // nothing to do + const char *what() const throw() { return _string; } +}; + + +class InputBuf +{ +private: + int _inputfd; + int _size; + char *_buf; + char *_bp; + int _left; + void extend(); + InputBuf(const InputBuf& other); + InputBuf& operator= (const InputBuf& other); +public: + InputBuf(int fd); + ~InputBuf(); + bool blockRead(); + bool hasInput(); + void doInput(LLParser &via); + void doAllInput(LLParser &via); +}; + +} // namespace diff --git a/vespalog/src/logger/logger.cpp b/vespalog/src/logger/logger.cpp new file mode 100644 index 00000000000..fcd1ef9d57d --- /dev/null +++ b/vespalog/src/logger/logger.cpp @@ -0,0 +1,56 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> + +#include <vespa/log/llparser.h> +#include "llreader.h" + +using namespace ns_log; + +int main(int argc, char *argv[]) +{ + InputBuf input(0); + LLParser llparser; + + int ch; + while ((ch = getopt(argc, argv, "s:c:l:p:h")) != -1) { + switch (ch) { + case 's': + llparser.setService(optarg); + break; + case 'c': + llparser.setComponent(optarg); + break; + case 'l': { + Logger::LogLevel level = Logger::parseLevel(optarg); + if (level == Logger::NUM_LOGLEVELS) { + fprintf(stderr, "Unknown loglevel %s - using info\n", + optarg); + level = Logger::info; + } + llparser.setDefaultLevel(level); + break; + } + case 'p': + llparser.setPid(atoi(optarg)); + break; + default: + fprintf(stderr, "Usage: foo | %s [-s service] [-c component]" + "[-l level] [-p pid]\n", argv[0]); + exit(ch != 'h'); + } + } + + try { + input.doAllInput(llparser); + } catch (MsgException& ex) { + fprintf(stderr, "error: %s\n", ex.what()); + return 1; + } + return 0; +} diff --git a/vespalog/src/logger/logreplay.c b/vespalog/src/logger/logreplay.c new file mode 100644 index 00000000000..cc3ee798aa4 --- /dev/null +++ b/vespalog/src/logger/logreplay.c @@ -0,0 +1,58 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <sys/time.h> +#include <stdio.h> +#include <math.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +int +main(int argc, char **argv) +{ + char line[10000]; + + double delta = 0; + + if (argc != 1) { + fprintf(stderr, "Usage: %s < <vespa.log>\n" + "Replays a vespa log file with the same timing delta between each log message.\n" + "Reprints the log messages without timestamps.\n\n", + argv[0]); + exit(EXIT_FAILURE); + } + + while (fgets(line, sizeof line, stdin)) { + struct timeval tv; + char *s; + double log_time; + double now; + double delay; + + gettimeofday(&tv, NULL); + now = tv.tv_sec + 1e-6 * tv.tv_usec; + + log_time = strtod(line, NULL); + if (!delta) { + delta = now - log_time; + } + + delay = log_time + delta - now; + if (delay > 0) { + tv.tv_sec = floor(delay); + tv.tv_usec = 1e6 * (delay - floor(delay)); + /* fprintf(stderr, "Sleeping for %.3fs\n", delay); */ + select(0, NULL, NULL, NULL, &tv); + } + + s = strchr(line, '\t'); + if (s) { + s = s + 1; + } else { + s = line; + } + fwrite(s, strlen(s), 1, stdout); + fflush(stdout); + } + exit(EXIT_SUCCESS); +} diff --git a/vespalog/src/logger/runserver.cpp b/vespalog/src/logger/runserver.cpp new file mode 100644 index 00000000000..fde952720c7 --- /dev/null +++ b/vespalog/src/logger/runserver.cpp @@ -0,0 +1,470 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <time.h> +#include <signal.h> + +#include <sys/select.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <sys/file.h> +#include <fcntl.h> + +#include <vespa/defaults.h> +#include <vespa/log/llparser.h> +#include "llreader.h" +#include <vespa/log/log.h> + +LOG_SETUP("runserver"); + +// XXX should close all file descriptors, +// XXX but that makes logging go haywire + +static volatile sig_atomic_t gotstopsig = 0; +static volatile sig_atomic_t lastsig = 0; +static volatile sig_atomic_t unhandledsig = 0; + +extern "C" { +void termsig(int sig) { + lastsig = sig; + gotstopsig = 1; + unhandledsig = 1; +} +} + +class PidFile +{ +private: + char *_pidfile; + int _fd; + PidFile(const PidFile&); + PidFile& operator= (const PidFile&); +public: + PidFile(const char *pidfile) : _pidfile(strdup(pidfile)), _fd(-1) {} + ~PidFile() { free(_pidfile); if (_fd >= 0) close(_fd); } + int readPid(); + void writePid(); + bool writeOpen(); + bool isRunning(); + bool isMine(); + void cleanUp(); +}; + +void +PidFile::cleanUp() +{ + if (isMine() || !isRunning()) remove(_pidfile); + if (_fd >= 0) close(_fd); + _fd = -1; +} + +bool +PidFile::writeOpen() +{ + if (_fd >= 0) close(_fd); + int flags = O_CREAT | O_WRONLY | O_NONBLOCK; + _fd = open(_pidfile, flags, 0644); + if (_fd < 0) { + fprintf(stderr, "could not create pidfile %s: %s\n", _pidfile, + strerror(errno)); + return false; + } + // XXX should we use locking or not? + if (flock(_fd, LOCK_EX | LOCK_NB) != 0) { + fprintf(stderr, "could not lock pidfile %s: %s\n", _pidfile, + strerror(errno)); + close(_fd); + _fd = -1; + return false; + } + fcntl(_fd, F_SETFD, FD_CLOEXEC); + return true; +} + +void +PidFile::writePid() +{ + if (_fd < 0) abort(); + ftruncate(_fd, (off_t)0); + char buf[100]; + sprintf(buf, "%d\n", getpid()); + int l = strlen(buf); + ssize_t didw = write(_fd, buf, l); + if (didw != l) { + fprintf(stderr, "could not write pid to %s: %s\n", + _pidfile, strerror(errno)); + exit(1); + } + LOG(debug, "wrote '%s' to %s (fd %d)", buf, _pidfile, _fd); +} + +int +PidFile::readPid() +{ + FILE *pf = fopen(_pidfile, "r"); + if (pf == NULL) return 0; + char buf[100]; + strcpy(buf, "0"); + fgets(buf, 100, pf); + fclose(pf); + return atoi(buf); +} + +bool +PidFile::isRunning() +{ + int pid = readPid(); + if (pid < 1) return false; + return (kill(pid, 0) == 0 || errno == EPERM); +} + +bool +PidFile::isMine() +{ + int pid = readPid(); + return (pid == getpid()); +} + +using namespace ns_log; + +int loop(const char *svc, char * const * run) +{ + int pstdout[2]; + int pstderr[2]; + + if (pipe(pstdout) < 0 || pipe(pstderr) < 0) { + LOG(error, "pipe: %s", strerror(errno)); + exit(1); + } + LOG(debug, "stdout pipe %d <- %d; stderr pipe %d <- %d", + pstdout[0], pstdout[1], + pstderr[0], pstderr[1]); + + int high = 1 + pstdout[0] + pstderr[0]; + + pid_t child = fork(); + + if (child == 0) { + // I am the child process + dup2(pstdout[1], 1); + dup2(pstderr[1], 2); + close(pstdout[0]); + close(pstderr[0]); + close(pstdout[1]); + close(pstderr[1]); + execvp(run[0], run); + LOG(error, "exec %s: %s", run[0], strerror(errno)); + exit(1); + } + if (child < 0) { + LOG(error, "fork(): %s", strerror(errno)); + exit(1); + } + // I am the parent process + + LOG(debug, "started %s (pid %d)", run[0], (int)child); + std::string torun = run[0]; + for (char * const *arg = (run + 1); *arg != NULL; ++arg) { + torun += " "; + torun += *arg; + } + + { + torun += " (pid "; + char buf[20]; + sprintf(buf, "%d", (int)child); + torun += buf; + torun += ")"; + } + EV_STARTING(torun.c_str()); + + close(pstdout[1]); + close(pstderr[1]); + + LLParser outvia; + LLParser errvia; + outvia.setDefaultLevel(Logger::info); + errvia.setDefaultLevel(Logger::warning); + + outvia.setService(svc); + errvia.setService(svc); + outvia.setComponent("stdout"); + errvia.setComponent("stderr"); + outvia.setPid(child); + errvia.setPid(child); + + InputBuf outReader(pstdout[0]); + InputBuf errReader(pstderr[0]); + + bool outeof = false; + bool erreof = false; + + int wstat = 0; + + while (child || !outeof || !erreof) { + struct timeval timeout; + + timeout.tv_sec = 0; + timeout.tv_usec = 100000; // == 100 ms == 1/10 s + + fd_set pipes; + + FD_ZERO(&pipes); + if (!outeof) FD_SET(pstdout[0], &pipes); + if (!erreof) FD_SET(pstderr[0], &pipes); + + int n = select(high, &pipes, NULL, NULL, &timeout); + if (n > 0) { + if (FD_ISSET(pstdout[0], &pipes)) { + LOG(debug, "out reader has input"); + if (outReader.blockRead()) { + while (outReader.hasInput()) { + LOG(debug, "process out reader input"); + outReader.doInput(outvia); + } + } else { + LOG(debug, "eof on stdout"); + outeof = true; // EOF on stdout + close(pstdout[0]); + } + } + if (FD_ISSET(pstderr[0], &pipes)) { + LOG(debug, "err reader has input"); + if (errReader.blockRead()) { + while (errReader.hasInput()) { + LOG(debug, "process err reader input"); + errReader.doInput(errvia); + } + } else { + LOG(debug, "eof on stderr"); + erreof = true; // EOF on stderr + close(pstderr[0]); + } + } + } + + if (child != 0) { + int cpid = waitpid(child, &wstat, WNOHANG); + if (cpid == child) { + if (WIFSTOPPED(wstat)) { + LOG(info, "child %d stopped, waiting for it to continue", + cpid); + } else if (WIFEXITED(wstat)) { + // child terminated + LOG(debug, "child %d exit status: %d", cpid, + (int)WEXITSTATUS(wstat)); + EV_STOPPED(torun.c_str(), (int)child, (int)WEXITSTATUS(wstat)); + child = 0; + } else if (WIFSIGNALED(wstat)) { + if (WTERMSIG(wstat) != lastsig) { + LOG(warning, "child died from signal: %d", WTERMSIG(wstat)); + if (WCOREDUMP(wstat)) { + LOG(info, "child %d dumped core", cpid); + } + } + child = 0; + } else { + LOG(error, "unexpected status %d from waidpit", wstat); + abort(); + } + } else if (cpid < 0) { + LOG(error, "waitpid: %s", strerror(errno)); + abort(); + } else if (cpid != 0) { + LOG(warning, "unexpected status %d for pid %d", + wstat, cpid); + abort(); + } + } + if (unhandledsig && child != 0) { + LOG(debug, "got signal %d, sending to pid %d", + (int)lastsig, (int)child); + char why[256]; + sprintf(why, "got signal %d", (int)lastsig); + EV_STOPPING(torun.c_str(), why); + kill(child, lastsig); + unhandledsig = 0; + } + } + if (WIFSIGNALED(wstat)) + return WTERMSIG(wstat); + return WEXITSTATUS(wstat); +} + +void usage(char *prog, int es) +{ + fprintf(stderr, "Usage: %s\n" + " [-s service] [-r restartinterval] [-p pidfile]" + " program [args ...]\n" + "or: [-p pidfile] [-k killcmd] -S\n", prog); + exit(es); +} + +int main(int argc, char *argv[]) +{ + bool doStop = false; + int restart = 0; + const char *service = "runserver"; + const char *pidfile = "vespa-runserver.pid"; // XXX bad default? + const char *killcmd = NULL; + + int ch; + while ((ch = getopt(argc, argv, "k:s:r:p:Sh")) != -1) { + switch (ch) { + case 's': + service = optarg; + break; + case 'r': + restart = atoi(optarg); + break; + case 'p': + pidfile = optarg; + break; + case 'S': + doStop = true; + break; + case 'k': + killcmd = optarg; + break; + default: + usage(argv[0], ch != 'h'); + } + } + + const char *envROOT = getenv("ROOT"); + if (envROOT == NULL || envROOT[0] == '\0') { + envROOT = vespa::Defaults::vespaHome(); + setenv("ROOT", envROOT, 1); + } + if (chdir(envROOT) != 0) { + fprintf(stderr, "Cannot chdir to %s: %s\n", envROOT, strerror(errno)); + exit(1); + } + + PidFile mypf(pidfile); + if (doStop) { + if (mypf.isRunning()) { + int pid = mypf.readPid(); + if (killcmd != NULL) { + fprintf(stdout, "%s was running with pid %d, running '%s' to stop it\n", + service, pid, killcmd); + if (system(killcmd) != 0) { + fprintf(stderr, "WARNING: stop command '%s' had some problem\n", killcmd); + } + } else { + fprintf(stdout, "%s was running with pid %d, sending SIGTERM\n", + service, pid); + if (killpg(pid, SIGTERM) != 0) { + fprintf(stderr, "could not signal %d: %s\n", pid, + strerror(errno)); + exit(1); + } + } + fprintf(stdout, "Waiting for exit (up to 60 seconds)\n"); + for (int cnt(0); cnt < 1800; cnt++) { + usleep(100000); // wait 0.1 seconds + if ((cnt > 300) && (cnt % 100 == 0)) { + killpg(pid, SIGTERM); + } + if (killpg(pid, 0) == 0) { + if (cnt%10 == 0) { + fprintf(stdout, "."); + fflush(stdout); + } + } else { + fprintf(stdout, "DONE\n"); + break; + } + if (cnt == 900) { + printf("\ngiving up, sending KILL signal\n"); + killpg(pid, SIGKILL); + } + } + } else { + fprintf(stdout, "%s not running according to %s\n", + service, pidfile); + } + mypf.cleanUp(); + exit(0); + } + if (optind >= argc || killcmd != NULL) { + usage(argv[0], 1); + } + + if (mypf.isRunning()) { + fprintf(stderr, "runserver already running with pid %d\n", + mypf.readPid()); + exit(0); + } + + if (!mypf.writeOpen()) { + perror(pidfile); + return 1; + } + + pid_t rsp = fork(); + if (rsp == 0) { + close(0); + if (open("/dev/null", O_RDONLY) != 0) { + perror("open /dev/null for reading failed"); + exit(1); + } + close(1); + if (open("/dev/null", O_WRONLY) != 1) { + perror("open /dev/null for writing failed"); + exit(1); + } + dup2(1, 2); + if (setsid() < 0) { + perror("setsid"); + exit(1); + } + struct sigaction act; + struct sigaction oact; + + memset(&act, 0, sizeof(act)); + + act.sa_handler = termsig; + + sigaction(SIGINT, &act, &oact); + sigaction(SIGTERM, &act, &oact); + + int stat = 0; + try { + mypf.writePid(); + do { + time_t laststart = time(NULL); + stat = loop(service, argv+optind); + if (restart > 0 && !gotstopsig) { + int wt = restart + laststart - time(NULL); + if (wt < 0) wt = 0; + LOG(info, "will restart in %d seconds", wt); + } + while (!gotstopsig && time(NULL) - laststart < restart) { + sleep(1); + } + } while (!gotstopsig && restart > 0); + } catch (MsgException& ex) { + LOG(error, "exception: '%s'", ex.what()); + exit(1); + } + if (restart > 0) { + LOG(debug, "final exit status: %d", stat); + } + mypf.cleanUp(); + exit(stat); + } + + if (rsp < 0) { + perror("fork"); + return 1; + } + printf("runserver(%s) running with pid: %d\n", service, rsp); + return 0; +} diff --git a/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java b/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java new file mode 100644 index 00000000000..eb55939b3be --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/DefaultLevelController.java @@ -0,0 +1,66 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.logging.Level; + +/** + * a levelcontroller that just implements a simple default + * (possibly controlled by a system property or environment) + **/ +class DefaultLevelController implements LevelController { + private String levelstring; + private Level levelLimit = LogLevel.EVENT; + + DefaultLevelController(String env) { + if (LogUtil.empty(env)) { + env = "all -debug -spam"; + } + + //level string is: fatal, error, warning, config, info, event, debug, spam + if (env.equals("all")) { + levelLimit = LogLevel.ALL; + levelstring = " ON ON ON ON ON ON ON ON"; + } else { + StringBuilder sb = new StringBuilder(); + for (Level level : LogLevel.getLevels().values()) { + String levelName = level.getName(); + if (hasNegWord(levelName, env) || (!hasWord("all", env) && !hasWord(levelName, env))) { + sb.append(" OFF"); + } else { + sb.append(" ON"); + if ((level.intValue() < levelLimit.intValue())) { + levelLimit = level; + } + } + } + levelstring = sb.toString(); + } + // System.err.println("default level controller levelstring: "+levelstring); + } + + private boolean hasWord(String levelName, String inputLevels) { + return inputLevels.contains(levelName.toLowerCase()); + } + + private boolean hasNegWord(String levelName, String inputLevels) { + int pos = inputLevels.indexOf(levelName.toLowerCase()); + if (pos > 0) { + String c = inputLevels.substring(pos - 1, pos); + return (c != null && c.equals("-")); + } else { + return false; + } + } + + public String getOnOffString() { + return levelstring; + } + + public Level getLevelLimit() { + return levelLimit; + } + public boolean shouldLog(Level level) { + return (level.intValue() >= levelLimit.intValue()); + } + public void checkBack() { } +} diff --git a/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java b/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java new file mode 100644 index 00000000000..73087daaeb1 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/FileLogTarget.java @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.*; + +/** + * @author lulf + * @since 5.1 + */ +public class FileLogTarget implements LogTarget { + private final File file; + private FileOutputStream fileOutputStream; + + public FileLogTarget(File target) throws FileNotFoundException { + this.file = target; + this.fileOutputStream = null; + } + + public synchronized OutputStream open() { + try { + close(); + fileOutputStream = new FileOutputStream(file, true); + } catch (FileNotFoundException e) { + throw new RuntimeException("Unable to open output stream", e); + } + return fileOutputStream; + } + + public synchronized void close() { + try { + if (fileOutputStream != null) { + fileOutputStream.close(); + fileOutputStream = null; + } + } catch (IOException e) { + throw new RuntimeException("Unable to close output stream", e); + } + } + + @Override + public String toString() { + return file.getAbsolutePath(); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java b/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java new file mode 100644 index 00000000000..a005f5098c8 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/InvalidLogFormatException.java @@ -0,0 +1,18 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +/** + * This (checked) exception is used to flag invalid log messages, + * primarily for use in the factory methods of LogMessage. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class InvalidLogFormatException extends Exception +{ + public InvalidLogFormatException (String msg) { + super(msg); + } + + public InvalidLogFormatException () { + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/LevelController.java b/vespalog/src/main/java/com/yahoo/log/LevelController.java new file mode 100644 index 00000000000..d0220ca4105 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LevelController.java @@ -0,0 +1,38 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * This is the interface for controlling the log level of a + * component logger. This hides the actual controlling + * mechanism. + * + * @author arnej27959 + * + */ + +/** + * @author arnej27959 + **/ +package com.yahoo.log; + +import java.util.logging.Level; + +public interface LevelController { + + /** + * should we actually publish a log message with the given Level now? + **/ + public boolean shouldLog(Level level); + + /** + * return a string suitable for printing in a logctl file. + * the string must be be 4 * 8 characters, where each group + * of 4 characters is either " ON" or " OFF". + **/ + public String getOnOffString(); + + /** + * check the current state of logging and reflect it into the + * associated Logger instance, if available. + **/ + public void checkBack(); + public Level getLevelLimit(); +} diff --git a/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java new file mode 100644 index 00000000000..65d6a6341f3 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LevelControllerRepo.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +/** + * The level controller repository is an interface towards something that is able to provide level + * controllers for a given component. + * + * @author lulf + * @since 5.1 + */ +public interface LevelControllerRepo { + /** + * Return the level controller for a given component. + * @param component The component name string. + * @return The LevelController corresponding to that component. Return null if not found. + */ + public LevelController getLevelController(String component); + + /** + * Close down the level controller repository. Cleanup should be done here. + */ + public void close(); +} diff --git a/vespalog/src/main/java/com/yahoo/log/LogLevel.java b/vespalog/src/main/java/com/yahoo/log/LogLevel.java new file mode 100644 index 00000000000..032f6e009fd --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogLevel.java @@ -0,0 +1,167 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.HashMap; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.logging.Level; + + +/** + * Note that the log levels defined in VESPA applications are the + * following. + * + * <UL> + * <LI> LogLevel.EVENT (1201) + * <LI> LogLevel.FATAL (1151) + * <LI> LogLevel.ERROR (1101) + * <LI> <em>LogLevel.SEVERE (1000)</em> + * <LI> LogLevel.WARNING (900) + * <LI> LogLevel.INFO (800) + * <LI> LogLevel.CONFIG (700) + * <LI> LogLevel.DEBUG (501) + * <LI> LogLevel.SPAM (299) + * </UL> + * + * <P> + * Note that the EVENT level is somewhat special and you must + * <b>never</b> log one of these messages manually, but use + * the {@link com.yahoo.log.event.Event} class for this. + * + * @author Bjorn Borud + * @author arnej27959 + */ + +public class LogLevel extends Level { + /** A map from the name of the log level to the instance */ + private static LinkedHashMap<String, Level> nameToLevel; + + /** A map from the java.util.logging loglevel to VESPA's loglevel */ + private static Map<Level, Level> javaToVespa; + + public static final int IntValEVENT = 1201; + public static final int IntValFATAL = 1161; + public static final int IntValERROR = 1101; + public static final int IntValUNKNOWN = 1001; + public static final int IntValSEVERE = 1000; + public static final int IntValWARNING = 900; + public static final int IntValINFO = 800; + public static final int IntValCONFIG = 700; + public static final int IntValDEBUG = 501; + public static final int IntValFINE = 500; + public static final int IntValFINER = 400; + public static final int IntValFINEST = 300; + public static final int IntValSPAM = 299; + + // these define the ordering of the Vespa levels logcontrol files. + // it must match the values of the LogLevel enum in <log/log.h> + // for the C++ framework: + // fatal, error, warning, config, info, event, debug, spam, NUM_LOGLEVELS + + public static final int LogCtlFATAL = 0; + public static final int LogCtlERROR = 1; + public static final int LogCtlWARNING = 2; + public static final int LogCtlCONFIG = 3; + public static final int LogCtlINFO = 4; + public static final int LogCtlEVENT = 5; + public static final int LogCtlDEBUG = 6; + public static final int LogCtlSPAM = 7; + public static final int LogCtlNumLevels = 8; + + // ordinary log levels + public static LogLevel UNKNOWN = new LogLevel("UNKNOWN", IntValUNKNOWN); + public static LogLevel EVENT = new LogLevel("EVENT", IntValEVENT); + public static LogLevel FATAL = new LogLevel("FATAL", IntValFATAL); + public static LogLevel ERROR = new LogLevel("ERROR", IntValERROR); + public static LogLevel DEBUG = new LogLevel("DEBUG", IntValDEBUG); + public static LogLevel SPAM = new LogLevel("SPAM", IntValSPAM); + + // overlapping ones, only mentioned for illustration + // + // public static LogLevel WARNING = new LogLevel("WARNING",900); + // public static LogLevel INFO = new LogLevel("INFO",800); + // public static LogLevel CONFIG = new LogLevel("CONFIG",700); + + static { + // define mapping from Java log levels to VESPA log + // levels. + javaToVespa = new HashMap<Level, Level>(); + javaToVespa.put(Level.SEVERE, ERROR); + javaToVespa.put(Level.WARNING, WARNING); + javaToVespa.put(Level.INFO, INFO); + javaToVespa.put(Level.CONFIG, CONFIG); + javaToVespa.put(Level.FINE, DEBUG); + javaToVespa.put(Level.FINER, DEBUG); + javaToVespa.put(Level.FINEST, SPAM); + + // need the VESPA ones too + javaToVespa.put(UNKNOWN, UNKNOWN); + javaToVespa.put(FATAL, FATAL); + javaToVespa.put(ERROR, ERROR); + javaToVespa.put(EVENT, EVENT); + javaToVespa.put(DEBUG, DEBUG); + javaToVespa.put(SPAM, SPAM); + + // manually enter the valid log levels we shall recognize + // in VESPA + nameToLevel = new LinkedHashMap<String, Level>(15); + nameToLevel.put("fatal", FATAL); + nameToLevel.put("error", ERROR); + nameToLevel.put("warning", WARNING); + nameToLevel.put("config", CONFIG); + nameToLevel.put("info", INFO); + nameToLevel.put("event", EVENT); + nameToLevel.put("debug", DEBUG); + nameToLevel.put("spam", SPAM); + } + + private LogLevel(String name, int value) { + super(name, value); + } + + /** + * Semi-Case sensitive parsing of log levels. <b>Log levels are + * in either all upper case or all lower case. Not mixed + * case. </b>. Returns static instance representing log level or + * the UNKNOWN LogLevel instance. + * + * @param name Name of loglevel in uppercase or lowercase. + * @return Returns the static (immutable) LogLevel instance + * equivalent to the name given. + * + */ + public static Level parse(String name) { + Level l = nameToLevel.get(name); + if (l == null) { + return UNKNOWN; + } + return l; + } + + /** + * Static method for mapping Java log level to VESPA log level. + * + * @param level The Java loglevel we want mapped to its VESPA + * counterpart + * @return The VESPA LogLevel instance representing the corresponding + * log level or the UNKNOWN instance if the log level was + * unknown (ie. not contained in the mapping. Should never + * happen). + */ + public static Level getVespaLogLevel(Level level) { + Level ll = javaToVespa.get(level); + if (ll != null) { + return ll; + } + return UNKNOWN; + } + + /** + * Static method returning a map from Vespa level name to Level + * + * @return a map from Vespa level name to Level + */ + public static HashMap<String, Level> getLevels() { + return nameToLevel; + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessage.java b/vespalog/src/main/java/com/yahoo/log/LogMessage.java new file mode 100644 index 00000000000..123c10d0d95 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogMessage.java @@ -0,0 +1,151 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.yahoo.log.event.Event; +import com.yahoo.log.event.MalformedEventException; + +/** + * This class implements the common ground log message used by + * the logserver. A LogMessage is immutable. Note that we have + * chosen the name LogMessage to avoid confusion with LogRecord + * which is used in java.util.logging. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class LogMessage +{ + private static Logger log = Logger.getLogger(LogMessage.class.getName()); + + private static Pattern nativeFormat = + Pattern.compile("^(\\d[^\t]+)\t" + // time + "([^\t]+)\t" + // host + "([^\t]+)\t" + // threadProcess + "([^\t]+)\t" + // service + "([^\t]+)\t" + // component + "([^\t]+)\t" + // level + "(.+)$" // payload + ); + + private long time; + private String timeStr; + private String host; + private String threadProcess; + private String service; + private String component; + private Level level; + private String payload; + private Event event; + + /** + * Private constructor. Log messages should never be instantiated + * directly; only as the result of a static factory method. + */ + private LogMessage (String timeStr, Long time, String host, String threadProcess, + String service, String component, Level level, + String payload) + { + this.timeStr = timeStr; + this.time = time; + this.host = host; + this.threadProcess = threadProcess; + this.service = service; + this.component = component; + this.level = level; + this.payload = payload; + } + + public long getTime () {return time;} + public long getTimeInSeconds () {return time / 1000;} + public String getHost () {return host;} + public String getThreadProcess () {return threadProcess;} + public String getService () {return service;} + public String getComponent () {return component;} + public Level getLevel () {return level;} + public String getPayload () {return payload;} + + /** + * Make a log message from the native format of the logging + * package. + * + * @param msg The log message + * @return Returns a LogMessage instance + * @throws InvalidLogFormatException if the log message + * can not be parsed, ie. is invalid, we throw this + * exception. + */ + public static LogMessage parseNativeFormat(String msg) throws InvalidLogFormatException { + Matcher m = nativeFormat.matcher(msg); + if (! m.matches()) { + throw new InvalidLogFormatException(msg); + } + + Level msgLevel = LogLevel.parse(m.group(6)); + Long timestamp = parseTimestamp(m.group(1)); + + return new LogMessage(m.group(1), timestamp, m.group(2), m.group(3), + m.group(4), m.group(5), msgLevel, + m.group(7)); + } + + private static long parseTimestamp(String timeStr) throws InvalidLogFormatException { + try { + return (long) (Double.parseDouble(timeStr) * 1000); + } catch (NumberFormatException e) { + throw new InvalidLogFormatException("Invalid time string:" + timeStr); + } + } + + /** + * If the LogMessage was an EVENT then this method can + * be used to get the Event instance representing the + * event. The event instance created the first time + * this method is called and then cached. + * + * TODO: make sure this throws exception! + * + * @return Returns Event instance if this is an event message + * and the payload is correctly formatted. Otherwise + * it will return <code>null</code>. + * + */ + public Event getEvent () throws MalformedEventException { + if ((level == LogLevel.EVENT) && (event == null)) { + try { + event = Event.parse(getPayload()); + event.setTime(time); + } + catch (MalformedEventException e) { + log.log(LogLevel.DEBUG, "Got malformed event: " + getPayload()); + throw e; + } + } + return event; + } + + /** + * Return valid representation of log message. + */ + public String toString () { + return new StringBuilder(timeStr.length() + + host.length() + + threadProcess.length() + + service.length() + + component.length() + + level.toString().length() + + payload.length() + + 1) + .append(timeStr).append("\t") + .append(host).append("\t") + .append(threadProcess).append("\t") + .append(service).append("\t") + .append(component).append("\t") + .append(level.toString().toLowerCase()).append("\t") + .append(payload).append("\n") + .toString(); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java b/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java new file mode 100644 index 00000000000..f11dc9ec139 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogMessageTimeComparator.java @@ -0,0 +1,45 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.Serializable; +import java.util.Comparator; + +/** + * Order LogMessage instances based on timestamp. The default ordering is ascending. + * This may be reversed by the constructor argument. + * + * Note: this comparator imposes orderings that are inconsistent with equals. + * This is due to only looking at the timestamp, so two different messages with + * the same timestamp would appear "equal" to this comparator. + * + * @author vlarsen + * + */ +public class LogMessageTimeComparator implements Comparator<LogMessage>, Serializable { + private static final long serialVersionUID = 1L; + + /** Indicate the ordering of the timecomparison */ + private boolean ascending = true; + + /** + * Create a Time comparator for logmessages. Order is ascending. + * + */ + public LogMessageTimeComparator() {} + + /** + * Create a Time comparator for logmessages. The chronological order is dependent + * on the argument to the constructor. + * + * @param ascending true if you want LogMessages ordered ascending according to timestamp. + */ + public LogMessageTimeComparator(boolean ascending) { + this.ascending = ascending; + } + + public int compare(LogMessage message1, LogMessage message2) { + return ascending ? + Long.valueOf(message1.getTime()).compareTo(Long.valueOf(message2.getTime())) + : Long.valueOf(message2.getTime()).compareTo(Long.valueOf(message1.getTime())); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/LogSetup.java b/vespalog/src/main/java/com/yahoo/log/LogSetup.java new file mode 100644 index 00000000000..581a71e2b2b --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogSetup.java @@ -0,0 +1,198 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Enumeration; +import java.util.logging.*; +import java.util.Timer; + +/** + * Sets up Vespa logging. Call a setup method to set up this. + * + * @author Bjorn Borud + * @author arnej27959 + */ +public class LogSetup { + + private static Timer taskRunner = new Timer(true); + /** A global task thread */ + public static Timer getTaskRunner() { return taskRunner; } + + /** The log handler used by this */ + private static VespaLogHandler logHandler; + + public static void clearHandlers () { + Enumeration<String> names = LogManager.getLogManager().getLoggerNames(); + while (names.hasMoreElements()) { + String name = names.nextElement(); + Logger logger = Logger.getLogger(name); + + Handler[] handlers = logger.getHandlers(); + for (Handler handler : handlers) { + logger.removeHandler(handler); + } + } + } + + private static boolean isInitialized = false; + + /** + * Every Vespa application should call initVespaLogging exactly + * one time. This should be done from the main() method or from a + * static initializer in the main class. The library will pick up + * the environment variables usually set by the Vespa + * config-sentinel (VESPA_LOG_LEVEL, VESPA_LOG_TARGET, + * VESPA_SERVICE_NAME, VESPA_LOG_CONTROL_DIR) but it's possible to + * override these by setting system properties before calling + * initVespaLogging. This may be useful for unit testing etc: + * <br> + * System.setProperty("vespa.log.level", "all") + * <br> + * System.setProperty("vespa.log.target", "file:foo.log") + * <br> + * System.setProperty("vespa.service.name", "my.name") + * <br> + * System.setProperty("vespa.log.control.dir", ".") + * <br> + * System.setProperty("vespa.log.control.file", "my.logcontrol") + * <br> + * vespa.log.control.file is used if it's set, otherwise it's + * vespa.log.control.dir + "/" + vespa.service.name + ".logcontrol" + * if both of those variables are set, otherwise there will be no + * runtime log control. + * + * @param programName the name of the program that is running; + * this is added as a prefix to the logger name to form the + * "component" part of the log message. (Usually the logger name + * is the name of the class that logs something, so the + * programName should be kept short and simple.) + **/ + public static void initVespaLogging(String programName) { + if (isInitialized) { + System.err.println("WARNING: initVespaLogging called twice"); + } + isInitialized = true; + + // prefer Java system properties + String logLevel = System.getProperty("vespa.log.level"); + String logTarget = System.getProperty("vespa.log.target"); + String logService = System.getProperty("vespa.service.name"); + String logControlDir = System.getProperty("vespa.log.control.dir"); + String logControlFile = System.getProperty("vespa.log.control.file"); + if (programName == null || programName.equals("")) { + throw new RuntimeException("invalid programName: "+programName); + } + + // then try environment values + if (logTarget == null) logTarget = System.getenv("VESPA_LOG_TARGET"); + if (logService == null) logService = System.getenv("VESPA_SERVICE_NAME"); + if (logControlDir == null) logControlDir = System.getenv("VESPA_LOG_CONTROL_DIR"); + if (logControlFile == null) logControlFile = System.getenv("VESPA_LOG_CONTROL_FILE"); + if (logLevel == null) logLevel = System.getenv("VESPA_LOG_LEVEL"); + + // then hardcoded defaults + if (logTarget == null) logTarget = "fd:2"; + if (logLevel == null) logLevel = "all -debug -spam"; + + if (logControlFile == null && + logControlDir != null && + logService != null && + !logService.equals("") && + !logService.equals("-")) + { + logControlFile = logControlDir + "/" + logService + ".logcontrol"; + } + + // for backwards compatibility - XXX should be removed + if (logService == null) logService = System.getProperty("config.id"); + if (logService == null) logService = "-"; + + System.setProperty("vespa.service.name", logService); + System.setProperty("vespa.program.name", programName); + + try { + initInternal(logTarget, logService, logControlFile, programName, logLevel); + } catch (FileNotFoundException e) { + throw new RuntimeException("Unable to initialize logging", e); + } + } + + private static LogTarget getLogTargetFromString(String target) throws FileNotFoundException { + if ("fd:2".equals(target)) { + return new StderrLogTarget(); + } else if ("fd:1".equals(target)) { + return new StdoutLogTarget(); + } else if (target.startsWith("file:")) { + return new FileLogTarget(new File(target.substring(5))); + } + throw new IllegalArgumentException("Target '" + target + "' is not a valid target"); + } + + private static void initInternal(String target, + String service, + String logCtlFn, + String app, + String lev) throws FileNotFoundException { + clearHandlers(); + + if (app != null && app.length() > 64) app = app.substring(0, 63); + + if (logHandler != null) { + logHandler.cleanup(); + Logger.getLogger("").removeHandler(logHandler); + } + Logger.getLogger("").setLevel(Level.ALL); + logHandler = new VespaLogHandler(getLogTargetFromString(target), new VespaLevelControllerRepo(logCtlFn, lev, app), service, app); + String zookeeperLogFile = System.getProperty("zookeeperlogfile"); + if (zookeeperLogFile != null) { + logHandler.setFilter(new ZooKeeperFilter(zookeeperLogFile)); + } + Logger.getLogger("").addHandler(logHandler); + } + + /** Returns the log handler set up by this class */ + public static VespaLogHandler getLogHandler() { + return logHandler; + } + + /** + * Class that has an isLoggable methods that handles log records that + * start with "org.apache.zookeeper." in + * a special way (writing them to the logfile specified in the system property + * zookeeperlogfile, "/tmp/zookeeper.log" if not set) and returning false. + * For other log records, isLoggable returns true + */ + static class ZooKeeperFilter implements Filter { + private FileHandler fileHandler; + + ZooKeeperFilter(String logFile) { + try { + fileHandler = new FileHandler(logFile, true); + fileHandler.setFormatter(new VespaFormatter()); + } catch (IOException e) { + System.out.println("Not able to create " + logFile); + fileHandler = null; + } + } + + /** + * Return true if loggable (ordinary log record), returns false if this filter + * logs it itself + * @param record a #{@link LogRecord} + * @return true if loggable, false otherwise + */ + @Override + public boolean isLoggable(LogRecord record) { + if (record.getLoggerName() == null) return true; + if (!record.getLoggerName().startsWith("org.apache.zookeeper.") && + !record.getLoggerName().startsWith("com.netflix.curator")) { + return true; + } + fileHandler.publish(record); + return false; + } + } + +} diff --git a/vespalog/src/main/java/com/yahoo/log/LogTarget.java b/vespalog/src/main/java/com/yahoo/log/LogTarget.java new file mode 100644 index 00000000000..5ba5ba7769f --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogTarget.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.OutputStream; + +/** + * @author lulf + * @since 5.1 + */ +public interface LogTarget { + /** + * Opens an output stream for the target. If already open, the stream should be reopened. + * @return a new outputstream for the log target. + */ + public OutputStream open(); + + /** + * Close the log target, ensuring that all data is written. + */ + public void close(); +} diff --git a/vespalog/src/main/java/com/yahoo/log/LogUtil.java b/vespalog/src/main/java/com/yahoo/log/LogUtil.java new file mode 100644 index 00000000000..e803ee48156 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/LogUtil.java @@ -0,0 +1,12 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +/** + * @author lulf + * @since 5.1 + */ +class LogUtil { + static boolean empty(String s) { + return (s == null || s.equals("")); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java new file mode 100644 index 00000000000..9cfc25a2dae --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelController.java @@ -0,0 +1,120 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import com.yahoo.text.Utf8; + +import java.nio.MappedByteBuffer; +import java.util.logging.Level; + +/** + * a level controller that does lookup in a file via a memory-mapped + * buffer for realtime logging control. + **/ +class MappedLevelController implements LevelController { + private static final int ONVAL = 0x20204f4e; // equals " ON" in file + private static final int OFFVAL = 0x204f4646; // equals " OFF" in file + + private MappedByteBuffer mapBuf; + private int offset; + private java.util.logging.Logger associate; + public MappedLevelController(MappedByteBuffer buf, + int firstoffset, + String name) + { + this.mapBuf = buf; + this.offset = firstoffset; + this.associate = java.util.logging.Logger.getLogger(name); + } + + /** + * return the current state as a string + * (directly fetched from the file via the mapping buffer) + **/ + public String getOnOffString() { + byte[] levels = new byte[4 * VespaLevelControllerRepo.numLevels]; + for (int i = 0; i < levels.length; i++) { + levels[i] = mapBuf.get(offset + i); + } + return Utf8.toString(levels); + } + + /** + * check that each controlled level is either ON or OFF. + **/ + public static boolean checkOnOff(MappedByteBuffer mapBuf, + int offset) + { + for (int i = 0; i < VespaLevelControllerRepo.numLevels; i++) { + int off = offset + 4 * i; + int val = mapBuf.getInt(off); + if (val != ONVAL && val != OFFVAL) { + System.err.println("bad on/off value: "+val); + return false; + } + } + return true; + } + + /** + * make sure our associated java.util.Logger instance + * gets the correct logging level so it can avoid sending + * us lots of debug and spam log messages that will + * be discarded in the usual case. + **/ + public void checkBack() { + associate.setLevel(getLevelLimit()); + } + public Level getLevelLimit() { + Level lvl; + if (isOn(LogLevel.LogCtlSPAM)) { + lvl = LogLevel.ALL; + } else if (isOn(LogLevel.LogCtlDEBUG)) { + lvl = LogLevel.FINE; + } else if (isOn(LogLevel.LogCtlCONFIG)) { + lvl = LogLevel.CONFIG; + } else if (isOn(LogLevel.LogCtlINFO)) { + lvl = LogLevel.INFO; + } else if (isOn(LogLevel.LogCtlWARNING)) { + lvl = LogLevel.WARNING; + } else { + lvl = LogLevel.SEVERE; + } + return lvl; + } + + /** + * is a specific Vespa level ON or OFF in the file? + **/ + private boolean isOn(int num) { + int off = offset + num*4; + int val = mapBuf.getInt(off); + if (val == OFFVAL) + return false; + return true; + } + + /** + * should we publish a log messages on the given java Level? + **/ + public boolean shouldLog(Level level) { + int val = level.intValue(); + + // event is special and handled first: + if (val == LogLevel.IntValEVENT) { return isOn(LogLevel.LogCtlEVENT); } + + // all other levels are handled in "severity order": + + if (val >= LogLevel.IntValFATAL) { return isOn(LogLevel.LogCtlFATAL); } + // LogLevel.ERROR between here + if (val >= LogLevel.IntValSEVERE) { return isOn(LogLevel.LogCtlERROR); } + if (val >= LogLevel.IntValWARNING) { return isOn(LogLevel.LogCtlWARNING); } + if (val >= LogLevel.IntValINFO) { return isOn(LogLevel.LogCtlINFO); } + if (val >= LogLevel.IntValCONFIG) { return isOn(LogLevel.LogCtlCONFIG); } + // LogLevel.DEBUG between here + // LogLevel.FINE between here + if (val >= LogLevel.IntValFINER) { return isOn(LogLevel.LogCtlDEBUG); } + // LogLevel.FINEST and + // LogLevel.SPAM: + return isOn(LogLevel.LogCtlSPAM); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java new file mode 100644 index 00000000000..eb37ca47835 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/MappedLevelControllerRepo.java @@ -0,0 +1,114 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import com.yahoo.text.Utf8; + +import java.nio.MappedByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** + * Contains a repository of mapped log level controllers. + * + * @author lulf + * @since 5.1 + */ +public class MappedLevelControllerRepo { + private final Map<String, LevelController> levelControllerMap = new HashMap<>(); + private final MappedByteBuffer mapBuf; + private final int controlFileHeaderLength; + private final int numLevels; + private final String logControlFilename; + + public MappedLevelControllerRepo(MappedByteBuffer mapBuf, int controlFileHeaderLength, int numLevels, String logControlFilename) { + this.mapBuf = mapBuf; + this.controlFileHeaderLength = controlFileHeaderLength; + this.numLevels = numLevels; + this.logControlFilename = logControlFilename; + buildMap(); + } + + private void buildMap() { + int len = mapBuf.capacity(); + int startOfLine = controlFileHeaderLength; + + int numLine = 1; + int i = 0; + while (i < len) { + if (mapBuf.get(i) == '\n') { + startOfLine = ++i; + ++numLine; + } else if (i < controlFileHeaderLength) { + ++i; + } else if (mapBuf.get(i) == ':') { + int endOfName = i; + int levels = i; + levels += 2; + while ((levels % 4) != 0) { + levels++; + } + int endLine = levels + 4*numLevels; + + if (checkLine(startOfLine, endOfName, levels, endLine)) { + int l = endOfName - startOfLine; + if (l > 1 && mapBuf.get(startOfLine) == '.') { + ++startOfLine; + --l; + } + byte[] namebytes = new byte[l]; + for (int j = 0; j < l; j++) { + namebytes[j] = mapBuf.get(startOfLine + j); + } + String name = Utf8.toString(namebytes); + if (name.equals("default")) { + name = ""; + } + MappedLevelController ctrl = new MappedLevelController(mapBuf, levels, name); + levelControllerMap.put(name, ctrl); + i = endLine; + continue; // good line + } + // bad line, skip + while (i < len && mapBuf.get(i) != '\n') { + i++; + } + int bll = i - startOfLine; + byte[] badline = new byte[bll]; + for (int j = 0; j < bll; j++) { + badline[j] = mapBuf.get(startOfLine + j); + } + System.err.println("bad loglevel line "+numLine+" in " + + logControlFilename + ": " + Utf8.toString(badline)); + } else { + i++; + } + } + } + + private boolean checkLine(int sol, int endnam, int levstart, int eol) { + if (eol >= mapBuf.capacity()) { + System.err.println("line would end after end of file"); + return false; + } + if (mapBuf.get(eol) != '\n') { + System.err.println("line must end with newline, was: "+mapBuf.get(eol)); + return false; + } + if (endnam < sol + 1) { + System.err.println("name must be at least one character after start of line"); + return false; + } + return MappedLevelController.checkOnOff(mapBuf, levstart); + } + + public LevelController getLevelController(String suffix) { + + return levelControllerMap.get(suffix); + } + + public void checkBack() { + for (LevelController ctrl : levelControllerMap.values()) { + ctrl.checkBack(); + } + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/RejectFilter.java b/vespalog/src/main/java/com/yahoo/log/RejectFilter.java new file mode 100644 index 00000000000..980ce52ecb1 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/RejectFilter.java @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.ArrayList; +import java.util.List; + +/** + * A RejectFilter can be queried to see if a log message should be rejected. + * + * @author lulf + * @since 5.1 + */ +public class RejectFilter { + private final List<String> rejectedMessages = new ArrayList<>(); + + public boolean shouldReject(String message) { + if (message == null) + return false; + + for (String rejectedMessage : rejectedMessages) { + if (message.contains(rejectedMessage)) { + return true; + } + } + return false; + } + + public void addRejectedMessage(String rejectedMessage) { + rejectedMessages.add(rejectedMessage); + } + + public static RejectFilter createDefaultRejectFilter() { + RejectFilter reject = new RejectFilter(); + reject.addRejectedMessage("Using FILTER_NONE: This must be paranoid approved, and since you are using FILTER_NONE you must live with this error."); + return reject; + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java b/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java new file mode 100644 index 00000000000..fa52cedbb2b --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/StderrLogTarget.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.OutputStream; + +/** + * @author lulf + * @since 5.1 + */ +public class StderrLogTarget implements LogTarget { + + @Override + public OutputStream open() { + return new UncloseableOutputStream(System.err); + } + + @Override + public void close() { + // ignore + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java b/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java new file mode 100644 index 00000000000..e10cf1c6792 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/StdoutLogTarget.java @@ -0,0 +1,21 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.OutputStream; + +/** + * @author lulf + * @since 5.1 + */ +public class StdoutLogTarget implements LogTarget { + + @Override + public OutputStream open() { + return new UncloseableOutputStream(System.out); + } + + @Override + public void close() { + // ignore + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java b/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java new file mode 100644 index 00000000000..67bd48f0654 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/UncloseableOutputStream.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + * @since 5.1.14 + */ +class UncloseableOutputStream extends OutputStream { + + private final OutputStream out; + + public UncloseableOutputStream(OutputStream out) { + this.out = out; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void close() { + // ignore + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/Util.java b/vespalog/src/main/java/com/yahoo/log/Util.java new file mode 100644 index 00000000000..062d6acf960 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/Util.java @@ -0,0 +1,61 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.InetAddress; + +/** + * + * @author Bjorn Borud + * @author arnej27959 + * + */ +public class Util { + + /** + * We do not have direct access to the <code>gethostname</code> + * system call, so we have to fake it. + */ + public static String getHostName () { + String hostname = "-"; + try { + Process p = Runtime.getRuntime().exec("hostname"); + BufferedReader r = new BufferedReader( + new InputStreamReader(p.getInputStream(), "UTF-8")); + hostname = r.readLine(); + if (hostname != null) { + return hostname; + } + } + catch (java.io.IOException e) {} + try { + hostname = InetAddress.getLocalHost().getHostName(); + return hostname; + } + catch (java.net.UnknownHostException e) {} + return "-"; + } + + /** + * Emulate the getpid() system call + **/ + public static String getPID() { + try { + Process p = Runtime.getRuntime().exec( + new String[] {"perl", "-e", "print getppid().\"\\n\";"} + ); + BufferedReader r = new BufferedReader( + new InputStreamReader(p.getInputStream(), "UTF-8")); + String line = r.readLine(); + p.destroy(); + int pid = Integer.parseInt(line); + if (pid > 0) { + return Integer.toString(pid); + } + } catch(Exception e) { + // any problem handled by return below + } + return "-"; + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormat.java b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java new file mode 100644 index 00000000000..a1efd02513e --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormat.java @@ -0,0 +1,196 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Vespa log formatting utility methods. + * Contains some code based on Util.java in Cloudname https://github.com/Cloudname/cloudname + * written by Bjørn Borud, licensed under the Apache 2.0 license. + * + * @author arnej27959 + */ +public class VespaFormat { + + private static final Pattern special = Pattern.compile("[\r\\\n\\\t\\\\]+"); + private static final Pattern newLine = Pattern.compile("\n"); + private static final Pattern carriage = Pattern.compile("\r"); + private static final Pattern tab = Pattern.compile("\t"); + private static final Pattern backSlash = Pattern.compile("\\\\"); + + private static final String hostname; + private static final String processID; + + static { + hostname = Util.getHostName(); + processID = Util.getPID(); + } + + /** + * This static method is used to detect if a message needs + * to be escaped, and if so, performs the escaping. Since the + * common case is most likely that escaping is <em>not</em> + * needed, the code is optimized for this case. The forbidden + * characters are: + * + * <UL> + * <LI> newline + * <LI> tab + * <LI> backslash + * </UL> + * + * <P> + * Also handles the case where the message is <code>null</code> + * and replaces the null message with a tag saying that the + * value was "(empty)". + * + * @param s String that might need escaping + * @return Returns escaped string + * + */ + public static String escape (String s) { + if (s == null) { + return "(empty)"; + } + + Matcher m = special.matcher(s); + if (! m.find()) { + return s; + } + + // invariant: we had special characters + + m = backSlash.matcher(s); + if (m.find()) { + s = m.replaceAll("\\\\\\\\"); + } + + m = newLine.matcher(s); + if (m.find()) { + s = m.replaceAll("\\\\n"); + } + + m = carriage.matcher(s); + if (m.find()) { + s = m.replaceAll("\\\\r"); + } + + m = tab.matcher(s); + if (m.find()) { + s = m.replaceAll("\\\\t"); + } + + return s; + } + + + /** + * It is easier to slice and dice strings in Java than formatting + * numbers... + */ + public static void formatTime (long time, StringBuilder sbuffer) { + String timeString = Long.toString(time); + int len = timeString.length(); + + // something wrong. handle it by just returning the input + // long as a string. we prefer this to just crashing in + // the substring handling. + if (len < 3) { + sbuffer.append(timeString); + return; + } + sbuffer.append(timeString.substring(0, len - 3)); + sbuffer.append('.'); + sbuffer.append(timeString.substring(len - 3)); + } + + public static String format(String levelName, + String component, + String componentPrefix, + long millis, + String threadId, + String serviceName, + String formattedMessage, + Throwable t) + { + StringBuilder sbuf = new StringBuilder(300); // initial guess + + // format the time + formatTime(millis, sbuf); + sbuf.append("\t"); + + sbuf.append(hostname).append("\t"); + + sbuf.append(processID); + if (threadId != null) { + sbuf.append("/").append(threadId); + } + sbuf.append("\t"); + + sbuf.append(serviceName).append("\t"); + + if (component == null && componentPrefix == null) { + sbuf.append("-"); + } else if (component == null) { + sbuf.append(componentPrefix); + } else if (componentPrefix == null) { + sbuf.append(".").append(component); + } else { + sbuf.append(componentPrefix).append(".").append(component); + } + sbuf.append("\t"); + + sbuf.append(levelName).append("\t"); + + sbuf.append(escape(formattedMessage)); + if (t != null) { + formatException(t, sbuf); + } + sbuf.append("\n"); + return sbuf.toString(); + } + + /** + * Format throwable into given StringBuffer. + * + * @param t The Throwable we want to format + * @param sbuf The stringbuffer into which we wish to + * format the Throwable + */ + public static void formatException (Throwable t, StringBuilder sbuf) { + Throwable last = t; + int depth = 0; + while (last != null) { + sbuf.append("\\nmsg=\""); + sbuf.append(escape(last.getMessage())); + sbuf.append("\"\\nname=\""); + sbuf.append(escape(last.getClass().getName())); + sbuf.append("\"\\nstack=\"\\n"); + + // loop through stack frames and format them + StackTraceElement[] st = last.getStackTrace(); + int stopAt = Math.min(st.length, 15); + boolean first = true; + for (int i = 0; i < stopAt; i++) { + if (first) { + first = false; + } else { + sbuf.append("\\n"); + } + sbuf.append(escape(st[i].toString())); + } + + // tell the reader if we chopped off part of the stacktrace + if (stopAt < st.length) { + sbuf.append("\\n[...]"); + } + sbuf.append("\\n\""); + + last = last.getCause(); + depth++; + } + sbuf.append(" nesting=").append(depth); + } + +} diff --git a/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java new file mode 100644 index 00000000000..ee0f6e90b7c --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/VespaFormatter.java @@ -0,0 +1,172 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// $Id$ +package com.yahoo.log; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.logging.LogRecord; +import java.util.logging.SimpleFormatter; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.yahoo.log.event.Event; + +/** + * This class implements a log formatter which takes care of + * formatting messages according to the VESPA common log format. + * + * @author Bjorn Borud + * @author arnej27959 + * + */ +public class VespaFormatter extends SimpleFormatter { + + private static final Pattern backSlash = Pattern.compile("\\\\"); + + // other way around + private static final Pattern backSlashN = Pattern.compile("\\\\n"); + private static final Pattern backSlashT = Pattern.compile("\\\\t"); + private static final Pattern backSlash2 = Pattern.compile("\\\\\\\\"); + + private static final String hostname; + private static final String processID; + + public static final String serviceNameUnsetValue = "-"; + + static { + hostname = Util.getHostName(); + processID = Util.getPID(); + } + + private String serviceName; + private String componentPrefix; + + /** + * Default constructor + */ + public VespaFormatter() { + this.serviceName = serviceNameUnsetValue; + } + + /** + * @param serviceName The VESPA service name. + * @param componentPrefix The application name. + */ + public VespaFormatter(String serviceName, String componentPrefix) { + if (serviceName == null) { + this.serviceName = serviceNameUnsetValue; + } else { + this.serviceName = serviceName; + } + this.componentPrefix = componentPrefix; + } + + /** + * Un-escapes previously escaped string. + * note: look at com.yahoo.config.StringNode.unescapeQuotedString() + * + * @param s String that might need un-escaping + * @return Returns un-escaped string + */ + public static String unEscape(String s) { + Matcher m = backSlash.matcher(s); + if (! m.find()) { + return s; + } + m = backSlashN.matcher(s); + if (m.find()) { + s = m.replaceAll("\n"); + } + m = backSlashT.matcher(s); + if (m.find()) { + s = m.replaceAll("\t"); + } + m = backSlash2.matcher(s); + if (m.find()) { + s = m.replaceAll("\\\\"); + } + return s; + } + + public String format(LogRecord r) { + StringBuilder sbuf = new StringBuilder(300); // initial guess + + String levelName = LogLevel.getVespaLogLevel(r.getLevel()) + .toString() + .toLowerCase(); + + String component = r.getLoggerName(); + + // format the time + VespaFormat.formatTime(r.getMillis(), sbuf); + sbuf.append("\t"); + + sbuf.append(hostname).append("\t") + .append(processID).append("/") + .append(r.getThreadID()).append("\t") + .append(serviceName).append("\t"); + + if (component == null && componentPrefix == null) { + sbuf.append("-"); + } else if (component == null) { + sbuf.append(componentPrefix); + } else if (componentPrefix == null) { + sbuf.append(".").append(component); + } else { + sbuf.append(componentPrefix).append(".").append(component); + } + + sbuf.append("\t").append(levelName).append("\t"); + + // for events, there is no ordinary message string; + // instead we render a string represantion of the event object: + if (r.getLevel() == LogLevel.EVENT) { + Event event = (Event) r.getParameters()[0]; + sbuf.append(VespaFormat.escape(event.toString())); + } else { + // otherwise, run standard java text formatting on the message + sbuf.append(VespaFormat.escape(formatMessage(r))); + } + appendException(r.getThrown(), sbuf); + + + sbuf.append("\n"); + return sbuf.toString(); + } + + private void appendException(Throwable throwable, StringBuilder builder) { + if (throwable == null) + return; + + String escapedStackTrace = VespaFormat.escape(stackTrace(throwable)); + builder.append("\\n").append("exception=").append("\\n").append(escapedStackTrace); + } + + private String stackTrace(Throwable throwable) { + StringWriter writer = new StringWriter(); + PrintWriter wrappedWriter = new PrintWriter(writer); + throwable.printStackTrace(wrappedWriter); + wrappedWriter.close(); + return writer.toString(); + } + + + /** + * Set the service name (usually the VESPA config-id) of this + * formatter. + * + * @param serviceName The service name + */ + public void setServiceName (String serviceName) { + this.serviceName = serviceName; + } + + /** + * Get the service name for this formatter. + * + * @return Returns the service name. + */ + public String getServiceName () { + return serviceName; + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java new file mode 100644 index 00000000000..f2542437e5f --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/VespaLevelControllerRepo.java @@ -0,0 +1,236 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import com.yahoo.text.Utf8; + +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Enumeration; +import java.util.TimerTask; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +/** + * @author lulf + * @since 5.1 + */ +public class VespaLevelControllerRepo implements LevelControllerRepo { + + private RandomAccessFile ctlFile; + private MappedByteBuffer mapBuf; + private MappedLevelControllerRepo levelControllerRepo; + private final String logControlFilename; + private final String appPrefix; + + private static final int maxPrefix = 64; + private static final String CFHEADER = "Vespa log control file version 1\n"; + private static final String CFPREPRE = "Prefix: "; + + /** + * length of fixed header content of a control file, constant: + **/ + public static final int controlFileHeaderLength; + /** + * number of distinctly controlled levels (in logctl files), + * must be compatible with C++ Vespa logging + **/ + public static final int numLevels = 8; + + static { + controlFileHeaderLength = CFHEADER.length() + + CFPREPRE.length() + + 1 // newline + + maxPrefix + + 1; // newline + } + /** + * level controller for default levels (first-time logging + * or logging without a logcontrol file) + **/ + private LevelController defaultLevelCtrl; + private String defaultLogLevel; + + public VespaLevelControllerRepo(String logCtlFn, String logLevel, String applicationPrefix) { + this.logControlFilename = logCtlFn; + this.defaultLogLevel = logLevel; + this.appPrefix = applicationPrefix; + defaultLevelCtrl = new DefaultLevelController(defaultLogLevel); + openCtlFile(); + } + + private void resetCtlFile() { + // System.err.println("reset limit to: "+defaultLevelCtrl.getLevelLimit()); + Logger.getLogger("").setLevel(defaultLevelCtrl.getLevelLimit()); + try { + if (ctlFile != null) { + ctlFile.close(); + } + } catch (java.io.IOException ign) {} + ctlFile = null; + mapBuf = null; + levelControllerRepo = null; + } + + private void openCtlFile() { + if (ctlFile != null) { + // already done this + return; + } + if (logControlFilename == null) { + // System.err.println("initialize limit to: "+defaultLevelCtrl.getLevelLimit()); + Logger.getLogger("").setLevel(defaultLevelCtrl.getLevelLimit()); + // only default level controller, very little can be done + return; + } + + try { + ctlFile = new RandomAccessFile(logControlFilename, "rw"); + ensureHeader(); + extendMapping(); + + if (checkBackRunner == null) { + checkBackRunner = new CheckBackRunner(); + LogSetup.getTaskRunner().schedule(checkBackRunner, 1000, 9999); + } + } catch (java.io.IOException e) { + System.err.println("problem opening logcontrol file " + +logControlFilename+": "+e); + resetCtlFile(); + } + } + + private void ensureHeader() throws java.io.IOException { + byte[] hbytes = Utf8.toBytes(CFHEADER); + byte[] rbytes = new byte[hbytes.length]; + + ctlFile.seek(0); + int l = ctlFile.read(rbytes); + if (l != hbytes.length + || !java.util.Arrays.equals(hbytes, rbytes)) + { + ctlFile.seek(0); + ctlFile.write(hbytes); + ctlFile.writeBytes(CFPREPRE); + int appLen = 0; + if (appPrefix != null) { + appLen = appPrefix.length(); + ctlFile.writeBytes(appPrefix); + } + ctlFile.writeBytes("\n"); + for (int i = appLen; i < maxPrefix; i++) { + byte space = ' '; + ctlFile.write(space); + } + ctlFile.writeBytes("\n"); + ctlFile.setLength(ctlFile.getFilePointer()); + if (ctlFile.getFilePointer() != controlFileHeaderLength) { + System.err.println("internal error, bad header length: " + + ctlFile.getFilePointer() + + " (should have been: " + + controlFileHeaderLength + + ")"); + } + } + } + + private void extendMapping() throws java.io.IOException { + if (ctlFile == null) return; + long pos = 0; + long len = ctlFile.length(); + if (mapBuf == null || mapBuf.capacity() != len) { + mapBuf = ctlFile.getChannel().map(FileChannel.MapMode.READ_ONLY, pos, len); + } + levelControllerRepo = new MappedLevelControllerRepo(mapBuf, controlFileHeaderLength, numLevels, logControlFilename); + } + + public LevelController getLevelControl(String suffix) { + LevelController ctrl = null; + if (levelControllerRepo != null) { + if (suffix == null || suffix.equals("default")) { + suffix = ""; + } + ctrl = levelControllerRepo.getLevelController(suffix); + if (ctrl != null) { + return ctrl; + } + synchronized(this) { + LevelController inherit = null; + + int lastdot = suffix.lastIndexOf('.'); + if (lastdot != -1) { + // inherit from level above + inherit = getLevelControl(suffix.substring(0, lastdot)); + } else if (suffix.equals("")) { + // the one and only toplevel inherits from other mechanism + inherit = defaultLevelCtrl; + } else { + // everything else inherits from toplevel + inherit = getLevelControl(""); + } + try { + long len = ctlFile.length(); + String append; + if (suffix.equals("")) { + append = "default" + ": "; + } else { + append = "." + suffix + ": "; + } + while ((len + append.length()) % 4 != 0) { + append = append + " "; + } + append = append + inherit.getOnOffString() + "\n"; + ctlFile.seek(ctlFile.length()); + ctlFile.writeBytes(append); + extendMapping(); + ctrl = levelControllerRepo.getLevelController(suffix); + } catch(java.nio.channels.ClosedByInterruptException e) { + // happens during shutdown, ignore + // System.err.println("interrupt, reset logcontrol file: "+e); + resetCtlFile(); + } catch(java.io.IOException e) { + System.err.println("error extending logcontrol file: "+e); + e.printStackTrace(); + resetCtlFile(); + } + } + } + if (ctrl == null) { + return defaultLevelCtrl; + } else { + return ctrl; + } + } + + + private void checkBack() { + if (levelControllerRepo != null) { + Enumeration<String> e = LogManager.getLogManager().getLoggerNames(); + while (e.hasMoreElements()) { + String name = e.nextElement(); + LevelController ctrl = getLevelControl(name); + ctrl.checkBack(); + } + levelControllerRepo.checkBack(); + } + } + + @Override + public LevelController getLevelController(String component) { + return getLevelControl(component); + } + + private class CheckBackRunner extends TimerTask { + public void run() { + checkBack(); + } + } + private CheckBackRunner checkBackRunner; + + @Override + public void close() { + if (checkBackRunner != null) { + checkBackRunner.cancel(); + } + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java new file mode 100644 index 00000000000..26208340132 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/VespaLogHandler.java @@ -0,0 +1,136 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.UnsupportedEncodingException; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.StreamHandler; + +/** + * @author Bjorn Borud + * @author arnej27959 + */ +class VespaLogHandler extends StreamHandler { + + private final LogTarget logTarget; + private final String serviceName; + private final String appPrefix; + private final LevelControllerRepo repo; + private final RejectFilter logRejectFilter; + + /** + * Construct handler which logs to specified logTarget. The logTarget + * may be of the following formats: + * + * <DL> + * <DT> <code>fd:<number></code> + * <DD> Log to specified file descriptor number. Only "fd:2" + * is supported. + * + * <DT> <code>file:<filename></code> + * <DD> Log to specified file in append mode + * </DL> + */ + public VespaLogHandler(LogTarget logTarget, + LevelControllerRepo levelControllerRepo, String serviceName, + String applicationPrefix) { + this.logTarget = logTarget; + this.serviceName = serviceName; + this.appPrefix = applicationPrefix; + this.repo = levelControllerRepo; + this.logRejectFilter = RejectFilter.createDefaultRejectFilter(); + initialize(); + } + + /** + * Publish a log record into the Vespa log target. + */ + public synchronized void publish (LogRecord record) { + Level level = record.getLevel(); + String component = record.getLoggerName(); + + LevelController ctrl = getLevelControl(component); + if (!ctrl.shouldLog(level)) { + return; + } + + if (logRejectFilter.shouldReject(record.getMessage())) { + return; + } + + try { + // provokes rotation of target + setOutputStream(logTarget.open()); + } catch (RuntimeException e) { + LogRecord r = new LogRecord(Level.SEVERE, + "Unable to open file target"); + r.setThrown(e); + emergencyLog(r); + setOutputStream(System.err); + } + super.publish(record); + flush(); + closeFileTarget(); + } + + public LevelController getLevelControl(String component) { + return repo.getLevelController(component); + } + + /** + * Initalize the handler. The main invariant is that + * outputStream is always set to something valid when this method + * returns. + */ + private void initialize () { + try { + setFormatter(new VespaFormatter(serviceName, appPrefix)); + setLevel(LogLevel.ALL); + setEncoding("UTF-8"); + // System.err.println("initialize vespa logging, default level: "+defaultLogLevel); + setOutputStream(logTarget.open()); + } + catch (UnsupportedEncodingException uee) { + LogRecord r = new LogRecord(Level.SEVERE, "Unable to set log encoding to UTF-8"); + r.setThrown(uee); + emergencyLog(r); + } + catch (RuntimeException e) { + LogRecord r = new LogRecord(Level.SEVERE, + "Unable to open file target"); + r.setThrown(e); + emergencyLog(r); + setOutputStream(System.err); + } + } + + + /** Closes the target log file, if there is one */ + public void closeFileTarget() { + try { + logTarget.close(); + } + catch (RuntimeException e) { + LogRecord r = new LogRecord(Level.WARNING, "Unable to close log"); + r.setThrown(e); + emergencyLog(r); + } + } + + /** + * If the logging system experiences problems we can't be expected + * to log it through normal channels, so we have an emergency log + * method which just uses STDERR for formatting the log messages. + * (Which might be right, and might be wrong). + * + * @param record The log record to be logged + */ + private void emergencyLog(LogRecord record) { + record.setLoggerName(VespaLogHandler.class.getName()); + System.err.println(getFormatter().format(record)); + } + + public void cleanup() { + repo.close(); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Collection.java b/vespalog/src/main/java/com/yahoo/log/event/Collection.java new file mode 100644 index 00000000000..da5c1180721 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Collection.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Collection extends Event { + public Collection () { + } + + public Collection (long collectionId, String name) { + setValue("collectionId", Long.toString(collectionId)); + setValue("name", name); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Count.java b/vespalog/src/main/java/com/yahoo/log/event/Count.java new file mode 100644 index 00000000000..f4a61f85de6 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Count.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Count extends Event { + public Count () { + } + + public Count (String name, double value) { + setValue("name", name); + setValue("value", Double.toString(value)); + } + + /** + * Set a property. + * + * @param name The name of the property + * @param value The value of the property + */ + @Override + public Event setValue (String name, String value) { + if (name.equals("value")) { + super.setValue(name, Long.toString((new Double(value)).longValue())); + } else { + super.setValue(name , value); + } + return this; + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java b/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java new file mode 100755 index 00000000000..65d94670680 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/CountGroup.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +public class CountGroup extends Event { + public CountGroup () { + } + + public CountGroup (String name, String values) { + init(name, values); + } + + private void init (String name, String counts) { + setValue("name", name); + setValue("values", counts); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Crash.java b/vespalog/src/main/java/com/yahoo/log/event/Crash.java new file mode 100644 index 00000000000..a86124c1d9e --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Crash.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Crash extends Event { + public Crash () { + } + + public Crash (String name, int pid, int signal) { + setValue("name", name); + setValue("pid", Integer.toString(pid)); + setValue("signal", Integer.toString(signal)); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Event.java b/vespalog/src/main/java/com/yahoo/log/event/Event.java new file mode 100644 index 00000000000..46caea4fc20 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Event.java @@ -0,0 +1,494 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +import com.yahoo.log.LogLevel; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * The Event class is the base class for all VESPA defined events. + * All specific Event classes extend this abstract class. An event + * is more or less a type and a set of properties. For convenience + * we use the logging system to transport Event instances, so the + * typical use is to serialize events into the payload of log + * messages. + * + * <P> + * Note that the static methods started(), stopped() etc are for use + * with this class so using them in the subclasses isn't really + * sanctioned. These methods are what the user wants to use for + * logging events, rather than making events him/herself and stuffing + * them through the logging API. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public abstract class Event implements Serializable { + private static Logger log = Logger.getLogger(Event.class.getName()); + + // states for the event parameters + private static final int INITIAL = 0; + private static final int IN_NAME = 1; + private static final int IN_UNQUOTED = 2; + private static final int IN_QUOTED = 3; + private static final int EQUALS = 4; + + private static Pattern whitespace = Pattern.compile("\\s"); + private static Pattern eventFmt = Pattern.compile("^([^/]+)/(\\d+)(.*)$"); + + // stash the package name for the instantiation + private static String packageName = Event.class.getPackage().getName(); + + private Map<String,String> values = new LinkedHashMap<String,String>(5); + + // default version number is always 1 + private int version = 1; + + private long time = -1; + + protected Event () {} + + /** + * Set a property. + * + * @param name The name of the property + * @param value The value of the property + */ + public Event setValue (String name, String value) { + values.put(name, value); + return this; + } + + /** + * Get a property value. + */ + public String getValue (String name) { + return values.get(name); + } + + /** + * Set the timestamp of this event. + */ + public void setTime (long time) { + this.time = time; + } + + /** + * Get the timestamp of this event + * + * @return returns the timestamp of this event + */ + public long getTime () { + return time; + } + + /** + * Set event version + * + * @param version The version of the event. + */ + public Event setVersion (int version) { + this.version = version; + return this; + } + + /** + * Get the event version. + * + * @return event version + */ + public int getVersion () { + return version; + } + + /** + * Convenience method which returns a property. If the + * property contains whitespace the property will be + * enclosed in quotes. + * + * FIXME: quotes inside the value are not quoted + * + */ + public String getValuePossiblyQuote (String name) { + String tmp = values.get(name); + if (tmp == null) { + return ""; + } + + Matcher m = whitespace.matcher(tmp); + if (m.find()) { + return new StringBuffer(tmp.length() + 2) + .append("\"") + .append(tmp) + .append("\"") + .toString(); + } + + return tmp; + } + + /** + * Get the name of the event instance. + * + * @return the name of the event instance. + */ + public String getName() { + String tmp = this.getClass().getName(); + int last = tmp.lastIndexOf("."); + if (last == -1) { + return tmp.toLowerCase(); + } + + return tmp.substring(last+1).toLowerCase(); + } + + /** + * This method returns the string representation of the + * event and must return something that can be parsed + * by the parse method. + */ + public String toString () { + StringBuilder buff = new StringBuilder(128) + .append(getName()) + .append("/") + .append(version); + + for (String name : values.keySet()) { + buff.append(" ") + .append(name) + .append("=") + .append(getValuePossiblyQuote(name)); + } + + return buff.toString(); + } + + + /** + * Okay, so I am good at making state machine based parsers, + * but every time I do it it feels uncomfortable because it + * isn't the sort of thing one ought to do by freehand :-). + * + * <P> + * Enjoy the graphic + * + * <PRE> + * ___ + * ',_`""\ .---, + * \ :~""``/` | + * `;' //`\ / + * / __ | ('. + * |_ ./O)\ \ `) \ + * _/-. ` `"` |`-. + * .-=: ` / `-. + * /o o \ ,_, . '. + * L._._;_.-' . `'-. + * `'-.` ' `'-. + * `. ' `-._ + * '-._. -' '. + * \ `\ + * + * + * + * </PRE> + */ + private static void parseValuePairs (String s, Event event) { + int state = INITIAL; + int i = 0; + int mark = 0; + String name = null; + + while (i < s.length()) { + switch(s.charAt(i)) { + + case ' ': + if (state == IN_UNQUOTED) { + state = INITIAL; + event.setValue(name, s.substring(mark, i)); + } + + if (state == INITIAL) { + mark = -1; + break; + } + + if (state == IN_QUOTED) { + break; + } + + throw new IllegalStateException("space not allowed at " + i); + + case '=': + if (state == IN_NAME) { + name = s.substring(mark, i); + state = EQUALS; + break; + } + if (state == IN_QUOTED) { + break; + } + + throw new IllegalStateException("'=' not allowed at " + i); + + case '"': + if (state == EQUALS) { + state = IN_QUOTED; + mark = i; + break; + } + + if (state == IN_QUOTED) { + // skip escaped + if (s.charAt(i-1) == '\\') { + break; + } + event.setValue(name, s.substring(mark+1, i)); + state = INITIAL; + break; + } + + throw new IllegalStateException("'\"' not allowed at " + i); + + // ordinary characters + default: + if (state == INITIAL) { + state = IN_NAME; + mark = i; + break; + } + + if (state == EQUALS) { + state = IN_UNQUOTED; + mark = i; + break; + } + } + i++; + } + + // mopping up. when there is no more input to be processed + // we need to take action if we are in one of the below states + switch (state) { + case IN_UNQUOTED: + event.setValue(name, s.substring(mark, i)); + break; + + case IN_QUOTED: + event.setValue(name, s.substring(mark+1, i)); + break; + + case IN_NAME: + throw new IllegalStateException("ended in name"); + + case EQUALS: + event.setValue(name, null); + break; + } + } + + + /** + * Parse string representation of Event and emit correct Event + * subtype. + * + * @param s A string containing an event + * @return Event represented by <code>s</code>. + * @throws MalformedEventException if unable to deciper Event + * from string. + */ + public static Event parse (String s) throws MalformedEventException { + Matcher m1 = eventFmt.matcher(s); + if (! m1.matches()) { + throw new MalformedEventException(s); + } + String eventName = m1.group(1); + String eventVersion = m1.group(2); + String rest = m1.group(3); + + String className = new StringBuffer(eventName.length() + + packageName.length() + + 1) + .append(packageName) + .append(".") + .append(eventName.substring(0,1).toUpperCase()) + .append(eventName.substring(1).toLowerCase()) + .toString(); + + Event event; + try { + event = (Event) Class.forName(className).newInstance(); + } + catch (ClassNotFoundException e) { + event = new Unknown().setName(eventName); + } + catch (Exception e) { + log.log(Level.WARNING, "Event instantiation problem", e); + return null; + } + + event.setVersion(Integer.parseInt(eventVersion)); + + try { + parseValuePairs(rest, event); + } + catch (IllegalStateException | NumberFormatException e) { + throw new MalformedEventException(e); + } + + return event; + } + + /** + * Find the stack frame of the last called before we entered + * the Event class and get the Logger belonging to that class. + * If for some reason we fail to do this, just return the + * default Logger. + * + * <P> + * Beware, if you come here for cleverness and enlightenment + * then know that this technique might not be exceptionally fast + * so don't abuse this mechanism blindly. (Do what I do: abuse + * it with both eyes open :-). + * + */ + private static final Logger getCallerLogger() { + StackTraceElement stack[] = (new Throwable()).getStackTrace(); + int i = 0; + while (i < stack.length) { + StackTraceElement frame = stack[i]; + String cname = frame.getClassName(); + if (cname.equals("com.yahoo.log.event.Event")) { + break; + } + i++; + } + + while (i < stack.length) { + StackTraceElement frame = stack[i]; + String cname = frame.getClassName(); + if (!cname.equals("com.yahoo.log.event.Event")) { + return Logger.getLogger(cname); + } + i++; + } + + return Logger.getLogger(""); + } + + /** + * Internal method which prepares Event log messages. Not + * the prettiest way to do it... + */ + private static final void log(Logger logger, Object param) { + LogRecord r = new LogRecord(LogLevel.EVENT, null); + r.setParameters(new Object[] {param}); + r.setLoggerName(logger.getName()); + logger.log(r); + } + + /** + * Static method for logging the <b>starting</b> event. + */ + public static final void starting (String name) { + log(getCallerLogger(), new Starting(name)); + } + + /** + * Static method for logging the <b>started</b> event. + */ + public static final void started (String name) { + log(getCallerLogger(), new Started(name)); + } + + /** + * Static method for logging the <b>stopping</b> event. + */ + public static final void stopping (String name, String why) { + log(getCallerLogger(), new Stopping(name, why)); + } + + /** + * Static method for logging the <b>stopped</b> event. + */ + public static final void stopped (String name, int pid, int exitcode) { + log(getCallerLogger(), new Stopped(name, pid, exitcode)); + } + + /** + * Static method for logging the <b>reloading</b> event. + */ + public static final void reloading (String name) { + log(getCallerLogger(), new Reloading(name)); + } + + /** + * Static method for logging the <b>reloaded</b> event. + */ + public static final void reloaded (String name) { + log(getCallerLogger(), new Reloaded(name)); + } + + /** + * Static method for logging the <b>count</b> event. + */ + public static final void count (String name, long value) { + log(getCallerLogger(), new Count(name, value)); + } + + /** + * Static method for logging the <b>value</b> event. + */ + public static final void value (String name, double value) { + log(getCallerLogger(), new Value(name, value)); + } + + /** + * Static method for logging the <b>histogram</b> event. + */ + public static final void histogram (String name, String value, + String representation) { + log(getCallerLogger(), new Histogram(name, value, + representation)); + } + + /** + * Static method for logging a set of <b>value</b> events. + */ + public static final void valueGroup (String name, String value) { + log(getCallerLogger(), new ValueGroup(name, value)); + } + + /** + * Static method for logging a set of <b>count</b> events. + */ + public static final void countGroup (String name, String value) { + log(getCallerLogger(), new CountGroup(name, value)); + } + + /** + * Static method for logging the <b>progress</b> event. + */ + public static final void progress (String name, long value, long total) { + log(getCallerLogger(), new Progress(name, value, total)); + } + + /** + * Static method for logging the <b>state</b> event. + */ + public static final void state (String name, String value) { + log(getCallerLogger(), new State(name, value)); + } + + /** + * Static method for logging the <b>crash</b> event. + */ + public static final void crash (String name, int pid, int signal) { + log(getCallerLogger(), new Crash(name, pid, signal)); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Histogram.java b/vespalog/src/main/java/com/yahoo/log/event/Histogram.java new file mode 100755 index 00000000000..219f801deb1 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Histogram.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +public class Histogram extends Event { + public Histogram () { + } + + public Histogram (String name, String value, String representation) { + init(name, value, representation); + } + + private void init (String name, String value, String representation) { + setValue("name", name); + setValue("counts", value); + setValue("representation", representation); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java b/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java new file mode 100644 index 00000000000..3b67a19c916 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/MalformedEventException.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +public class MalformedEventException extends Exception { + public MalformedEventException (Throwable cause) { + super(cause); + } + + public MalformedEventException (String msg) { + super(msg); + } + + public MalformedEventException () { + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Progress.java b/vespalog/src/main/java/com/yahoo/log/event/Progress.java new file mode 100644 index 00000000000..70df650764c --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Progress.java @@ -0,0 +1,33 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Progress extends Event { + public Progress () { + } + + public Progress (String name, String value, String total) { + init(name, value, total); + } + + public Progress (String name, double value, double total) { + init(name, Double.toString(value), Double.toString(total)); + } + + public Progress (String name, String value) { + init(name, value, ""); + } + + public Progress (String name, double value) { + init(name, Double.toString(value), ""); + } + + private void init (String name, String value, String total) { + setValue("name", name); + setValue("value", value); + setValue("total", total); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java new file mode 100644 index 00000000000..0919b847065 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Reloaded.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Reloaded extends Event { + public Reloaded () { + } + + public Reloaded (String name) { + setValue("name", name); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Reloading.java b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java new file mode 100644 index 00000000000..001eb00522c --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Reloading.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Reloading extends Event { + public Reloading () { + } + + public Reloading (String name) { + setValue("name", name); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Started.java b/vespalog/src/main/java/com/yahoo/log/event/Started.java new file mode 100644 index 00000000000..a955dc0b2b2 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Started.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Started extends Event { + public Started () { + } + + public Started (String name) { + setValue("name", name); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Starting.java b/vespalog/src/main/java/com/yahoo/log/event/Starting.java new file mode 100644 index 00000000000..beb0740323f --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Starting.java @@ -0,0 +1,15 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Starting extends Event { + public Starting () { + } + + public Starting (String name) { + setValue("name", name); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/State.java b/vespalog/src/main/java/com/yahoo/log/event/State.java new file mode 100755 index 00000000000..869178cb435 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/State.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +public class State extends Event { + public State () { + } + + public State (String name, String value) { + init(name, value); + } + + private void init (String name, String value) { + setValue("name", name); + setValue("value", value); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopped.java b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java new file mode 100644 index 00000000000..e61239b4b67 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopped.java @@ -0,0 +1,17 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Stopped extends Event { + public Stopped () { + } + + public Stopped (String name, int pid, int exitcode) { + setValue("name", name); + setValue("pid", Integer.toString(pid)); + setValue("exitcode", Integer.toString(exitcode)); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Stopping.java b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java new file mode 100644 index 00000000000..49ad9d94db2 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Stopping.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Stopping extends Event { + public Stopping () { + } + + public Stopping (String name, String why) { + setValue("name", name); + setValue("why", why); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Unknown.java b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java new file mode 100644 index 00000000000..864aa2adf0f --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Unknown.java @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud </a> + */ +public class Unknown extends Event { + public Unknown() { + } + + private String name; + + public Unknown setName(String name) { + this.name = name; + return this; + } + + @Override + public String getName() { + return this.name.toLowerCase(); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/Value.java b/vespalog/src/main/java/com/yahoo/log/event/Value.java new file mode 100644 index 00000000000..ac6b80aad40 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/Value.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +/** + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class Value extends Event { + public Value () { + } + + public Value (String name, double value) { + setValue("name", name); + setValue("value", Double.toString(value)); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java b/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java new file mode 100755 index 00000000000..d201d643159 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/ValueGroup.java @@ -0,0 +1,16 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +public class ValueGroup extends Event { + public ValueGroup () { + } + + public ValueGroup (String name, String values) { + init(name, values); + } + + private void init (String name, String value) { + setValue("name", name); + setValue("values", value); + } +} diff --git a/vespalog/src/main/java/com/yahoo/log/event/package-info.java b/vespalog/src/main/java/com/yahoo/log/event/package-info.java new file mode 100644 index 00000000000..8e69bd447fc --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/event/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.log.event; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespalog/src/main/java/com/yahoo/log/package-info.java b/vespalog/src/main/java/com/yahoo/log/package-info.java new file mode 100644 index 00000000000..ca784ea8283 --- /dev/null +++ b/vespalog/src/main/java/com/yahoo/log/package-info.java @@ -0,0 +1,7 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +@PublicApi +package com.yahoo.log; + +import com.yahoo.api.annotations.PublicApi; +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/vespalog/src/test/.gitignore b/vespalog/src/test/.gitignore new file mode 100644 index 00000000000..dde71217083 --- /dev/null +++ b/vespalog/src/test/.gitignore @@ -0,0 +1,14 @@ +.depend +Makefile +bufferedlogskiptest +bufferedlogskiptest.log +bufferedlogtest +bufferedlogtest.log +empty.* +logtest +testthreads +vespa.log +/rejectfiltertest +vespalog_bufferedlogskiptest_app +vespalog_bufferedlogtest_app +vespalog_rejectfiltertest_app diff --git a/vespalog/src/test/CMakeLists.txt b/vespalog/src/test/CMakeLists.txt new file mode 100644 index 00000000000..f9b2f360fa8 --- /dev/null +++ b/vespalog/src/test/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalog_bufferedlogtest_app + SOURCES + bufferedlogtest.cpp + bufferedlogtest.logger1.cpp + bufferedlogtest.logger2.cpp + DEPENDS + vespalog +) +vespa_add_test(NAME vespalog_bufferedlogtest_app NO_VALGRIND COMMAND sh bufferedlogtest_test.sh) +vespa_add_executable(vespalog_bufferedlogskiptest_app + SOURCES + bufferedlogskiptest.cpp + DEPENDS + vespalog +) +vespa_add_test(NAME vespalog_bufferedlogskiptest_app NO_VALGRIND COMMAND sh bufferedlogskiptest_test.sh) +vespa_add_executable(vespalog_rejectfiltertest_app + SOURCES + rejectfiltertest.cpp + DEPENDS + vespalog +) +vespa_add_test(NAME vespalog_rejectfiltertest_app NO_VALGRIND COMMAND vespalog_rejectfiltertest_app) diff --git a/vespalog/src/test/bufferedlogskiptest.cpp b/vespalog/src/test/bufferedlogskiptest.cpp new file mode 100644 index 00000000000..5b386d18e90 --- /dev/null +++ b/vespalog/src/test/bufferedlogskiptest.cpp @@ -0,0 +1,90 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> + +#include <fstream> +#include <iostream> +#include <sstream> + +#include <sys/types.h> +#include <stdlib.h> +#include <unistd.h> +#include <signal.h> + +LOG_SETUP("bufferedlogskiptest", + "$Id$"); + +std::string readFile(const std::string& file) { + std::ostringstream ost; + std::ifstream is(file.c_str()); + std::string line; + while (std::getline(is, line)) { + std::string::size_type pos = line.find('\t'); + if (pos == std::string::npos) continue; + std::string::size_type pos2 = line.find('\t', pos + 1); + if (pos2 == std::string::npos) continue; + std::string result = line.substr(0, pos) + + "\tlocalhost" + + line.substr(pos2); + // Ignore debug entries. (Log adds some itself with timestamp we + // can't control) + if (result.find("\tdebug\t") == std::string::npos) { + ost << result << "\n"; + } + } + return ost.str(); +} + +void testSkipBufferOnDebug(const std::string& file, uint64_t & timer) +{ + std::cerr << "testSkipBufferOnDebug ...\n"; + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + + timer = 200 * 1000000; + for (uint32_t i=0; i<10; ++i) { + LOGBP(info, "Message"); + timer += 1; + LOGBM(info, "Message"); + timer += 1; + LOGBT(info, "Message", "Message"); + timer += 1; + } + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogskiptest.skipped.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testSkipBufferOnDebug\n"; + system(("diff -u " + file + + " bufferedlogskiptest.skipped.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); + +} + +void reset(uint64_t & timer) { + timer = 0; + ns_log::BufferedLogger::logger.setMaxCacheSize(10); + ns_log::BufferedLogger::logger.setMaxEntryAge(300); + ns_log::BufferedLogger::logger.setCountFactor(5); +} + +int +main(int argc, char **argv) +{ + if (argc != 2) { + std::cerr << "bufferedlogskiptest must be called with one argument\n"; + exit(EXIT_FAILURE); + } + ns_log::Logger::fakePid = true; + uint64_t timer; + logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + ns_log::BufferedLogger::logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + + reset(timer); + testSkipBufferOnDebug(argv[1], timer); + + exit(EXIT_SUCCESS); +} diff --git a/vespalog/src/test/bufferedlogskiptest.skipped.log b/vespalog/src/test/bufferedlogskiptest.skipped.log new file mode 100644 index 00000000000..c0c2cc36309 --- /dev/null +++ b/vespalog/src/test/bufferedlogskiptest.skipped.log @@ -0,0 +1,32 @@ +1220016833.684527 example.yahoo.com 15527 - bufferedlogskiptest.log debug log.cpp:111 (1.77): prefix was set to 'bufferedlogskiptest' +0.000000 example.yahoo.com -1/-1 - bufferedlogskiptest info Starting up, using logfile bufferedlogskiptest.log +200.000000 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000001 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000002 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000003 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000004 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000005 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000006 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000007 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000008 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000009 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000010 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000011 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000012 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000013 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000014 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000015 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000016 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000017 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000018 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000019 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000020 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000021 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000022 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000023 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000024 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000025 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000026 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000027 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000028 example.yahoo.com -1/-1 - bufferedlogskiptest info Message +200.000029 example.yahoo.com -1/-1 - bufferedlogskiptest info Message diff --git a/vespalog/src/test/bufferedlogskiptest_test.sh b/vespalog/src/test/bufferedlogskiptest_test.sh new file mode 100755 index 00000000000..50887394bf6 --- /dev/null +++ b/vespalog/src/test/bufferedlogskiptest_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +rm -f bufferedlogskiptest.log +VESPA_LOG_TARGET="file:bufferedlogskiptest.log" VESPA_LOG_LEVEL="all -spam" ./vespalog_bufferedlogskiptest_app bufferedlogskiptest.log diff --git a/vespalog/src/test/bufferedlogtest.cpp b/vespalog/src/test/bufferedlogtest.cpp new file mode 100644 index 00000000000..0e27bd65988 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.cpp @@ -0,0 +1,408 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> + +#include "bufferedlogtest.logger1.h" +#include "bufferedlogtest.logger2.h" + +#include <fstream> +#include <iostream> +#include <sstream> + +#include <sys/types.h> +#include <stdlib.h> +#include <unistd.h> +#include <signal.h> + +LOG_SETUP("bufferedlogtest", + "$Id$"); + +std::string readFile(const std::string& file) { + std::ostringstream ost; + std::ifstream is(file.c_str()); + std::string line; + while (std::getline(is, line)) { + std::string::size_type pos = line.find('\t'); + if (pos == std::string::npos) continue; + std::string::size_type pos2 = line.find('\t', pos + 1); + if (pos2 == std::string::npos) continue; + std::string result = line.substr(0, pos) + + "\tlocalhost" + + line.substr(pos2); + ost << result << "\n"; + } + return ost.str(); +} + +void spamLog1(uint64_t& time, int diff) { + for (int i=0; i<100; ++i) { + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: NOT CONNECTED"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: NOT_READY"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: BAAAH"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: RPC FAILURE"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: COSMIC RADIATION"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: ITS SATURDAY"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: Yeah, Right!!"); + time += diff; + LOGBT(error, "Failed to send to node 4", + "Failed to send message to node 4: NOT CONNECTED"); + time += diff; + LOGBT(spam, "Failed to send to node 4", + "Failed to send message to node 4: NOT CONNECTED"); + time += diff; + LOGBT(warning, "Failed to send to node 4", + "Failed to send message to node 4: NOT CONNECTED"); + time += diff * 10; + LOGBT(warning, "Failed to send to node 3", + "Failed to send message to node 3: NOT CONNECTED"); + } +} + +void spamLog2(uint64_t& time, int diff) { + for (int i=0; i<100; ++i) { + time += diff; + std::ostringstream ost; + ost << "Message " << i; + LOGBT(info, ost.str(), ost.str().c_str()); + } +} + +void testThatEntriesWithHighCountIsKept(const std::string& file, + uint64_t& timer) +{ + std::cerr << "testThatEntriesWithHighCountIsKept ...\n"; + timer = 10 * 1000000 + 4; + + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + timer = 100 * 1000000 + 4; + LOGBT(warning, "Token", "message"); + + spamLog1(timer, 1); + spamLog2(timer, 1); + spamLog1(timer, 1); + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.highcountkept.log")); + + if (result != expected) { + std::cerr << "Failed testThatEntriesWithHighCountIsKept\n"; + system(("diff -u " + file + " bufferedlogtest.highcountkept.log") + .c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +void testThatEntriesWithHighCountsAreEventuallyRemoved( + const std::string& file, uint64_t& timer) +{ + std::cerr << "testThatEntriesWithHighCountsAreEventuallyRemoved ...\n"; + // Same as above, just increase timer more between each log entry. + // Should eventually throw out the entries with high count + timer = 10 * 1000000 + 4; + // Make sure we don't remove due to age. + ns_log::BufferedLogger::logger.setMaxEntryAge(1000000); + // Let each count, count for 5 seconds. + ns_log::BufferedLogger::logger.setCountFactor(5); + + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + timer = 100 * 1000000 + 4; + LOGBT(warning, "Token", "message"); + + spamLog1(timer, 1); + spamLog2(timer, 10 * 1000000); + spamLog1(timer, 1); + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.highcountexpire.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testThatEntriesWithHighCountsAreEventuallyRemoved\n"; + system(("diff -u " + file + " bufferedlogtest.highcountexpire.log") + .c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +void testThatEntriesExpire( + const std::string& file, uint64_t& timer) +{ + std::cerr << "testThatEntriesExpire ...\n"; + // Test that we don't keep entries longer than max age + timer = 10 * 1000000 + 4; + // Time out after 120 seconds + ns_log::BufferedLogger::logger.setMaxEntryAge(120); + // Let counts count much, so they expire due to time instead + ns_log::BufferedLogger::logger.setCountFactor(100000); + + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + timer = 100 * 1000000 + 4; + LOGBT(warning, "Token", "message"); + + spamLog1(timer, 1); + spamLog2(timer, 10 * 1000000); + spamLog1(timer, 1); + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.expire.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testThatEntriesExpire\n"; + system(("diff -u " + file + " bufferedlogtest.expire.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +// Spam 10+ different entries lots of times, to fill cache with high count +// entries +void spamLog3(uint64_t& time, int diff) { + for (int i=0; i<100; ++i) { + time += diff; + LOGBT(warning, "Failed to talk to node 1", + "Failed to send message to node 0x1: NOT CONNECTED"); + time += diff; + LOGBT(warning, "Failed to talk to node 2", + "Failed to send message to node 0x2: NOT_READY"); + time += diff; + LOGBT(warning, "Failed to talk to node 3", + "Failed to send message to node 0x3: BAAAH"); + time += diff; + LOGBT(warning, "Failed to talk to node 4", + "Failed to send message to node 0x4: RPC FAILURE"); + time += diff; + LOGBT(warning, "Failed to talk to node 5", + "Failed to send message to node 0x5: COSMIC RADIATION"); + time += diff; + LOGBT(warning, "Failed to talk to node 6", + "Failed to send message to node 0x6: ITS SATURDAY"); + time += diff; + LOGBT(warning, "Failed to talk to node 7", + "Failed to send message to node 0x7: Yeah, Right!!"); + time += diff; + LOGBT(error, "Failed to talk to node 8", + "Failed to send message to node 0x8: NOT CONNECTED"); + time += diff; + LOGBT(info, "Failed to talk to node 9", + "Failed to send message to node 0x9: NOT CONNECTED"); + time += diff; + LOGBT(warning, "Failed to talk to node 10", + "Failed to send message to node 0xa: NOT CONNECTED"); + } +} + +void testThatHighCountEntriesDontStarveOthers( + const std::string& file, uint64_t& timer) +{ + std::cerr << "testThatHighCountEntriesDontStarveOthers ...\n"; + timer = 10 * 1000000 + 4; + // Long time out, we don't want to rely on timeout to prevent starvation + ns_log::BufferedLogger::logger.setMaxEntryAge(12000000); + // Let counts count much, so they score high + ns_log::BufferedLogger::logger.setCountFactor(100000); + + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + timer = 100 * 1000000; + LOGBT(warning, "Token", "message"); + + spamLog3(timer, 1); + spamLog1(timer, 1); + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.nostarve.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testThatHighCountEntriesDontStarveOthers\n"; + system(("diff -u " + file + " bufferedlogtest.nostarve.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +void testNoTokenMatchAcrossComponents(const std::string& file, + uint64_t& timer) +{ + std::cerr << "testNoTokenMatchAcrossComponents ...\n"; + + LOGBP(info, "Starting up, using logfile %s", file.c_str()); + + timer = 200 * 1000000; + for (uint32_t i=0; i<100; ++i) { + std::ostringstream ost; + ost << "Message " << i; + logWithLogger1("Message", ost.str()); + timer += 1; + logWithLogger2("Message", ost.str()); + timer += 1; + } + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.tokenacrossloggers.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testNoTokenMatchAcrossComponents\n"; + system(("diff -u " + file + + " bufferedlogtest.tokenacrossloggers.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +void testLogLocationAsToken(const std::string& file, uint64_t& timer) +{ + std::cerr << "testLogLocationAsToken ...\n"; + LOGBP(info, "Starting up, using logfile %s", file.c_str()); + + timer = 200 * 1000000; + for (uint32_t i=0; i<100; ++i) { + LOGBP(info, "Message %i", i); + timer += 1; + LOGBP(info, "Message %i", i); + timer += 1; + } + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.locationastoken.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testLogLocationAsToken\n"; + system(("diff -u " + file + + " bufferedlogtest.locationastoken.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +void testLogMessageAsToken(const std::string& file, uint64_t& timer) +{ + std::cerr << "testLogMessageAsToken ...\n"; + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + + timer = 200 * 1000000; + for (uint32_t i=0; i<100; ++i) { + LOGBM(info, "Message %i", i % 10); + timer += 1; + LOGBM(info, "Message %i", i % 10); + timer += 1; + } + + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.messageastoken.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testLogMessageAsToken\n"; + system(("diff -u " + file + + " bufferedlogtest.messageastoken.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); +} + +void testNonBufferedLoggerTriggersBufferedLogTrim(const std::string& file, + uint64_t& timer) +{ + std::cerr << "testNonBufferedLoggerTriggersBufferedLogTrim ...\n"; + // Write a lot of buffered log. + LOGBM(info, "Starting up, using logfile %s", file.c_str()); + + timer = 200 * 1000000; + for (uint32_t i=0; i<100; ++i) { + LOGBP(info, "Message %i", i); + timer += 1; + } + + // Advance time to time where we should have flushed long ago. + timer = 100000 * 1000000ull; + LOG(info, "This should cause buffered log flush"); + + // So flushing should have happened before this time. + timer = 200000 * 1000000ull; + LOGB_FLUSH(); + + std::string result(readFile(file)); + std::string expected(readFile("bufferedlogtest.trimcache.log")); + + if (result != expected) { + std::cerr << "Failed " + << "testNonBufferedLoggerTriggersBufferedLogTrim\n"; + system(("diff -u " + file + + " bufferedlogtest.trimcache.log").c_str()); + exit(EXIT_FAILURE); + } + unlink(file.c_str()); + +} + +void reset(uint64_t& timer) { + timer = 0; + ns_log::BufferedLogger::logger.setMaxEntryAge(300); + ns_log::BufferedLogger::logger.setCountFactor(5); +} + +int +main(int argc, char **argv) +{ + if (argc != 2) { + std::cerr << "bufferedlogtest must be called with one argument\n"; + exit(EXIT_FAILURE); + } + ns_log::Logger::fakePid = true; + ns_log::BufferedLogger::logger.setMaxCacheSize(10); + uint64_t timer; + logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + ns_log::BufferedLogger::logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + + reset(timer); + testThatEntriesWithHighCountIsKept(argv[1], timer); + reset(timer); + testThatEntriesWithHighCountsAreEventuallyRemoved(argv[1], timer); + reset(timer); + testThatEntriesExpire(argv[1], timer); + reset(timer); + testThatHighCountEntriesDontStarveOthers(argv[1], timer); + reset(timer); + testNoTokenMatchAcrossComponents(argv[1], timer); + reset(timer); + testLogLocationAsToken(argv[1], timer); + reset(timer); + testLogMessageAsToken(argv[1], timer); + reset(timer); + testNonBufferedLoggerTriggersBufferedLogTrim(argv[1], timer); + + exit(EXIT_SUCCESS); +} diff --git a/vespalog/src/test/bufferedlogtest.expire.log b/vespalog/src/test/bufferedlogtest.expire.log new file mode 100644 index 00000000000..b68b1c3422e --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.expire.log @@ -0,0 +1,110 @@ +10.000004 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +100.000004 example.yahoo.com -1/-1 - bufferedlogtest warning message +100.000005 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED +100.000024 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED +110.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 +120.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 1 +130.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 2 +140.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 3 +150.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 4 +160.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 5 +170.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 6 +180.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 7 +190.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 8 +200.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 9 +210.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 10 +220.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 11 +220.002004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED (Repeated 899 times since 100.000005) +220.002004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED (Repeated 99 times since 100.000024) +230.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 12 +240.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 13 +250.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 14 +260.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 15 +270.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 16 +280.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 17 +290.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 18 +300.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 19 +310.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 20 +320.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 21 +330.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 22 +340.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 23 +350.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 24 +360.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 25 +370.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 26 +380.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 27 +390.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 28 +400.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 29 +410.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 30 +420.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 31 +430.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 32 +440.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 33 +450.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 34 +460.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 35 +470.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 36 +480.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 37 +490.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 38 +500.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 39 +510.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 40 +520.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 41 +530.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 42 +540.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 43 +550.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 44 +560.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 45 +570.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 46 +580.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 47 +590.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 48 +600.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 49 +610.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 50 +620.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 51 +630.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 52 +640.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 53 +650.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 54 +660.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 55 +670.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 56 +680.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 57 +690.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 58 +700.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 59 +710.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 60 +720.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 61 +730.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 62 +740.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 63 +750.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 64 +760.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 65 +770.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 66 +780.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 67 +790.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 68 +800.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 69 +810.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 70 +820.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 71 +830.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 72 +840.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 73 +850.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 74 +860.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 75 +870.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 76 +880.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 77 +890.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 78 +900.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 79 +910.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 80 +920.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 81 +930.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 82 +940.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 83 +950.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 84 +960.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 85 +970.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 86 +980.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 87 +990.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 88 +1000.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 89 +1010.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 90 +1020.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 91 +1030.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 92 +1040.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 93 +1050.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 94 +1060.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 95 +1070.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 96 +1080.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 97 +1090.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 98 +1100.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 99 +1100.002005 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED +1100.002024 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED +1100.004004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED (Repeated 899 times since 1100.002005) +1100.004004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED (Repeated 99 times since 1100.002024) diff --git a/vespalog/src/test/bufferedlogtest.highcountexpire.log b/vespalog/src/test/bufferedlogtest.highcountexpire.log new file mode 100644 index 00000000000..fb07d45d924 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.highcountexpire.log @@ -0,0 +1,108 @@ +10.000004 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +100.000004 example.yahoo.com -1/-1 - bufferedlogtest warning message +100.000005 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED +100.000024 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED +110.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 +120.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 1 +130.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 2 +140.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 3 +150.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 4 +160.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 5 +170.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 6 +180.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 7 +190.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 8 +200.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 9 +210.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 10 +220.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 11 +230.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 12 +240.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 13 +250.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 14 +260.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 15 +270.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 16 +280.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 17 +290.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 18 +300.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 19 +310.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 20 +320.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 21 +330.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 22 +340.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 23 +350.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 24 +360.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 25 +370.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 26 +380.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 27 +390.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 28 +400.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 29 +410.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 30 +420.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 31 +430.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 32 +440.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 33 +450.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 34 +460.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 35 +470.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 36 +480.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 37 +490.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 38 +500.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 39 +510.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 40 +520.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 41 +530.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 42 +540.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 43 +550.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 44 +560.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 45 +570.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 46 +580.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 47 +590.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 48 +600.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 49 +610.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 50 +620.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 51 +630.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 52 +640.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 53 +650.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 54 +660.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 55 +670.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 56 +680.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 57 +680.002004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED (Repeated 99 times since 100.000024) +690.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 58 +700.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 59 +710.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 60 +720.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 61 +730.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 62 +740.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 63 +750.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 64 +760.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 65 +770.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 66 +780.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 67 +790.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 68 +800.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 69 +810.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 70 +820.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 71 +830.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 72 +840.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 73 +850.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 74 +860.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 75 +870.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 76 +880.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 77 +890.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 78 +900.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 79 +910.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 80 +920.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 81 +930.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 82 +940.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 83 +950.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 84 +960.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 85 +970.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 86 +980.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 87 +990.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 88 +1000.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 89 +1010.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 90 +1020.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 91 +1030.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 92 +1040.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 93 +1050.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 94 +1060.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 95 +1070.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 96 +1080.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 97 +1090.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 98 +1100.002004 example.yahoo.com -1/-1 - bufferedlogtest info Message 99 +1100.002024 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED +1100.004004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED (Repeated 1799 times since 100.000005) +1100.004004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED (Repeated 99 times since 1100.002024) diff --git a/vespalog/src/test/bufferedlogtest.highcountkept.log b/vespalog/src/test/bufferedlogtest.highcountkept.log new file mode 100644 index 00000000000..68679497033 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.highcountkept.log @@ -0,0 +1,106 @@ +10.000004 localhost -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +100.000004 localhost -1/-1 - bufferedlogtest warning message +100.000005 localhost -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED +100.000024 localhost -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED +100.002005 localhost -1/-1 - bufferedlogtest info Message 0 +100.002006 localhost -1/-1 - bufferedlogtest info Message 1 +100.002007 localhost -1/-1 - bufferedlogtest info Message 2 +100.002008 localhost -1/-1 - bufferedlogtest info Message 3 +100.002009 localhost -1/-1 - bufferedlogtest info Message 4 +100.002010 localhost -1/-1 - bufferedlogtest info Message 5 +100.002011 localhost -1/-1 - bufferedlogtest info Message 6 +100.002012 localhost -1/-1 - bufferedlogtest info Message 7 +100.002013 localhost -1/-1 - bufferedlogtest info Message 8 +100.002014 localhost -1/-1 - bufferedlogtest info Message 9 +100.002015 localhost -1/-1 - bufferedlogtest info Message 10 +100.002016 localhost -1/-1 - bufferedlogtest info Message 11 +100.002017 localhost -1/-1 - bufferedlogtest info Message 12 +100.002018 localhost -1/-1 - bufferedlogtest info Message 13 +100.002019 localhost -1/-1 - bufferedlogtest info Message 14 +100.002020 localhost -1/-1 - bufferedlogtest info Message 15 +100.002021 localhost -1/-1 - bufferedlogtest info Message 16 +100.002022 localhost -1/-1 - bufferedlogtest info Message 17 +100.002023 localhost -1/-1 - bufferedlogtest info Message 18 +100.002024 localhost -1/-1 - bufferedlogtest info Message 19 +100.002025 localhost -1/-1 - bufferedlogtest info Message 20 +100.002026 localhost -1/-1 - bufferedlogtest info Message 21 +100.002027 localhost -1/-1 - bufferedlogtest info Message 22 +100.002028 localhost -1/-1 - bufferedlogtest info Message 23 +100.002029 localhost -1/-1 - bufferedlogtest info Message 24 +100.002030 localhost -1/-1 - bufferedlogtest info Message 25 +100.002031 localhost -1/-1 - bufferedlogtest info Message 26 +100.002032 localhost -1/-1 - bufferedlogtest info Message 27 +100.002033 localhost -1/-1 - bufferedlogtest info Message 28 +100.002034 localhost -1/-1 - bufferedlogtest info Message 29 +100.002035 localhost -1/-1 - bufferedlogtest info Message 30 +100.002036 localhost -1/-1 - bufferedlogtest info Message 31 +100.002037 localhost -1/-1 - bufferedlogtest info Message 32 +100.002038 localhost -1/-1 - bufferedlogtest info Message 33 +100.002039 localhost -1/-1 - bufferedlogtest info Message 34 +100.002040 localhost -1/-1 - bufferedlogtest info Message 35 +100.002041 localhost -1/-1 - bufferedlogtest info Message 36 +100.002042 localhost -1/-1 - bufferedlogtest info Message 37 +100.002043 localhost -1/-1 - bufferedlogtest info Message 38 +100.002044 localhost -1/-1 - bufferedlogtest info Message 39 +100.002045 localhost -1/-1 - bufferedlogtest info Message 40 +100.002046 localhost -1/-1 - bufferedlogtest info Message 41 +100.002047 localhost -1/-1 - bufferedlogtest info Message 42 +100.002048 localhost -1/-1 - bufferedlogtest info Message 43 +100.002049 localhost -1/-1 - bufferedlogtest info Message 44 +100.002050 localhost -1/-1 - bufferedlogtest info Message 45 +100.002051 localhost -1/-1 - bufferedlogtest info Message 46 +100.002052 localhost -1/-1 - bufferedlogtest info Message 47 +100.002053 localhost -1/-1 - bufferedlogtest info Message 48 +100.002054 localhost -1/-1 - bufferedlogtest info Message 49 +100.002055 localhost -1/-1 - bufferedlogtest info Message 50 +100.002056 localhost -1/-1 - bufferedlogtest info Message 51 +100.002057 localhost -1/-1 - bufferedlogtest info Message 52 +100.002058 localhost -1/-1 - bufferedlogtest info Message 53 +100.002059 localhost -1/-1 - bufferedlogtest info Message 54 +100.002060 localhost -1/-1 - bufferedlogtest info Message 55 +100.002061 localhost -1/-1 - bufferedlogtest info Message 56 +100.002062 localhost -1/-1 - bufferedlogtest info Message 57 +100.002063 localhost -1/-1 - bufferedlogtest info Message 58 +100.002064 localhost -1/-1 - bufferedlogtest info Message 59 +100.002065 localhost -1/-1 - bufferedlogtest info Message 60 +100.002066 localhost -1/-1 - bufferedlogtest info Message 61 +100.002067 localhost -1/-1 - bufferedlogtest info Message 62 +100.002068 localhost -1/-1 - bufferedlogtest info Message 63 +100.002069 localhost -1/-1 - bufferedlogtest info Message 64 +100.002070 localhost -1/-1 - bufferedlogtest info Message 65 +100.002071 localhost -1/-1 - bufferedlogtest info Message 66 +100.002072 localhost -1/-1 - bufferedlogtest info Message 67 +100.002073 localhost -1/-1 - bufferedlogtest info Message 68 +100.002074 localhost -1/-1 - bufferedlogtest info Message 69 +100.002075 localhost -1/-1 - bufferedlogtest info Message 70 +100.002076 localhost -1/-1 - bufferedlogtest info Message 71 +100.002077 localhost -1/-1 - bufferedlogtest info Message 72 +100.002078 localhost -1/-1 - bufferedlogtest info Message 73 +100.002079 localhost -1/-1 - bufferedlogtest info Message 74 +100.002080 localhost -1/-1 - bufferedlogtest info Message 75 +100.002081 localhost -1/-1 - bufferedlogtest info Message 76 +100.002082 localhost -1/-1 - bufferedlogtest info Message 77 +100.002083 localhost -1/-1 - bufferedlogtest info Message 78 +100.002084 localhost -1/-1 - bufferedlogtest info Message 79 +100.002085 localhost -1/-1 - bufferedlogtest info Message 80 +100.002086 localhost -1/-1 - bufferedlogtest info Message 81 +100.002087 localhost -1/-1 - bufferedlogtest info Message 82 +100.002088 localhost -1/-1 - bufferedlogtest info Message 83 +100.002089 localhost -1/-1 - bufferedlogtest info Message 84 +100.002090 localhost -1/-1 - bufferedlogtest info Message 85 +100.002091 localhost -1/-1 - bufferedlogtest info Message 86 +100.002092 localhost -1/-1 - bufferedlogtest info Message 87 +100.002093 localhost -1/-1 - bufferedlogtest info Message 88 +100.002094 localhost -1/-1 - bufferedlogtest info Message 89 +100.002095 localhost -1/-1 - bufferedlogtest info Message 90 +100.002096 localhost -1/-1 - bufferedlogtest info Message 91 +100.002097 localhost -1/-1 - bufferedlogtest info Message 92 +100.002098 localhost -1/-1 - bufferedlogtest info Message 93 +100.002099 localhost -1/-1 - bufferedlogtest info Message 94 +100.002100 localhost -1/-1 - bufferedlogtest info Message 95 +100.002101 localhost -1/-1 - bufferedlogtest info Message 96 +100.002102 localhost -1/-1 - bufferedlogtest info Message 97 +100.002103 localhost -1/-1 - bufferedlogtest info Message 98 +100.002104 localhost -1/-1 - bufferedlogtest info Message 99 +100.004104 localhost -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED (Repeated 1799 times since 100.000005) +100.004104 localhost -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED (Repeated 199 times since 100.000024) diff --git a/vespalog/src/test/bufferedlogtest.locationastoken.log b/vespalog/src/test/bufferedlogtest.locationastoken.log new file mode 100644 index 00000000000..c171ca2800a --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.locationastoken.log @@ -0,0 +1,5 @@ +0.000000 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +200.000000 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 +200.000001 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 (Repeated 99 times since 200.000000) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 (Repeated 99 times since 200.000001) diff --git a/vespalog/src/test/bufferedlogtest.logger1.cpp b/vespalog/src/test/bufferedlogtest.logger1.cpp new file mode 100644 index 00000000000..992de19fea5 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.logger1.cpp @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> +#include "bufferedlogtest.logger1.h" + +LOG_SETUP(".logger1"); + +void logWithLogger1(const std::string& token, const std::string message) +{ + LOGBT(info, token, message.c_str()); +} diff --git a/vespalog/src/test/bufferedlogtest.logger1.h b/vespalog/src/test/bufferedlogtest.logger1.h new file mode 100644 index 00000000000..e5336efec82 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.logger1.h @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string> + +void logWithLogger1(const std::string& token, const std::string message); + diff --git a/vespalog/src/test/bufferedlogtest.logger2.cpp b/vespalog/src/test/bufferedlogtest.logger2.cpp new file mode 100644 index 00000000000..688ecd153aa --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.logger2.cpp @@ -0,0 +1,11 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/log.h> +#include "bufferedlogtest.logger2.h" + +LOG_SETUP(".logger2"); + +void logWithLogger2(const std::string& token, const std::string message) +{ + LOGBT(info, token, message.c_str()); +} diff --git a/vespalog/src/test/bufferedlogtest.logger2.h b/vespalog/src/test/bufferedlogtest.logger2.h new file mode 100644 index 00000000000..0f18607a13d --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.logger2.h @@ -0,0 +1,8 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <string> + +void logWithLogger2(const std::string& token, const std::string message); + diff --git a/vespalog/src/test/bufferedlogtest.messageastoken.log b/vespalog/src/test/bufferedlogtest.messageastoken.log new file mode 100644 index 00000000000..e98fcd6caea --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.messageastoken.log @@ -0,0 +1,21 @@ +0.000000 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +200.000000 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 +200.000002 example.yahoo.com -1/-1 - bufferedlogtest info Message 1 +200.000004 example.yahoo.com -1/-1 - bufferedlogtest info Message 2 +200.000006 example.yahoo.com -1/-1 - bufferedlogtest info Message 3 +200.000008 example.yahoo.com -1/-1 - bufferedlogtest info Message 4 +200.000010 example.yahoo.com -1/-1 - bufferedlogtest info Message 5 +200.000012 example.yahoo.com -1/-1 - bufferedlogtest info Message 6 +200.000014 example.yahoo.com -1/-1 - bufferedlogtest info Message 7 +200.000016 example.yahoo.com -1/-1 - bufferedlogtest info Message 8 +200.000018 example.yahoo.com -1/-1 - bufferedlogtest info Message 9 +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 (Repeated 19 times since 200.000000) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 1 (Repeated 19 times since 200.000002) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 2 (Repeated 19 times since 200.000004) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 3 (Repeated 19 times since 200.000006) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 4 (Repeated 19 times since 200.000008) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 5 (Repeated 19 times since 200.000010) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 6 (Repeated 19 times since 200.000012) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 7 (Repeated 19 times since 200.000014) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 8 (Repeated 19 times since 200.000016) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest info Message 9 (Repeated 19 times since 200.000018) diff --git a/vespalog/src/test/bufferedlogtest.nostarve.log b/vespalog/src/test/bufferedlogtest.nostarve.log new file mode 100644 index 00000000000..aa507e00921 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.nostarve.log @@ -0,0 +1,26 @@ +10.000004 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +100.000000 example.yahoo.com -1/-1 - bufferedlogtest warning message +100.000001 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x1: NOT CONNECTED +100.000002 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x2: NOT_READY +100.000003 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x3: BAAAH +100.000004 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x4: RPC FAILURE +100.000005 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x5: COSMIC RADIATION +100.000006 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x6: ITS SATURDAY +100.000007 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x7: Yeah, Right!! +100.000008 example.yahoo.com -1/-1 - bufferedlogtest error Failed to send message to node 0x8: NOT CONNECTED +100.000009 example.yahoo.com -1/-1 - bufferedlogtest info Failed to send message to node 0x9: NOT CONNECTED +100.000010 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0xa: NOT CONNECTED +100.001001 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED +100.001001 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x1: NOT CONNECTED (Repeated 99 times since 100.000001) +100.001020 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED +100.001020 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x2: NOT_READY (Repeated 99 times since 100.000002) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x3: BAAAH (Repeated 99 times since 100.000003) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x4: RPC FAILURE (Repeated 99 times since 100.000004) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x5: COSMIC RADIATION (Repeated 99 times since 100.000005) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x6: ITS SATURDAY (Repeated 99 times since 100.000006) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0x7: Yeah, Right!! (Repeated 99 times since 100.000007) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest error Failed to send message to node 0x8: NOT CONNECTED (Repeated 99 times since 100.000008) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest info Failed to send message to node 0x9: NOT CONNECTED (Repeated 99 times since 100.000009) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 0xa: NOT CONNECTED (Repeated 99 times since 100.000010) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 4: NOT CONNECTED (Repeated 899 times since 100.001001) +100.003000 example.yahoo.com -1/-1 - bufferedlogtest warning Failed to send message to node 3: NOT CONNECTED (Repeated 99 times since 100.001020) diff --git a/vespalog/src/test/bufferedlogtest.tokenacrossloggers.log b/vespalog/src/test/bufferedlogtest.tokenacrossloggers.log new file mode 100644 index 00000000000..2c175701509 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.tokenacrossloggers.log @@ -0,0 +1,5 @@ +0.000000 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +200.000000 example.yahoo.com -1/-1 - bufferedlogtest.logger1 info Message 0 +200.000001 example.yahoo.com -1/-1 - bufferedlogtest.logger2 info Message 0 +200.000200 example.yahoo.com -1/-1 - bufferedlogtest.logger1 info Message 0 (Repeated 99 times since 200.000000) +200.000200 example.yahoo.com -1/-1 - bufferedlogtest.logger2 info Message 0 (Repeated 99 times since 200.000001) diff --git a/vespalog/src/test/bufferedlogtest.trimcache.log b/vespalog/src/test/bufferedlogtest.trimcache.log new file mode 100644 index 00000000000..d99e7357808 --- /dev/null +++ b/vespalog/src/test/bufferedlogtest.trimcache.log @@ -0,0 +1,4 @@ +0.000000 example.yahoo.com -1/-1 - bufferedlogtest info Starting up, using logfile bufferedlogtest.log +200.000000 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 +100000.000000 example.yahoo.com -1/-1 - bufferedlogtest info This should cause buffered log flush +100000.000000 example.yahoo.com -1/-1 - bufferedlogtest info Message 0 (Repeated 99 times since 200.000000) diff --git a/vespalog/src/test/bufferedlogtest_test.sh b/vespalog/src/test/bufferedlogtest_test.sh new file mode 100755 index 00000000000..0ff927b946d --- /dev/null +++ b/vespalog/src/test/bufferedlogtest_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +rm -f bufferedlogtest.log +VESPA_LOG_TARGET="file:bufferedlogtest.log" ./vespalog_bufferedlogtest_app bufferedlogtest.log diff --git a/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java b/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java new file mode 100644 index 00000000000..deecebe0b81 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/FileLogTargetTest.java @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.Test; +import org.junit.After; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; + +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class FileLogTargetTest { + @Test(expected = RuntimeException.class) + public void requireThatExceptionIsThrowIfFileNotFound() throws IOException { + File file = new File("mydir1"); + file.delete(); + assertTrue(file.mkdir()); + new FileLogTarget(file).open(); + } + + @After + public void cleanup() { new File("mydir1").delete(); } + + @Test + public void requireThatFileCanOpened() throws IOException { + FileLogTarget logTarget = new FileLogTarget(File.createTempFile("logfile", ".log")); + assertNotNull(logTarget.open()); + } + + @Test + public void requireThatFileIsReopened() throws IOException { + FileLogTarget logTarget = new FileLogTarget(File.createTempFile("logfile", ".log")); + OutputStream out1 = logTarget.open(); + assertNotNull(out1); + OutputStream out2 = logTarget.open(); + assertNotNull(out2); + assertThat(out1, is(not(out2))); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java new file mode 100644 index 00000000000..cef2acd99da --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/LogLevelTestCase.java @@ -0,0 +1,78 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.Set; +import java.util.HashSet; +import java.util.logging.Level; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Testcases for LogLevel + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class LogLevelTestCase { + + /** + * Ensure that all the log levels we need are present + * and that they are distinct. + */ + @Test + public void testLogLevels () { + assertNotNull(LogLevel.UNKNOWN); + assertNotNull(LogLevel.FATAL); + assertNotNull(LogLevel.ERROR); + assertNotNull(LogLevel.WARNING); + assertNotNull(LogLevel.INFO); + assertNotNull(LogLevel.CONFIG); + assertNotNull(LogLevel.EVENT); + assertNotNull(LogLevel.DEBUG); + assertNotNull(LogLevel.SPAM); + + // use a set to verify that all are distinct + Set<Level> seen = new HashSet<Level>(); + assertTrue(seen.add(LogLevel.UNKNOWN)); + assertTrue(seen.add(LogLevel.FATAL)); + assertTrue(seen.add(LogLevel.ERROR)); + assertTrue(seen.add(LogLevel.WARNING)); + assertTrue(seen.add(LogLevel.INFO)); + assertTrue(seen.add(LogLevel.CONFIG)); + assertTrue(seen.add(LogLevel.EVENT)); + assertTrue(seen.add(LogLevel.DEBUG)); + assertTrue(seen.add(LogLevel.SPAM)); + + // verify that set would trigger error (not necessary) + assertTrue(! seen.add(LogLevel.SPAM)); + } + + /** + * Test that given the log level name we are able to + * map it to the correct static instance. + */ + @Test + public void testNameToLevelMapping () { + assertEquals(LogLevel.UNKNOWN, LogLevel.parse("unknown")); + assertEquals(LogLevel.FATAL, LogLevel.parse("fatal")); + assertEquals(LogLevel.ERROR, LogLevel.parse("error")); + assertEquals(LogLevel.WARNING, LogLevel.parse("warning")); + assertEquals(LogLevel.INFO, LogLevel.parse("info")); + assertEquals(LogLevel.CONFIG, LogLevel.parse("config")); + assertEquals(LogLevel.EVENT, LogLevel.parse("event")); + assertEquals(LogLevel.DEBUG, LogLevel.parse("debug")); + assertEquals(LogLevel.SPAM, LogLevel.parse("spam")); + } + + @Test + public void testJavaToLevelMapping () { + assertTrue(LogLevel.ERROR == LogLevel.getVespaLogLevel(Level.SEVERE)); + assertTrue(LogLevel.WARNING == LogLevel.getVespaLogLevel(Level.WARNING)); + assertTrue(LogLevel.INFO == LogLevel.getVespaLogLevel(Level.INFO)); + assertTrue(LogLevel.CONFIG == LogLevel.getVespaLogLevel(Level.CONFIG)); + assertTrue(LogLevel.DEBUG == LogLevel.getVespaLogLevel(Level.FINE)); + assertTrue(LogLevel.DEBUG == LogLevel.getVespaLogLevel(Level.FINER)); + assertTrue(LogLevel.SPAM == LogLevel.getVespaLogLevel(Level.FINEST)); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java new file mode 100644 index 00000000000..9b5eb0ed6ce --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/LogSetupTestCase.java @@ -0,0 +1,153 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.io.IOException; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import java.util.logging.Level; +import java.io.File; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.Before; +import org.junit.rules.TemporaryFolder; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.is; + +/** + * Make sure we can install the logging stuff properly. + * + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class LogSetupTestCase { + // For testing zookeeper log records + protected static LogRecord zookeeperLogRecord; + protected static LogRecord zookeeperLogRecordError; + protected static LogRecord curatorLogRecord; + protected static LogRecord curatorLogRecordError; + protected static String zookeeperLogRecordString; + protected static LogRecord notzookeeperLogRecord; + + protected static String hostname; + protected static String pid; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Before + public void setUp() { + System.setProperty("config.id", "my-test-config-id"); + System.setProperty("vespa.log.target", "file:test3"); + + zookeeperLogRecord = new LogRecord(Level.WARNING, "zookeeper log record"); + zookeeperLogRecord.setLoggerName("org.apache.zookeeper.server.NIOServerCnxn"); + zookeeperLogRecord.setMillis(1107011348029L); + + curatorLogRecord = new LogRecord(Level.WARNING, "curator log record"); + curatorLogRecord.setLoggerName("com.netflix.curator.utils.DefaultTracerDriver"); + curatorLogRecord.setMillis(1107011348029L); + + hostname = Util.getHostName(); + pid = Util.getPID(); + + zookeeperLogRecordString = "1107011348.029\t" + + hostname + + "\t" + + pid + + "/" + zookeeperLogRecord.getThreadID() + "\t-\t.org.apache.zookeeper.server.NIOServerCnxn" + + "\twarning\tzookeeper log record"; + + zookeeperLogRecordError = new LogRecord(Level.SEVERE, "zookeeper error"); + zookeeperLogRecordError.setLoggerName("org.apache.zookeeper.server.NIOServerCnxn"); + zookeeperLogRecordError.setMillis(1107011348029L); + + curatorLogRecordError = new LogRecord(Level.SEVERE, "curator log record"); + curatorLogRecordError.setLoggerName("com.netflix.curator.utils.DefaultTracerDriver"); + curatorLogRecordError.setMillis(1107011348029L); + + notzookeeperLogRecord = new LogRecord(Level.WARNING, "not zookeeper log record"); + notzookeeperLogRecord.setLoggerName("org.apache.foo.Bar"); + notzookeeperLogRecord.setMillis(1107011348029L); + } + + @Test + public void testSetup() throws IOException { + try { + final File zookeeperLogFile = folder.newFile("zookeeper.log"); + System.setProperty("zookeeperlogfile", zookeeperLogFile.getAbsolutePath()); + LogSetup.initVespaLogging("TST"); + Logger.getLogger("").log(VespaLogHandlerTestCase.record2); + Logger.getLogger("").log(VespaLogHandlerTestCase.record1); + Logger.getLogger("").log(VespaLogHandlerTestCase.record2); + Logger.getLogger("").log(zookeeperLogRecord); // Should not be written to log, due to use of ZooKeeperFilter + Logger.getLogger("").log(curatorLogRecord); // Should not be written to log, due to use of ZooKeeperFilter + Logger.getLogger("").log(notzookeeperLogRecord); + String[] lines = VespaLogHandlerTestCase.readFile("test3"); + assertEquals(2, lines.length); + assertEquals(VespaLogHandlerTestCase.record1String, lines[0]); + } finally { + LogSetup.clearHandlers(); + LogSetup.getLogHandler().closeFileTarget(); + assertTrue(new File("test3").delete()); + } + } + + // Note: This test will generate warnings about initVespaLogging being called twice, which can be ignored + @Test + public void testLogLevelSetting() { + + // levels are ordered like: + // C++ has fatal, error, warning, config, info, event, debug, spam + // Java has fatal, error, warning, event, config, info, debug, spam + + // logctl file values: fatal, error, warning, config, info, event, debug, spam + + System.setProperty("vespa.log.target", "fd:2"); + setupAndCheckLevels(null, " ON ON ON ON ON ON OFF OFF", LogLevel.INFO, LogLevel.DEBUG); + setupAndCheckLevels("", " ON ON ON ON ON ON OFF OFF", LogLevel.INFO, LogLevel.DEBUG); + setupAndCheckLevels("all", " ON ON ON ON ON ON ON ON", LogLevel.SPAM, null); + setupAndCheckLevels("all -spam", " ON ON ON ON ON ON ON OFF", LogLevel.DEBUG, LogLevel.SPAM); + setupAndCheckLevels("all -debug", " ON ON ON ON ON ON OFF ON", LogLevel.INFO, null); + setupAndCheckLevels("all -debug -spam", " ON ON ON ON ON ON OFF OFF", LogLevel.INFO, LogLevel.DEBUG); + // INFO is higher in level value than CONFIG, so one should rather use -info -config + setupAndCheckLevels("all -debug -spam -info", " ON ON ON ON OFF ON OFF OFF", LogLevel.WARNING, LogLevel.DEBUG); + setupAndCheckLevels("all -debug -spam -info -config", " ON ON ON OFF OFF ON OFF OFF", LogLevel.WARNING, LogLevel.INFO); + setupAndCheckLevels("all debug", " ON ON ON ON ON ON ON ON", LogLevel.SPAM, null); + setupAndCheckLevels("debug", " OFF OFF OFF OFF OFF OFF ON OFF", null, null); + setupAndCheckLevels("debug error", " OFF ON OFF OFF OFF OFF ON OFF", LogLevel.ERROR, LogLevel.SPAM); + } + + @Test + public void testZooKeeperFilter() throws IOException { + final File file = folder.newFile("zookeeper.log"); + LogSetup.ZooKeeperFilter filter = new LogSetup.ZooKeeperFilter(file.getAbsolutePath()); + assertThat(filter.isLoggable(zookeeperLogRecord), is(false)); + //assertThat(filter.isLoggable(zookeeperLogRecordError), is(true)); + assertThat(filter.isLoggable(notzookeeperLogRecord), is(true)); + String[] lines = VespaLogHandlerTestCase.readFile(file.getAbsolutePath()); + assertThat(lines.length, is(1)); + assertEquals(zookeeperLogRecordString, lines[0]); + } + + private void setupAndCheckLevels(String levelString, String expectedOnOffString, Level shouldLog, Level shouldNotLog) { + try { + if (levelString != null) { + System.setProperty("vespa.log.level", levelString); + } + LogSetup.initVespaLogging("TST"); + Logger.getLogger("TEST").log(LogLevel.DEBUG, "DEBUG"); + LevelController levelController = LogSetup.getLogHandler().getLevelControl("TST"); + assertNotNull(levelController); + assertThat(levelController.getOnOffString(), is(expectedOnOffString)); + if (shouldLog != null) { + assertTrue(levelController.shouldLog(shouldLog)); + } + if (shouldNotLog != null) { + assertFalse(levelController.shouldLog(shouldNotLog)); + } + } finally { + LogSetup.clearHandlers(); + } + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java b/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java new file mode 100644 index 00000000000..2a0b1e1caaa --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/LogUtilTest.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * @author lulf + * @since 5.1 + */ +public class LogUtilTest { + @Test + public void testEmpty() { + assertTrue(LogUtil.empty(null)); + assertTrue(LogUtil.empty("")); + assertFalse(LogUtil.empty("f")); + assertFalse(LogUtil.empty("fo")); + assertFalse(LogUtil.empty("foo")); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java b/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java new file mode 100644 index 00000000000..a562aa1cb01 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/RejectFilterTest.java @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.Test; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class RejectFilterTest { + @Test + public void testBasicPatternMatching() { + RejectFilter rejectFilter = new RejectFilter(); + assertFalse(rejectFilter.shouldReject("This is a test")); + rejectFilter.addRejectedMessage("This is a test"); + assertTrue(rejectFilter.shouldReject("This is a test")); + rejectFilter.addRejectedMessage("This is not a test"); + assertTrue(rejectFilter.shouldReject("This is a test")); + assertTrue(rejectFilter.shouldReject("This is not a test")); + assertFalse(rejectFilter.shouldReject("This is not not a test")); + assertFalse(rejectFilter.shouldReject(null)); + } + + @Test + public void testDefaultRejectPattern() { + RejectFilter filter = RejectFilter.createDefaultRejectFilter(); + assertTrue(filter.shouldReject("E 23-235018.067240 14650 23/10/2012 23:50:18 yjava_preload.so: [preload.c:350] Using FILTER_NONE: This must be paranoid approved, and since you are using FILTER_NONE you must live with this error.")); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/TimeComparatorTestCase.java b/vespalog/src/test/java/com/yahoo/log/TimeComparatorTestCase.java new file mode 100644 index 00000000000..28d34d38a86 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/TimeComparatorTestCase.java @@ -0,0 +1,71 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Unit test for the LogMessage TimeComparator + * + * @author vlarsen + * + */ +public class TimeComparatorTestCase { + private String msgstr1 = "1098709000\t" + + "nalle.puh.com\t" + + "23234\t" + + "serviceName\t" + + "tst\t" + + "info\t" + + "this is a test\n"; + private LogMessage msg1; + + private String msgstr2 = "1098709021\t" + + "nalle.puh.com\t" + + "23234\t" + + "serviceName\t" + + "tst\t" + + "info\t" + + "this is a test\n"; + private LogMessage msg2; + + @Test + public void testAscendingOrder() { + try { + msg1 = LogMessage.parseNativeFormat(msgstr1.trim()); + msg2 = LogMessage.parseNativeFormat(msgstr2.trim()); + } catch (InvalidLogFormatException e) { + fail(); + } + Set<LogMessage> msglist = new TreeSet<LogMessage>(new LogMessageTimeComparator()); + msglist.add(msg1); + msglist.add(msg2); + + Iterator<LogMessage> it = msglist.iterator(); + assertEquals(msg1, it.next()); + assertEquals(msg2, it.next()); + } + + @Test + public void testDescendingOrder() { + try { + msg1 = LogMessage.parseNativeFormat(msgstr1.trim()); + msg2 = LogMessage.parseNativeFormat(msgstr2.trim()); + } catch (InvalidLogFormatException e) { + fail(); + } + Set<LogMessage> msglist = new TreeSet<LogMessage>(new LogMessageTimeComparator(false)); + msglist.add(msg1); + msglist.add(msg2); + + // note, last gets fetched first + Iterator<LogMessage> it = msglist.iterator(); + assertEquals(msg2, it.next()); + assertEquals(msg1, it.next()); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java b/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java new file mode 100644 index 00000000000..7fa98791b86 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/UncloseableOutputStreamTestCase.java @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * @author <a href="mailto:simon@yahoo-inc.com">Simon Thoresen Hult</a> + */ +public class UncloseableOutputStreamTestCase { + + @Test + public void requireThatWriteIsProxied() throws IOException { + OutputStream out = Mockito.mock(OutputStream.class); + new UncloseableOutputStream(out).write(69); + Mockito.verify(out).write(69); + } + + @Test + public void requireThatCloseIsIgnored() { + OutputStream out = Mockito.mock(OutputStream.class); + new UncloseableOutputStream(out).close(); + Mockito.verifyNoMoreInteractions(out); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java b/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java new file mode 100644 index 00000000000..3f3da8436bd --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/UtilTestCase.java @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class UtilTestCase { + + /** + * Just make sure the static getHostName() method returns something + * that looks half sensible. + */ + @Test + public void testSimple () { + String name = Util.getHostName(); + assertNotNull(name); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java new file mode 100644 index 00000000000..300514a4f3f --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/VespaFormatterTestCase.java @@ -0,0 +1,264 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import org.junit.Test; +import org.junit.Before; +import org.junit.Ignore; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.is; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class VespaFormatterTestCase { + + private String hostname; + private String pid; + private String serviceName = "serviceName"; + private String app = "tst"; + private LogRecord testRecord1; + private LogRecord testRecord2; + private String expected1; + private String expected2; + private String expected3; + private String expected4; + + + @Before + public void setUp () { + hostname = Util.getHostName(); + pid = Util.getPID(); + + testRecord1 = new LogRecord(Level.INFO, "this is a test"); + testRecord1.setMillis(1098709021843L); + testRecord1.setThreadID(123); + + expected1 = "1098709021.843\t" + + hostname + "\t" + + pid + + "/123" + "\t" + + "serviceName\t" + + "tst\t" + + "info\t" + + "this is a test\n"; + + expected2 = "1098709021.843\t" + + hostname + "\t" + + pid + + "/123" + "\t" + + "serviceName\t" + + "-\t" + + "info\t" + + "this is a test\n"; + + + testRecord2 = new LogRecord(Level.INFO, "this is a test"); + testRecord2.setMillis(1098709021843L); + testRecord2.setThreadID(123); + testRecord2.setLoggerName("org.foo"); + + expected3 = "1098709021.843\t" + + hostname + "\t" + + pid + + "/123" + "\t" + + "serviceName\t" + + ".org.foo\t" + + "info\t" + + "this is a test\n"; + + expected4 = "1098709021.843\t" + + hostname + "\t" + + pid + + "/123" + "\t" + + "serviceName\t" + + "tst.org.foo\t" + + "info\t" + + "this is a test\n"; + } + + /** + * Just make sure that the messages are parsed as expected. + */ + @Test + public void testFormatting () { + VespaFormatter formatter = new VespaFormatter(serviceName, app); + assertEquals(expected1, formatter.format(testRecord1)); + assertEquals(expected4, formatter.format(testRecord2)); + + VespaFormatter fmt2 = new VespaFormatter(serviceName, null); + assertEquals(expected2, fmt2.format(testRecord1)); + assertEquals(expected3, fmt2.format(testRecord2)); + } + + + /** + * test that {0} etc is replaced properly + */ + @Test + public void testTextFormatting () { + VespaFormatter formatter = new VespaFormatter(serviceName, app); + + LogRecord testRecord = new LogRecord(Level.INFO, "this {1} is {0} test"); + testRecord.setMillis(1098709021843L); + testRecord.setThreadID(123); + testRecord.setLoggerName("org.foo"); + Object[] params = { "a small", "message" }; + testRecord.setParameters(params); + + String expected = "1098709021.843\t" + + hostname + "\t" + + pid + + "/123" + "\t" + + "serviceName\t" + + app + + ".org.foo\t" + + "info\t" + + "this message is a small test\n"; + + assertEquals(expected, formatter.format(testRecord)); + } + + /** + * Make sure that the LogMessage parser used in the log server is + * able to parse these messages too. + */ + @Test + public void testParsing () { + VespaFormatter formatter = new VespaFormatter(serviceName, app); + LogMessage m; + try { + m = LogMessage.parseNativeFormat(expected1.trim()); + assertEquals("this is a test", m.getPayload()); + + m = LogMessage.parseNativeFormat(formatter.format(testRecord1).trim()); + assertEquals("this is a test", m.getPayload()); + + assertTrue(true); + } + catch (InvalidLogFormatException e) { + System.out.println(e.toString()); + fail(); + } + } + + /** + * This method was used for testing the speed of the formatter. + * We usually do not want to run this in normal testing, it was + * just added when trying to analyze the abysmal performance of + * double formatting. + */ + @Test + @Ignore + public void testSpeed () { + VespaFormatter formatter = new VespaFormatter(serviceName, app); + int reps = 1000000; + long start = System.currentTimeMillis(); + for (int i = 0; i < reps; i++) { + formatter.format(testRecord1); + } + long diff = System.currentTimeMillis() - start; + double dd = (reps / (double) diff); + System.out.println("Took " + diff + " milliseconds, per ms=" + dd); + } + + + /** test backslash, newline and tab escaping. one instance of each */ + @Test + public void testEscaping () { + String a = "This line contains a \\ backslash and \n newline and \t tab."; + String b = "This line contains a \\\\ backslash and \\n newline and \\t tab."; + assertEquals(b, VespaFormat.escape(a)); + assertEquals(a, VespaFormatter.unEscape(b)); + } + + /** test multiple instances of backslash */ + @Test + public void testMultipleEscaping () { + String a = "This line contains a \\ backslash and \\ more backslash."; + String b = "This line contains a \\\\ backslash and \\\\ more backslash."; + assertEquals(b, VespaFormat.escape(a)); + assertEquals(a, VespaFormatter.unEscape(b)); + } + + @Test + public void testThrowable () { + // VespaFormatter formatter = new VespaFormatter(serviceName, app); + LogRecord r = new LogRecord(Level.INFO, "foo bar"); + r.setThrown(new IllegalStateException("this was fun")); + } + + /** + * Got a NullPointerException earlier when trying to format + * a null message. Added unit test to make sure this does + * not happen again. Makes sure that NullPointerException + * is not thrown and that the formatted string contains the + * expected message for saying that there was no log message. + */ + @Test + public void testNullMessage () { + VespaFormatter formatter = new VespaFormatter(serviceName, app); + LogRecord r = new LogRecord(LogLevel.ERROR, null); + assertNotNull(r); + try { + String s = formatter.format(r); + assertNotNull(s); + assertTrue(s.indexOf("(empty)") > 0); + } + catch (NullPointerException e) { + fail("unable to handle null message!"); + } + } + + @Test + public void testLowLogLevelExceptionFormatting () { + VespaFormatter formatter = new VespaFormatter(serviceName, app); + LogRecord r = new LogRecord(LogLevel.INFO, "meldingen her"); + r.setThrown(new IllegalStateException()); + String result = formatter.format(r); + assertTrue(formatter.format(r).contains("meldingen her")); + } + + /** + * This test doesn't really do that much. It is just here + * to ensure this doesn't crash. XXX TODO: make this test + * actually test something more than just the non-generation + * of runtime errors. -bb + */ + @Test + public void testExceptionFormatting () { + StringBuilder sb = new StringBuilder(128); + Exception e = new Exception("testing", new Exception("nested")); + VespaFormat.formatException(e, sb); + } + + + @Test + public void testGeneralFormat() { + String[] expected = new String[] { + "54.321", + "hostname", + "26019/UnitTest-Thread-37", + "UnitTestRunner", + "com.UnitTest", + "INFO", + "Just check it looks OK\\nmsg=\"boom\"\\nname=\"java.lang.Throwable\"\\nstack=\"\\n" + this.getClass().getName() // Clover rewrites class names, get the current one to avoid test failure + }; + String formatted = VespaFormat.format("INFO", + "UnitTest", "com", 54321L, + "UnitTest-Thread-37", "UnitTestRunner", + "Just check it looks OK", + new Throwable("boom")); + String[] split = formatted.split("\t"); + assertEquals(expected[0], split[0]); + assertEquals(expected[2].split("/")[1], split[2].split("/")[1]); + assertEquals(expected[3], split[3]); + assertEquals(expected[4], split[4]); + assertEquals(expected[5], split[5]); + assertEquals(expected[6], split[6].substring(0, expected[6].length())); + assertEquals(expected.length, split.length); + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java new file mode 100644 index 00000000000..948931450f4 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java @@ -0,0 +1,107 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.Test; +import org.junit.Ignore; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * @author lulf + * @since 5.1 + */ +public class VespaLevelControllerRepoTest { + + static int findControlString(RandomAccessFile f, String s) { + try { + String toFind = "\n." + s + ": "; + byte[] contents = new byte[(int) f.length()]; + f.seek(0); + f.read(contents); + f.seek(0); + String c_as_s = new String(contents, "US-ASCII"); + int off = c_as_s.indexOf(toFind); + if (off < 0) { + System.err.println("did not find control line for level '"+s+"' in logcontrol file:"); + System.err.println(c_as_s); + throw new RuntimeException("bad state in logcontrol file"); + } + off += toFind.length(); + while ((off % 4) != 0) ++off; + return off; + } catch (IOException e) { + throw new RuntimeException("problem reading logcontrol file", e); + } + } + + @Test + public void testLogCtl () throws InterruptedException, IOException { + File lcf = new File("./my-testilol-config-id.logcontrol"); + try { + lcf.delete(); + Logger.getLogger("com.yahoo.log.test").setLevel(null); + assertEquals(null, + Logger.getLogger("com.yahoo.log.test").getLevel()); + + LevelControllerRepo repo = new VespaLevelControllerRepo(lcf.getName(), "all -debug -spam", "TST"); + + long timeout = System.currentTimeMillis() + 60000; + while (System.currentTimeMillis() < timeout) { + if (Level.CONFIG == Logger.getLogger("com.yahoo.log.test").getLevel()) + break; + Thread.sleep(100); + } + + RandomAccessFile lcfile = new RandomAccessFile(lcf, "rw"); + + lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength); + assertEquals(lcfile.readByte(), 'd'); + lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength + 7); + assertEquals(lcfile.readByte(), ':'); + assertEquals(0, (VespaLevelControllerRepo.controlFileHeaderLength+9) % 4); + lcfile.seek(VespaLevelControllerRepo.controlFileHeaderLength + 9); + assertEquals(0x20204f4e, lcfile.readInt()); + + int off = findControlString(lcfile, "com.yahoo.log.test"); + lcfile.seek(off); + assertEquals(0x20204f4e, lcfile.readInt()); + assertEquals(0x20204f4e, lcfile.readInt()); + assertEquals(0x20204f4e, lcfile.readInt()); + assertEquals(0x20204f4e, lcfile.readInt()); + assertEquals(0x20204f4e, lcfile.readInt()); + assertEquals(0x20204f4e, lcfile.readInt()); + + assertEquals(0x204f4646, lcfile.readInt()); + assertEquals(0x204f4646, lcfile.readInt()); + lcfile.seek(off); + lcfile.writeInt(0x204f4646); + lcfile.writeInt(0x204f4646); + lcfile.writeInt(0x204f4646); + lcfile.writeInt(0x204f4646); + lcfile.writeInt(0x204f4646); + lcfile.writeInt(0x204f4646); + + lcfile.writeInt(0x20204f4e); + lcfile.writeInt(0x20204f4e); + lcfile.close(); + assertFalse(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.FATAL)); + assertFalse(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.ERROR)); + assertFalse(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.WARNING)); + assertFalse(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.INFO)); + assertFalse(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.EVENT)); + assertFalse(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.CONFIG)); + assertTrue(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.DEBUG)); + assertTrue(repo.getLevelController("com.yahoo.log.test").shouldLog(LogLevel.SPAM)); + } finally { + lcf.delete(); + } + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java new file mode 100644 index 00000000000..c231efed2dd --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/VespaLogHandlerTestCase.java @@ -0,0 +1,440 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.*; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.*; + +/** + * @author <a href="mailto:borud@yahoo-inc.com">Bjorn Borud</a> + */ +public class VespaLogHandlerTestCase { + protected static String hostname; + protected static String pid; + + protected static LogRecord record1; + protected static String record1String; + + protected static LogRecord record2; + protected static String record2String; + + protected static LogRecord record3; + protected static String record3String; + + protected static LogRecord record4; + protected static String record4String; + + static { + hostname = Util.getHostName(); + pid = Util.getPID(); + + record1 = new LogRecord(Level.INFO, "This is a test"); + record1.setMillis(1100011348029L); + record1String = "1100011348.029\t" + + hostname + + "\t" + + pid + + "/" + + record1.getThreadID() + + "\tmy-test-config-id\tTST\tinfo\tThis is a test"; + + record2 = new LogRecord(Level.FINE, "This is a test too"); + record2.setMillis(1100021348029L); + record2.setLoggerName("com.yahoo.log.test"); + record2String = "1100021348.029\t" + + hostname + + "\t" + + pid + + "/" + record2.getThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test\tdebug\tThis is a test too"; + + record3 = new LogRecord(Level.WARNING, "another test"); + record3.setLoggerName("com.yahoo.log.test"); + record3.setMillis(1107011348029L); + record3String = "1107011348.029\t" + + hostname + + "\t" + + pid + + "/" + record3.getThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test" + + "\twarning\tanother test"; + + record4 = new LogRecord(Level.WARNING, "unicode \u00E6\u00F8\u00E5 test \u7881 unicode"); + record4.setLoggerName("com.yahoo.log.test"); + record4.setMillis(1107011348029L); + record4String = "1107011348.029\t" + + hostname + + "\t" + + pid + + "/" + record4.getThreadID() + "\tmy-test-config-id\tTST.com.yahoo.log.test" + + "\twarning\tunicode \u00E6\u00F8\u00E5 test \u7881 unicode"; + } + + @Before + public void setUp() { + // System.setProperty("vespa.log.level", "all"); + // System.setProperty("vespa.log.control.dir", "."); + // System.setProperty("config.id", "my-test-config-id"); + } + + @After + public void tearDown() { + new File("./my-test-config-id.logcontrol").delete(); + } + + /** + * Perform simple test + */ + @Test + public void testSimple () throws IOException { + File logfile = new File("./test1"); + File ctlfile = new File("./my-test-config-id.logcontrol"); + try { + logfile.delete(); + ctlfile.delete(); + VespaLogHandler h + = new VespaLogHandler(new FileLogTarget(new File("test1")), + new VespaLevelControllerRepo(ctlfile.getName(), "all", "TST"), + "my-test-config-id", "TST"); + h.publish(record1); + h.cleanup(); + h.close(); + + File f = new File("test1"); + assertTrue(f.exists()); + + String[] lines = readFile(f.getName()); + + // make sure there is only one line in the file + assertEquals(1, lines.length); + + // make sure that ine line matches what is expected + assertEquals(record1String, lines[0]); + } + finally { + logfile.delete(); + ctlfile.delete(); + } + } + + @Test + public void testFallback() throws FileNotFoundException { + File file = new File("mydir2"); + file.delete(); + assertTrue(file.mkdir()); + try { + new VespaLogHandler(new FileLogTarget(file), new VespaLevelControllerRepo("my-test-config-id.logcontrol", "all", "TST"), "my-test-config-id", "TST"); + } catch (Exception e) { + fail("Should not throw exception: " + e.getMessage()); + } + } + + @After + public void cleanup() { new File("mydir2").delete(); } + + /** + * Perform simple test + */ + @Test + public void testLogCtl () throws InterruptedException, FileNotFoundException { + MockLevelController ctl = new MockLevelController(); + MockLevelControllerRepo ctlRepo = new MockLevelControllerRepo(ctl); + MockLogTarget target = new MockLogTarget(); + VespaLogHandler h = new VespaLogHandler(target, + ctlRepo, + "my-test-config-id", "TST"); + ctl.setShouldLog(Level.INFO); + h.publish(record1); + h.publish(record2); + h.publish(record3); + h.publish(record4); + + ctl.setShouldLog(Level.CONFIG); + h.publish(record1); + h.publish(record2); + h.publish(record3); + h.publish(record4); + + ctl.setShouldLog(Level.WARNING); + h.publish(record1); + h.publish(record2); + h.publish(record3); + h.publish(record4); + + ctl.setShouldLog(Level.FINE); + h.publish(record1); + h.publish(record2); + h.publish(record3); + h.publish(record4); + + h.close(); + String [] lines = target.getLines(); + assertThat(lines.length, is(4)); + assertEquals(record1String, lines[0]); + assertEquals(record3String, lines[1]); + //assertEquals(record4String, lines[2]); + assertEquals(record2String, lines[3]); + } + + /** + * Make sure rotation works + */ + @Test + public void testRotate () throws IOException { + // Doesn't work in Windows. TODO: Fix the logging stuff + if (System.getProperty("os.name").toLowerCase().indexOf("win")>=0) + return; + try { + VespaLogHandler h + = new VespaLogHandler(new FileLogTarget(new File("test2")), + new VespaLevelControllerRepo("my-test-config-id.logcontrol", "all", "TST"), "my-test-config-id", + "TST" + ); + h.publish(record1); + + // before rename + assertTrue(new File("test2").exists()); + assertFalse(new File("test2-rotated").exists()); + + // rename file + assertTrue("Could rename test2 to test2-rotated", + new File("test2").renameTo(new File("test2-rotated"))); + + // log next entry + h.publish(record2); + h.cleanup(); + h.close(); + + // now make sure both files exist + assertTrue(new File("test2").exists()); + assertTrue(new File("test2-rotated").exists()); + + // read both files + String[] lines1 = readFile("test2"); + String[] lines2 = readFile("test2-rotated"); + + // make sure they have the correct number of lines + assertEquals(1, lines1.length); + assertEquals(1, lines2.length); + + // make sure they match the correct strings + assertEquals(record2String, lines1[0]); + assertEquals(record1String, lines2[0]); + } + finally { + new File("test2").delete(); + new File("test2-rotated").delete(); + } + } + + + /** + * This test was made in order to look for a race condition + * which occurs when file rotation in the VespaLogHandler + * occurs concurrently (which it should never do). + */ + @Test + public void testRaceCondition () throws FileNotFoundException { + int numThreads = 10; + Thread[] t = new Thread[numThreads]; + final CyclicBarrier barrier = new CyclicBarrier(numThreads); + final int numLogEntries = 100; + + try { + final VespaLogHandler h = + new VespaLogHandler(new FileLogTarget(new File("test4")), + new VespaLevelControllerRepo("my-test-config-id.logcontrol", "all", "TST"), "my-test-config-id", + "TST" + ); + + class LogRacer implements Runnable { + private int n; + + public LogRacer (int n) { + this.n = n; + } + + public void run () { + try { + barrier.await(); + logLikeCrazy(); + } + catch (BrokenBarrierException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + public void logLikeCrazy () { + for (int j = 0; j < numLogEntries; j++) { + try { + h.publish(record1); + h.publish(record2); + } + catch (Throwable t) { + fail(t.getMessage()); + } + } + } + } + + for (int i = 0; i < numThreads; i++) { + t[i] = new Thread(new LogRacer(i)); + t[i].start(); + } + + for (int i = 0; i < numThreads; i++) { + t[i].join(); + } + + String[] lines = readFile("test4"); + assertTrue("num lines was " + lines.length + + " should be " + (2 * numLogEntries * numThreads), + (lines.length == (2 * numLogEntries * numThreads))); + + h.cleanup(); + h.close(); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + finally { + new File("test4").delete(); + } + + } + + /** + * Make sure unicode characters in log message works + */ + @Test + public void testUnicodeLog () throws IOException { + try { + VespaLogHandler h + = new VespaLogHandler(new FileLogTarget(new File("test4")), + new VespaLevelControllerRepo("my-test-config-id.logcontrol", "all", "TST"), "my-test-config-id", + "TST" + ); + h.publish(record4); + h.cleanup(); + h.close(); + + // check that file exists + assertTrue(new File("test4").exists()); + + // read file + String[] lines = readFile("test4"); + + // make sure the file have the correct number of lines + assertEquals(1, lines.length); + + // make sure the read lines match the correct string + assertEquals(record4String, lines[0]); + } + finally { + new File("test4").delete(); + } + } + + /** + * read a text file into a string array + * + */ + protected static String[] readFile (String fileName) { + BufferedReader br = null; + List<String> lines = new LinkedList<String>(); + try { + br = new BufferedReader( + new InputStreamReader(new FileInputStream(new File(fileName)), "UTF-8")); + for (String line = br.readLine(); + line != null; + line = br.readLine()) + { + lines.add(line); + } + return lines.toArray(new String[lines.size()]); + } + catch (Throwable e) { + return new String[0]; + } + finally { + if (br != null) { + try { + br.close(); + } + catch (IOException e) {} + } + } + } + + private static class MockLevelControllerRepo implements LevelControllerRepo { + private LevelController levelController; + public MockLevelControllerRepo(LevelController controller) { + this.levelController = controller; + } + + @Override + public LevelController getLevelController(String component) { + return levelController; + } + + @Override + public void close() { } + } + + private static class MockLevelController implements LevelController { + + private Level logLevel = Level.ALL; + + @Override + public boolean shouldLog(Level level) { + return (level.equals(logLevel)); + } + + public void setShouldLog(Level level) { + this.logLevel = level; + } + + + @Override + public String getOnOffString() { return ""; } + + @Override + public void checkBack() { } + + @Override + public Level getLevelLimit() { + return logLevel; + } + } + + private static class MockLogTarget implements LogTarget { + private final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + public String[] getLines() { + return baos.toString().split("\n"); + } + @Override + public OutputStream open() { + return baos; + } + + @Override + public void close() { + try { + baos.close(); + } catch (IOException e) { + fail("Test failed: " + e.getMessage()); + } + } + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java b/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java new file mode 100644 index 00000000000..de116bc2e74 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/event/EventTestCase.java @@ -0,0 +1,199 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +import java.util.logging.Logger; + +import com.yahoo.log.VespaFormatter; +import com.yahoo.log.event.Collection; +import com.yahoo.log.event.Count; +import com.yahoo.log.event.Crash; +import com.yahoo.log.event.Event; +import com.yahoo.log.event.MalformedEventException; +import com.yahoo.log.event.Progress; +import com.yahoo.log.event.Reloaded; +import com.yahoo.log.event.Reloading; +import com.yahoo.log.event.Started; +import com.yahoo.log.event.Starting; +import com.yahoo.log.event.Stopped; +import com.yahoo.log.event.Stopping; +import com.yahoo.log.event.Unknown; +import com.yahoo.log.event.Value; + +public class EventTestCase extends junit.framework.TestCase { + Count countEvent; + Count floatCountEvent; + Value valueEvent; + Value floatValueEvent; + Crash crashEvent; + Progress progressEvent1; + Progress progressEvent2; + Reloading reloadingEvent; + Reloaded reloadedEvent; + Starting startingEvent; + Started startedEvent; + Stopping stoppingEvent; + Stopped stoppedEvent; + Collection collectionEvent; + Unknown unknownEvent; + + public EventTestCase (String name) { + super(name); + } + + public void setUp() { + countEvent = new Count("thecounter", 1234); + floatCountEvent= new Count("thecounter", 1234.23); + valueEvent = new Value("thevalue", 4566); + floatValueEvent= new Value("thevalue", 4566.23); + crashEvent = new Crash("appname", 1234, 11); + progressEvent1 = new Progress("thename", 1); + progressEvent2 = new Progress("thename", 1.0, 2.0); + reloadingEvent = new Reloading("thefilewereloading"); + reloadedEvent = new Reloaded("thefilewereloaded"); + startingEvent = new Starting("startingName"); + startedEvent = new Started("startedName"); + stoppingEvent = new Stopping("stoppingName", "because we want to"); + stoppedEvent = new Stopped("stoppedName", 1234, 1); + collectionEvent=new Collection(123456, "thename"); + unknownEvent = new Unknown(); + } + + // make sure we can make the test instances okay + public void testExists () { + assertNotNull(countEvent); + assertNotNull(floatCountEvent); + assertNotNull(valueEvent); + assertNotNull(floatValueEvent); + assertNotNull(crashEvent); + assertNotNull(progressEvent1); + assertNotNull(progressEvent2); + assertNotNull(reloadingEvent); + assertNotNull(reloadedEvent); + assertNotNull(startingEvent); + assertNotNull(startedEvent); + assertNotNull(stoppingEvent); + assertNotNull(stoppedEvent); + assertNotNull(collectionEvent); + assertNotNull(unknownEvent); + } + + public void testEvents () { + assertEquals("count/1 name=thecounter value=1234", + countEvent.toString()); + + assertEquals("count/1 name=thecounter value=1234", + floatCountEvent.toString()); + + assertEquals("crash/1 name=appname pid=1234 signal=11", + crashEvent.toString()); + + assertEquals("progress/1 name=thename value=1.0 total=", + progressEvent1.toString()); + + assertEquals("progress/1 name=thename value=1.0 total=2.0", + progressEvent2.toString()); + + assertEquals("reloaded/1 name=thefilewereloaded", + reloadedEvent.toString()); + + assertEquals("reloading/1 name=thefilewereloading", + reloadingEvent.toString()); + + assertEquals("started/1 name=startedName", + startedEvent.toString()); + + assertEquals("starting/1 name=startingName", + startingEvent.toString()); + + assertEquals("stopping/1 name=stoppingName why=\"because we want to\"", + stoppingEvent.toString()); + + assertEquals("collection/1 collectionId=123456 name=thename", + collectionEvent.toString()); + + assertEquals("stopped/1 name=stoppedName pid=1234 exitcode=1", + stoppedEvent.toString()); + } + + /** + * do the dirty work for testParser + */ + private void parseEvent (Event e) throws MalformedEventException { + assertEquals(e.toString(), Event.parse(e.toString()).toString()); + } + + public void testParser () throws MalformedEventException { + parseEvent(countEvent); + parseEvent(floatCountEvent); + parseEvent(crashEvent); + parseEvent(progressEvent1); + parseEvent(progressEvent2); + parseEvent(reloadingEvent); + parseEvent(reloadedEvent); + parseEvent(startingEvent); + parseEvent(startedEvent); + parseEvent(stoppingEvent); + parseEvent(stoppedEvent); + } + + public void testUnknownEvents () throws MalformedEventException { + String s = "crappyevent/2 first=\"value one\" second=more third=4"; + Event event = Event.parse(s); + assertEquals(s, event.toString()); + + String s2 = "notinventedhere/999"; + assertEquals(s2, Event.parse(s2).toString()); + } + + /** + * This test makes sure that the static event logging methods + * successfully manage to find the right name on the calling + * stack -- it should find the name of the calling class. + */ + public void testFindingCallingClassLogger() { + SingleHandler sh = new SingleHandler(); + assertNull(sh.lastRecord()); + + VespaFormatter formatter = new VespaFormatter(); + Logger log = Logger.getLogger(EventTestCase.class.getName()); + synchronized(log) { + log.setUseParentHandlers(false); + log.addHandler(sh); + Event.starting("mintest"); + + assertTrue(formatter.format(sh.lastRecord()). + indexOf("\t.com.yahoo.log.event.EventTestCase\tevent\tstarting/1 name=mintest") > -1); + + Event.starting("startingName"); + Event.started("startedName"); + Event.stopping("stoppingName", "whyParam"); + Event.stopped("stoppedName", 1, 2); + Event.reloading("reloadingName"); + Event.reloaded("reloadedName"); + Event.count("countName", 1); + Event.progress("progressName", 1, 2); + Event.crash("crashName", 1, 2); + } + } + + public void testFunnyEvent () { + String funnyEvent = "collection/1 collectionId=1111111111 name=\"syncrows\" params=\"column=0 badrow=1 goodrow=0\""; + try { + Event e = Event.parse(funnyEvent); + } + catch (MalformedEventException e) { + fail(); + } + } + + public void testFullParse () { + try { + Event event = Event.parse("count/1 name=\"data_searched_mb\" value=15115168.3940149993"); + assertTrue(event instanceof Count); + assertEquals("data_searched_mb", event.getValue("name")); + assertEquals("15115168", event.getValue("value")); + } catch (MalformedEventException e) { + fail("Malformed Event Exception on parsing"); + } + } +} diff --git a/vespalog/src/test/java/com/yahoo/log/event/SingleHandler.java b/vespalog/src/test/java/com/yahoo/log/event/SingleHandler.java new file mode 100644 index 00000000000..47e342c2523 --- /dev/null +++ b/vespalog/src/test/java/com/yahoo/log/event/SingleHandler.java @@ -0,0 +1,20 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.log.event; + +import java.util.logging.Handler; +import java.util.logging.LogRecord; + +public class SingleHandler extends Handler { + private LogRecord lastRecord; + + public void flush() {} + public void close() {} + + public void publish (LogRecord r) { + lastRecord = r; + } + + public LogRecord lastRecord () { + return lastRecord; + } +} diff --git a/vespalog/src/test/logtest.cpp b/vespalog/src/test/logtest.cpp new file mode 100644 index 00000000000..29902e418cc --- /dev/null +++ b/vespalog/src/test/logtest.cpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <stdlib.h> +#include <unistd.h> +#include <signal.h> + +#include <vespa/log/log.h> + +LOG_SETUP("logtest", + "$Id$"); + +int +main(int, char **argv) +{ + EV_STARTING("logtest"); + LOG(info, "Starting up, called as %s", argv[0]); + EV_STARTED("logtest"); + EV_RELOADING("something"); + EV_RELOADED("something"); + EV_CRASH("something", getpid(), SIGTERM); + EV_PROGRESS("batch-index", 7, 100); + EV_PROGRESS("unbounded-batch-index", 9); + EV_COUNT("hits", 3); + EV_VALUE("some value", 1./3); + + LOG(info, "backslash: \\"); + int n; + LOG(info, "Will log 20 spam messages now every 500ms"); + for (n = 1; n <= 20; n++) { + LOG(spam, "log message %d/%d", n, 100); + usleep(500000); + } + LOG(info, "Exiting."); + EV_STOPPING("logtest", "clean exit"); + EV_STOPPED("logtest", getpid(), 0); + exit(EXIT_SUCCESS); +} diff --git a/vespalog/src/test/rejectfiltertest.cpp b/vespalog/src/test/rejectfiltertest.cpp new file mode 100644 index 00000000000..a11c25e58e1 --- /dev/null +++ b/vespalog/src/test/rejectfiltertest.cpp @@ -0,0 +1,57 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/reject-filter.h> + +#include <sys/types.h> +#include <stdlib.h> +#include <unistd.h> +#include <iostream> + +using ns_log::RejectFilter; +using ns_log::Logger; + +void +assertShouldNotReject(RejectFilter & filter, Logger::LogLevel level, const char * msg) +{ + std::cerr << "Filter should not reject level '" << Logger::levelName(level) << "' message '" << (msg == NULL ? "NULL" : msg) << "' ...: "; + if (filter.shouldReject(level, msg)) { + std::cerr << "Failed!\n"; + exit(EXIT_FAILURE); + } + std::cerr << "Success!\n"; +} + +void +assertShouldReject(RejectFilter & filter, Logger::LogLevel level, const char * msg) +{ + std::cerr << "Filter should reject level '" << Logger::levelName(level) << "' message '" << (msg == NULL ? "NULL" : msg) << "' ...: "; + if (!filter.shouldReject(level, msg)) { + std::cerr << "Failed!\n"; + exit(EXIT_FAILURE); + } + std::cerr << "Success!\n"; +} + +int +main(int argc, char **argv) +{ + (void) argc; + (void) argv; + + ns_log::RejectFilter filter; + filter.addRejectRule(Logger::warning, "bar"); + assertShouldNotReject(filter, Logger::warning, NULL); + assertShouldNotReject(filter, Logger::warning, ""); + assertShouldNotReject(filter, Logger::warning, "foo"); + assertShouldReject(filter, Logger::warning, "bar"); + assertShouldReject(filter, Logger::warning, "barfoo"); + assertShouldReject(filter, Logger::warning, "foobar"); + assertShouldReject(filter, Logger::warning, "foobarbaz"); + + ns_log::RejectFilter defaultFilter = RejectFilter::createDefaultFilter(); + assertShouldReject(defaultFilter, Logger::warning, "E 23-235018.067240 14650 23/10/2012 23:50:18 yjava_preload.so: [preload.c:350] Using FILTER_NONE: This must be paranoid approved, and since you are using FILTER_NONE you must live with this error."); + assertShouldReject(defaultFilter, Logger::warning, ""); + assertShouldNotReject(defaultFilter, Logger::warning, "foobar"); + assertShouldNotReject(defaultFilter, Logger::event, NULL); + assertShouldReject(defaultFilter, Logger::warning, "E 18-140313.398540 10727 18/11/2012 14:03:13 yjava_preload.so: [preload.c:670] Accept failed: -1 (4)"); + exit(EXIT_SUCCESS); +} diff --git a/vespalog/src/test/simple/.gitignore b/vespalog/src/test/simple/.gitignore new file mode 100644 index 00000000000..d3c5aa01c49 --- /dev/null +++ b/vespalog/src/test/simple/.gitignore @@ -0,0 +1,4 @@ +/simple.lctl +/simple.logf +/simple_test +vespalog_simple_test_app diff --git a/vespalog/src/test/simple/CMakeLists.txt b/vespalog/src/test/simple/CMakeLists.txt new file mode 100644 index 00000000000..f915772d80f --- /dev/null +++ b/vespalog/src/test/simple/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalog_simple_test_app + SOURCES + logtest.cpp + DEPENDS + vespalog +) +vespa_add_test(NAME vespalog_simple_test_app COMMAND vespalog_simple_test_app) diff --git a/vespalog/src/test/simple/expect.lctl b/vespalog/src/test/simple/expect.lctl new file mode 100644 index 00000000000..5256fbb2821 --- /dev/null +++ b/vespalog/src/test/simple/expect.lctl @@ -0,0 +1,5 @@ +Vespa log control file version 1 +Prefix: logtest + +default: ON ON ON ON ON ON OFF OFF +.log: ON ON ON ON ON ON OFF OFF diff --git a/vespalog/src/test/simple/logtest.cpp b/vespalog/src/test/simple/logtest.cpp new file mode 100644 index 00000000000..b532b2a4b8f --- /dev/null +++ b/vespalog/src/test/simple/logtest.cpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <stdlib.h> +#include <unistd.h> +#include <signal.h> + +#include <vespa/log/log.h> + +LOG_SETUP("logtest", + "$Id: logtest.cpp 119439 2011-04-19 09:32:27Z arnej $"); + +int +main(int, char **argv) +{ + EV_STARTING("logtest"); + LOG(info, "Starting up, called as %s", argv[0]); + EV_STARTED("logtest"); + EV_RELOADING("something"); + EV_RELOADED("something"); + EV_CRASH("something", getpid(), SIGTERM); + EV_PROGRESS("batch-index", 7, 100); + EV_PROGRESS("unbounded-batch-index", 9); + EV_COUNT("hits", 3); + EV_VALUE("some value", 1./3); + + LOG(info, "backslash: \\"); + int n; + LOG(info, "Will log 20 spam messages now every 50ms"); + for (n = 1; n <= 5; n++) { + LOG(spam, "log message %d/%d", n, 100); + usleep(50000); + } + LOG(info, "Exiting."); + EV_STOPPING("logtest", "clean exit"); + EV_STOPPED("logtest", getpid(), 0); + exit(EXIT_SUCCESS); +} diff --git a/vespalog/src/test/threads/.gitignore b/vespalog/src/test/threads/.gitignore new file mode 100644 index 00000000000..238de5943e7 --- /dev/null +++ b/vespalog/src/test/threads/.gitignore @@ -0,0 +1,2 @@ +/threads_test +vespalog_threads_test_app diff --git a/vespalog/src/test/threads/CMakeLists.txt b/vespalog/src/test/threads/CMakeLists.txt new file mode 100644 index 00000000000..77928ed3213 --- /dev/null +++ b/vespalog/src/test/threads/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalog_threads_test_app + SOURCES + testthreads.cpp + DEPENDS + vespalog +) +vespa_add_test(NAME vespalog_threads_test_app COMMAND vespalog_threads_test_app vespa.log ENVIRONMENT "VESPA_LOG_TARGET=file:vespa.log") diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp new file mode 100644 index 00000000000..6e5f72da524 --- /dev/null +++ b/vespalog/src/test/threads/testthreads.cpp @@ -0,0 +1,139 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <vespa/log/log.h> +#include <iostream> +#include <string> + +using std::string; + + +LOG_SETUP(".threadtest"); + +class FileThread : public FastOS_Runnable +{ + bool _done; + string _file; +public: + FileThread(string file) : _done(false), _file(file) {} + virtual void Run(FastOS_ThreadInterface *thread, void *arg); + void stop() {_done = true; } +}; + +class LoggerThread : public FastOS_Runnable +{ + bool _done; +public: + bool _useLogBuffer; + LoggerThread() : _done(false), _useLogBuffer(false) {} + virtual void Run(FastOS_ThreadInterface *thread, void *arg); + void stop() {_done = true; } +}; + + + + +void +FileThread::Run(FastOS_ThreadInterface *, void *) +{ + unlink(_file.c_str()); + while (!_done) { + int fd = open(_file.c_str(), O_RDWR | O_CREAT | O_APPEND, 0644); + if (fd == -1) { + fprintf(stderr, "open failed: %s\n", strerror(errno)); + exit(1); + } + FastOS_Thread::Sleep(5); + struct stat buf; + fstat(fd, &buf); + if (buf.st_size != 0) { + fprintf(stderr, "%s isn't empty anymore\n", _file.c_str()); + exit(1); + } + if (close(fd) != 0) { + fprintf(stderr, "close of %d failed: %s\n", fd, strerror(errno)); + exit(1); + } + } +} + + +void +LoggerThread::Run(FastOS_ThreadInterface *, void *) +{ + int counter = 0; + while (!_done) { + if (_useLogBuffer) { + LOGBM(info, "bla bla bla %u", ++counter); + } else { + LOG(info, "bla bla bla"); + } + } +} + + +class ThreadTester : public FastOS_Application +{ +public: + int Main(); +}; + +int +ThreadTester::Main() +{ + std::cerr << "Testing that logging is threadsafe. 30 sec test.\n"; + FastOS_ThreadPool pool(128 * 1024); + + const int numWriters = 30; + const int numLoggers = 10; + + auto writers = std::array<std::unique_ptr<FileThread>, numWriters>(); + auto loggers = std::array<std::unique_ptr<LoggerThread>, numLoggers>(); + + for (int i = 0; i < numWriters; i++) { + char filename[100]; + sprintf(filename, "empty.%d", i); + writers[i] = std::make_unique<FileThread>(filename); + pool.NewThread(writers[i].get()); + } + for (int i = 0; i < numLoggers; i++) { + loggers[i] = std::make_unique<LoggerThread>(); + pool.NewThread(loggers[i].get()); + } + + FastOS_Time start; + start.SetNow(); + // Reduced runtime to half as the test now repeats itself to test with + // buffering. (To avoid test taking a minute) + while (start.MilliSecsToNow() < 15 * 1000) { + unlink(_argv[1]); + FastOS_Thread::Sleep(1); + } + // Then set to use logbuffer and continue + for (int i = 0; i < numLoggers; i++) { + loggers[i]->_useLogBuffer = true; + } + start.SetNow(); + while (start.MilliSecsToNow() < 15 * 1000) { + unlink(_argv[1]); + FastOS_Thread::Sleep(1); + } + + for (int i = 0; i < numLoggers; i++) { + loggers[i]->stop(); + } + for (int i = 0; i < numWriters; i++) { + writers[i]->stop(); + } + + pool.Close(); + + return 0; +} + +int main(int argc, char **argv) +{ + ThreadTester app; + return app.Entry(argc, argv); +} + + diff --git a/vespalog/src/vespa/log/.gitignore b/vespalog/src/vespa/log/.gitignore new file mode 100644 index 00000000000..7b45206e869 --- /dev/null +++ b/vespalog/src/vespa/log/.gitignore @@ -0,0 +1,8 @@ +*.So +*.so.* +.depend +Makefile +logctl +loglevelnames.cpp +logtest +logtest.logcontrol diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt new file mode 100644 index 00000000000..e36a55b1f8a --- /dev/null +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -0,0 +1,28 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalog + SOURCES + ${CMAKE_CURRENT_BINARY_DIR}/loglevelnames.cpp + log.cpp + bufferedlogger.cpp + log-target-fd.cpp + log-target-file.cpp + log-target.cpp + lock.cpp + internal.cpp + control-file.cpp + component.cpp + llparser.cpp + log-assert.cpp + reject-filter.cpp + INSTALL lib64 + DEPENDS + fastos +) + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/loglevelnames.cpp + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/mknm.pl < ${CMAKE_CURRENT_SOURCE_DIR}/log.h > lln.NEW && mv lln.NEW ${CMAKE_CURRENT_BINARY_DIR}/loglevelnames.cpp + MAIN_DEPENDENCY log.h + DEPENDENCIES mknm.pl +) + diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp new file mode 100644 index 00000000000..76fdcf1f66a --- /dev/null +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -0,0 +1,382 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/log/bufferedlogger.h> + +#include <iomanip> +#include <iostream> +#include <sstream> +#include <vector> +#include <stdarg.h> +#include <sys/time.h> + +#include <boost/multi_index_container.hpp> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/member.hpp> +#include <boost/multi_index/mem_fun.hpp> +#include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/sequenced_index.hpp> + +namespace ns_log { + +// implementation details for BufferedLogger +class BackingBuffer { + BackingBuffer(const BackingBuffer & rhs); + BackingBuffer & operator = (const BackingBuffer & rhs); +public: + std::unique_ptr<Timer> _timer; + /** Lock needed to access cache. */ + mutable FastOS_Mutex _mutex; + + static uint64_t _countFactor; + + /** Struct keeping information about log message. */ + struct Entry { + Logger::LogLevel _level; + std::string _file; + int _line; + std::string _token; + std::string _message; + uint32_t _count; + uint64_t _timestamp; + Logger* _logger; + + Entry(Logger::LogLevel level, const char* file, int line, + const std::string& token, const std::string& message, + uint64_t timestamp, Logger&); + + bool operator==(const Entry& entry) const; + bool operator<(const Entry& entry) const; + + uint64_t getAgeFactor() const; + + std::string toString() const; + }; + + typedef boost::multi_index_container< + Entry, + boost::multi_index::indexed_by< + boost::multi_index::sequenced<>, // Timestamp sorted + boost::multi_index::ordered_unique< + boost::multi_index::identity<Entry> + > + > + > LogCacheFront; + typedef boost::multi_index_container< + Entry, + boost::multi_index::indexed_by< + boost::multi_index::sequenced<>, // Timestamp sorted + boost::multi_index::ordered_unique< + boost::multi_index::identity<Entry> + >, + boost::multi_index::ordered_non_unique< + boost::multi_index::const_mem_fun< + Entry, uint64_t, &Entry::getAgeFactor + > + > + > + > LogCacheBack; + + /** Entry container indexes on insert order and token. */ + LogCacheFront _cacheFront; + /** Entry container indexed on insert order, token and age function. */ + LogCacheBack _cacheBack; + + uint32_t _maxCacheSize; + uint64_t _maxEntryAge; + + /** Empty buffer and write all log entries in it. */ + void flush(); + + /** Gives all current content of log buffer. Useful for debugging. */ + std::string toString() const; + + /** + * Flush parts of cache, so we're below max size and only have messages of + * acceptable age. Calling this, _mutex should already be locked. + */ + void trimCache(uint64_t currentTime); + + /** + * Trim the cache up to current time. Used externally to check if we + * need to empty buffer before new log messages arive. + */ + void trimCache() { + _mutex.Lock(); + trimCache(_timer->getTimestamp()); + _mutex.Unlock(); + } + + /** + * Log a given entry to underlying logger. Used when removing from cache. + * Calling this, _mutex should already be locked. + */ + void log(const Entry& e) const; + + BackingBuffer(); + ~BackingBuffer(); + + void logImpl(Logger& l, Logger::LogLevel level, + const char *file, int line, + const std::string& token, + const std::string& message); + +}; + +// Let each hit count for 5 seconds +uint64_t BackingBuffer::_countFactor = VESPA_LOG_COUNTAGEFACTOR * 1000 * 1000; + +BackingBuffer::Entry::Entry(Logger::LogLevel level, const char* file, int line, + const std::string& token, const std::string& msg, + uint64_t timestamp, Logger& l) + : _level(level), + _file(file), + _line(line), + _token(token), + _message(msg), + _count(1), + _timestamp(timestamp), + _logger(&l) +{ +} + +bool +BackingBuffer::Entry::operator==(const Entry& entry) const +{ + return (_token == entry._token); +} + +bool +BackingBuffer::Entry::operator<(const Entry& entry) const +{ + // Don't let tokens from different loggers match each other + if (_logger != entry._logger) { + return _logger < entry._logger; + } + // If in the same logger, you should have full control. Overlapping + // tokens if you want is a feature. + return (_token < entry._token); +} + +std::string +BackingBuffer::Entry::toString() const +{ + std::ostringstream ost; + ost << "Entry(" << _level << ", " << _file << ":" << _line << ": " + << _message << " [" << _token << "], count " << _count + << ", timestamp " << _timestamp << ")"; + return ost.str(); +} + +uint64_t +BackingBuffer::Entry::getAgeFactor() const +{ + return _timestamp + _countFactor * _count; +} + +BufferedLogger BufferedLogger::logger; + +BackingBuffer::BackingBuffer() + : _timer(new Timer), + _mutex(), + _cacheFront(), + _cacheBack(), + _maxCacheSize(VESPA_LOG_LOGBUFFERSIZE), + _maxEntryAge(VESPA_LOG_LOGENTRYMAXAGE * 1000 * 1000) +{ +} + +BackingBuffer::~BackingBuffer() +{ +} + +BufferedLogger::BufferedLogger() +{ + _backing = new BackingBuffer(); +} + +BufferedLogger::~BufferedLogger() +{ + delete _backing; _backing = NULL; +} + +namespace { + typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheFront, 0>::type LogCacheFrontTimestamp; + typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheFront, 1>::type LogCacheFrontToken; + typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 0>::type LogCacheBackTimestamp; + typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 1>::type LogCacheBackToken; + typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 2>::type LogCacheBackAge; +} + +void +BufferedLogger::doLog(Logger& l, Logger::LogLevel level, + const char *file, int line, + const std::string& mytoken, const char *fmt, ...) +{ + std::string token(mytoken); + va_list args; + va_start(args, fmt); + + const size_t sizeofPayload(4000); + std::vector<char> buffer(sizeofPayload); + vsnprintf(&buffer[0], buffer.capacity(), fmt, args); + std::string message(&buffer[0]); + // Empty token means to use message itself as token + if (token.empty()) token = message; + + _backing->logImpl(l, level, file, line, token, message); +} + +void +BackingBuffer::logImpl(Logger& l, Logger::LogLevel level, + const char *file, int line, + const std::string& token, + const std::string& message) +{ + Entry entry(level, file, line, token, message, _timer->getTimestamp(), l); + + _mutex.Lock(); + LogCacheFrontToken::iterator it1 = _cacheFront.get<1>().find(entry); + LogCacheBackToken::iterator it2 = _cacheBack.get<1>().find(entry); + if (it1 != _cacheFront.get<1>().end()) { + Entry copy(*it1); + ++copy._count; + _cacheFront.get<1>().replace(it1, copy); + } else if (it2 != _cacheBack.get<1>().end()) { + Entry copy(*it2); + ++copy._count; + _cacheBack.get<1>().replace(it2, copy); + } else { + // If entry didn't already exist, add it to the cache and log it + l.doLogCore(entry._timestamp, level, file, line, message.c_str(), message.size()); + _cacheFront.push_back(entry); + } + trimCache(entry._timestamp); + _mutex.Unlock(); +} + +void +BackingBuffer::flush() +{ + _mutex.Lock(); + for (LogCacheBack::const_iterator it = _cacheBack.begin(); + it != _cacheBack.end(); ++it) + { + log(*it); + } + _cacheBack.clear(); + for (LogCacheFront::const_iterator it = _cacheFront.begin(); + it != _cacheFront.end(); ++it) + { + log(*it); + } + _cacheFront.clear(); + _mutex.Unlock(); +} + +void +BufferedLogger::flush() { + _backing->flush(); +} + +void +BackingBuffer::trimCache(uint64_t currentTime) +{ + // Remove entries that have been in here too long. + while (!_cacheBack.empty() && + _cacheBack.front()._timestamp + _maxEntryAge < currentTime) + { + log(_cacheBack.front()); + _cacheBack.pop_front(); + } + while (!_cacheFront.empty() && + _cacheFront.front()._timestamp + _maxEntryAge < currentTime) + { + log(_cacheFront.front()); + _cacheFront.pop_front(); + } + // If cache front is larger than half max size, move to back. + for (uint32_t i = _cacheFront.size(); i > _maxCacheSize / 2; --i) { + Entry e(_cacheFront.front()); + _cacheFront.pop_front(); + _cacheBack.push_back(e); + } + // Remove entries from back based on count modified age. + for (uint32_t i = _cacheFront.size() + _cacheBack.size(); + i > _maxCacheSize; --i) + { + log(*_cacheBack.get<2>().begin()); + _cacheBack.get<2>().erase(_cacheBack.get<2>().begin()); + } +} + +void +BufferedLogger::trimCache() +{ + _backing->trimCache(); +} + +void +BackingBuffer::log(const Entry& e) const +{ + if (e._count > 1) { + std::ostringstream ost; + ost << e._message << " (Repeated " << (e._count - 1) + << " times since " << (e._timestamp / 1000000) << "." + << std::setw(6) << std::setfill('0') << (e._timestamp % 1000000) + << ")"; + e._logger->doLogCore(_timer->getTimestamp(), e._level, e._file.c_str(), + e._line, ost.str().c_str(), ost.str().size()); + } +} + +std::string +BackingBuffer::toString() const +{ + std::ostringstream ost; + ost << "Front log cache content:\n"; + _mutex.Lock(); + for (LogCacheFront::const_iterator it = _cacheFront.begin(); + it != _cacheFront.end(); ++it) + { + ost << " " << it->toString() << "\n"; + } + ost << "Back log cache content:\n"; + for (LogCacheBack::const_iterator it = _cacheBack.begin(); + it != _cacheBack.end(); ++it) + { + ost << " " << it->toString() << "\n"; + } + _mutex.Unlock(); + return ost.str(); +} + + +void +BufferedLogger::setMaxCacheSize(uint32_t size) { + _backing->_maxCacheSize = size; +} + +void +BufferedLogger::setMaxEntryAge(uint64_t seconds) { + _backing->_maxEntryAge = seconds * 1000000; +} + +void +BufferedLogger::setCountFactor(uint64_t factor) { + _backing->_countFactor = factor * 1000000; +} + +/** Set a fake timer to use for log messages. Used in unit testing. */ +void +BufferedLogger::setTimer(std::unique_ptr<Timer> timer) +{ + _backing->_timer = std::move(timer); +} + + +} // ns_log diff --git a/vespalog/src/vespa/log/bufferedlogger.h b/vespalog/src/vespa/log/bufferedlogger.h new file mode 100644 index 00000000000..4f20f46af4c --- /dev/null +++ b/vespalog/src/vespa/log/bufferedlogger.h @@ -0,0 +1,206 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * \class ns_log::BufferedLogger + * \ingroup log + * + * \brief Utility class for writing log less spammy. + * + * Uses string tokens to identify what messages are equal. Equal tokens means + * similar message. Use tokens similar to the messages, excluding the variable + * parts you don't want to mess up equality. + * + * Keeps a cache of log entries. When something is logged, check if the message + * is already in the log cache. If so, increase the count of the log entry. If + * it wasn't in the cache, log it to file and add it to the log cache. + * + * Trim the log cache if it is above maximum size or contains entries that are + * older than the max age they can live. + * + * There are no separate threads polling buffer to see if log entries are older + * than max age. Each doLog() call to the logbuffer will trigger a check. + * + * The log cache is a single static object, common for all loggers. This has + * the following advantages: + * - Only one instance to trigger to check for old entries. + * - We can keep a bigger cache, preventing issues with allocating large + * caches to lots of loggers or having to small of a cache for heavily used + * loggers. + * The disadvantage is that the cache itself needs to be able to use little + * resources even though there's many entries in the cache. + * + * For the log cache to be efficient, even if it gets large, it is indexed 3 + * ways. This means that there's no operation (except flushing) that needs to + * go through all log entries. Currently, ordered indexes accessed through + * binary search is used to give log N complexity. + * + * - We need to look up entries by token to figure out if entry already exist. + * - We need to get oldest entries to see whether they are older than max age. + * - We need to identify entry to remove when cache is too large. + * + * Deciding on what entry to remove when the cache is too large is not straight + * forward. We don't want a burst of unique log entries to clear the whole + * buffer. We also need to make sure that the low count entries aren't starved + * out by the higher count entries, such that no new high count entries can + * come in. + * + * Suggested algorithm for deciding what entry to remove next: + * - Split the cache in two by timestamp. The first half of the cache + * contains the newest entries added. When entries are added, the last + * entry in the first half is moved into the last half (if first half is + * full). This ensures that new entries are never starved, and will always + * be able to build up their counts. + * - In the second half, when an entry needs to be removed, calculate each + * entry's timestamp + X * count, and remove the entry with the lowest + * score. This will let higher count entries live longer, but + * will prevent them from living forever. X is the age factor, saying + * how much each count counts for. For instance, if X counts for 5 seconds, + * this means that a log entry that you will rather remove an entry seen + * once 10 seconds ago, than an entry seen twice less than 15 seconds ago. + * + * For efficient access of the log buffer itself, boost' multi_index is used. + * This is a container that keeps separate indexes. In the log buffer case it + * will keep 3 indexes. + * - Index 0 will be the insert order. This will also be the timestamp order + * of the log entries, such that new entries goes in the back, and if we + * need to remove an entry due to old age, it will always be the first. + * - Index 1 will be the token. This is used for efficient lookup to find + * whether we have seen this entry before. + * - Index 2 will be the timestamp + X * count, used to find which entry to + * remove next if the buffer is too full. Note that this index is only + * needed in the second half of the buffer. + * + * The buffer itself is split into two parts, because this was the only way I + * could think of to keep it log N, as the sequenced index don't have random + * access, such that finding the middle element has N complexity, also, the + * first half of the entries would pollute the timestamp + X * count index, such + * that one potentially would have to check many entries before finding the + * correct one to remove. + */ +#pragma once + +#define VESPA_LOG_LOGBUFFERSIZE 1000 // Max log entries cached. +#define VESPA_LOG_LOGENTRYMAXAGE 300 // Max seconds an entry can be cached +#define VESPA_LOG_COUNTAGEFACTOR 5 // How many seconds each count counts for + +#include <vespa/fastos/mutex.h> +#include <vespa/log/log.h> +#include <sstream> +#include <string> + +// Define LOG if using log buffer for regular LOG calls +// If logger logs debug messages, bypass buffer. +#ifdef VESPA_LOG_USELOGBUFFERFORREGULARLOG +#define LOG(level, ...) \ + do { \ + if (logger.wants(ns_log::Logger::level)) { \ + if (logger.wants(ns_log::Logger::debug)) { \ + logger.doLog(ns_log::Logger::level, \ + __FILE__, __LINE__, __VA_ARGS__); \ + ns_log::BufferedLogger::logger.trimCache(); \ + } else { \ + ns_log::BufferedLogger::logger.doLog(logger, \ + ns_log::Logger::level, __FILE__, __LINE__, \ + "", __VA_ARGS__); \ + } \ + } \ + } while (0) +#endif + +// Define LOGBM macro for logging buffered, using the message itself as a +// token. This is the same as LOG defined above if +// VESPA_LOG_USELOGBUFFERFORREGULARLOG is defined. +#define LOGBM(level, ...) \ + do { \ + if (logger.wants(ns_log::Logger::level)) { \ + if (logger.wants(ns_log::Logger::debug)) { \ + logger.doLog(ns_log::Logger::level, \ + __FILE__, __LINE__, __VA_ARGS__); \ + ns_log::BufferedLogger::logger.trimCache(); \ + } else { \ + ns_log::BufferedLogger::logger.doLog(logger, \ + ns_log::Logger::level, __FILE__, __LINE__, \ + "", __VA_ARGS__); \ + } \ + } \ + } while (0) + +// Define LOGBP macro for logging buffered, using the call point as token. +// (File/line of macro caller) +#define LOGBP(level, ARGS...) \ + do { \ + if (logger.wants(ns_log::Logger::level)) { \ + if (logger.wants(ns_log::Logger::debug)) { \ + logger.doLog(ns_log::Logger::level, \ + __FILE__, __LINE__, ##ARGS); \ + ns_log::BufferedLogger::logger.trimCache(); \ + } else { \ + std::ostringstream ost123; \ + ost123 << __FILE__ << ":" << __LINE__; \ + ns_log::BufferedLogger::logger.doLog(logger, \ + ns_log::Logger::level, \ + __FILE__, __LINE__, ost123.str(), ##ARGS); \ + } \ + } \ + } while (0) + +// Define LOGT calls for using the buffer specifically stating token +#define LOGBT(level, token, ...) \ + do { \ + if (logger.wants(ns_log::Logger::level)) { \ + if (logger.wants(ns_log::Logger::debug)) { \ + logger.doLog(ns_log::Logger::level, \ + __FILE__, __LINE__, __VA_ARGS__); \ + ns_log::BufferedLogger::logger.trimCache(); \ + } else { \ + ns_log::BufferedLogger::logger.doLog(logger, \ + ns_log::Logger::level, \ + __FILE__, __LINE__, token, __VA_ARGS__); \ + } \ + } \ + } while (0) + +#define LOGB_FLUSH() \ + ns_log::BufferedLogger::logger.flush() + +namespace ns_log { + +class BackingBuffer; + +class BufferedLogger { + BackingBuffer *_backing; + BufferedLogger(const BufferedLogger & buf); + BufferedLogger & operator = (const BufferedLogger & buf); + +public: + static BufferedLogger logger; + + BufferedLogger(); + ~BufferedLogger(); + + // Not intended to be used within real applications. Unit tests use these, + // to easier be able to test all aspects of the buffer, and be independent + // of the default settings for applications + void setMaxCacheSize(uint32_t size); + void setMaxEntryAge(uint64_t seconds); + void setCountFactor(uint64_t factor); + + void doLog(Logger&, Logger::LogLevel level, const char *file, int line, + const std::string& token, + const char *fmt, ...) __attribute__((format(printf,7,8))); + + /** Empty buffer and write all log entries in it. */ + void flush(); + + /** Gives all current content of log buffer. Useful for debugging. */ + std::string toString() const; + + /** Set a fake timer to use for log messages. Used in unit testing. */ + void setTimer(std::unique_ptr<Timer> timer); + + /** Trim the buffer. Removing old messages if wanted. */ + void trimCache(); + +}; + +} // ns_log + diff --git a/vespalog/src/vespa/log/component.cpp b/vespalog/src/vespa/log/component.cpp new file mode 100644 index 00000000000..5582366ef8d --- /dev/null +++ b/vespalog/src/vespa/log/component.cpp @@ -0,0 +1,121 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <cstdio> + +#include "log.h" +LOG_SETUP_INDIRECT(".log.control", "$Id$"); +#undef LOG +#define LOG LOG_INDIRECT + +#include "component.h" +#include "control-file.h" +#include "internal.h" + +namespace ns_log { + +bool +Component::matches(const char *pattern) +{ + // Return true if pattern matches the name of this component, + // false otherwise. + int pLen = strlen(pattern); + bool matched; + + if (strcmp(pattern, "default") == 0) { + return true; + } + + if (strcmp(pattern, ".") == 0) { + return matches("default."); + } + + if (pattern[pLen - 1] == '.') { + LOG(spam, "Component::matches -- exact match of '%s' vs name '%s'", + pattern, _name); + matched = (strncmp(pattern, _name, pLen - 1) == 0) + && _name[pLen - 1] == ':'; + } else { + LOG(spam, "Component::matches -- prefix match of '%s' vs name '%s'", + pattern, _name); + matched = strncmp(pattern, _name, pLen) == 0 + && (_name[pLen] == '.' || _name[pLen] == ':'); + } + LOG(spam, "Component::matches: Pattern '%s' %s match name '%s'", + pattern, matched ? "did" : "did not", _name); + return matched; +} + +void +Component::modifyLevels(char *levels) +{ + // levels is a comma-separated list of level={on|off} pairs. + + // the levels string can always be converted to a + // AND bitmask -- for all levels to be removed + // and an OR bitmask -- for all levels to be added + char *s = levels; + + LOG(spam, "Will modify levels for '%.*s' according to \"%s\"", + (int)strcspn(_name, " :\n"), _name, levels); + + while (s && *s) { + char *eq = strchr(s, '='); + if (!eq) { + throwInvalid("Missing \"=\" in levels string at \"%s\"", s); + } + *eq = 0; + Logger::LogLevel level = Logger::parseLevel(s); + if (level == Logger::NUM_LOGLEVELS) { + if (strcmp(s, "all") != 0) { + throwInvalid("Level name at \"%s\" is not valid", s); + } + } + + *eq = '='; + char *mod = eq + 1; + unsigned int newValue; + + if (strcmp(mod, "on") == 0 || strncmp(mod, "on,", 3) == 0) { + s = mod + 2; + newValue = CHARS_TO_UINT(' ', ' ', 'O', 'N'); + } else if (strcmp(mod, "off") == 0 || strncmp(mod, "off,", 4) == 0) { + s = mod + 3; + newValue = CHARS_TO_UINT(' ', 'O', 'F', 'F'); + } else { + throwInvalid("Invalid modification string at \"%s\", expected " + "\"on\" or \"off\"", mod); + } + if (*s == ',') { + ++s; + } + + if (level != Logger::NUM_LOGLEVELS) { + _intLevels[level] = newValue; + } else { + for (int n = 0; n != Logger::NUM_LOGLEVELS; ++n) { + _intLevels[n] = newValue; + } + } + } + display(); +} + +void +Component::display() +{ + int nlen = strcspn(_name, ":\n "); + printf("%-30.*s %.*s\n", nlen, _name, + (int)(sizeof(unsigned int) * Logger::NUM_LOGLEVELS), _charLevels); +} + +Component::Component(char *s) + : _name(s), + _charLevels(ControlFile::alignLevels(strchr(s, ':') + 2)), + _intLevels(reinterpret_cast<unsigned int *>(_charLevels)) +{ + if (_charLevels == reinterpret_cast<char *>(4)) { + throwInvalid("Invalid component instantiated"); + } + +} + +} // end namesmace ns_log diff --git a/vespalog/src/vespa/log/component.h b/vespalog/src/vespa/log/component.h new file mode 100644 index 00000000000..9dd364af456 --- /dev/null +++ b/vespalog/src/vespa/log/component.h @@ -0,0 +1,22 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace ns_log { + +class Component { + Component(); + Component& operator = (const Component &); + Component(const Component &); + char *_name; + char *_charLevels; + unsigned int *_intLevels; + +public: + bool matches(const char *pattern); + void modifyLevels(char *levels); + void display(); + explicit Component(char *); +}; + +} // end namespace ns_log + diff --git a/vespalog/src/vespa/log/control-file.cpp b/vespalog/src/vespa/log/control-file.cpp new file mode 100644 index 00000000000..3b9fe2834a9 --- /dev/null +++ b/vespalog/src/vespa/log/control-file.cpp @@ -0,0 +1,424 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <cstdlib> +#include <cstring> +#include <ctype.h> +#include <cstdio> +#include <sys/mman.h> +#include <sys/stat.h> +#include <errno.h> +#include <unistd.h> +#include <memory> + +#include "log.h" +LOG_SETUP_INDIRECT(".log.control", "$Id$"); +#undef LOG +#define LOG LOG_INDIRECT + +#include "control-file.h" +#include "internal.h" +#include "component.h" + +namespace ns_log { + +ControlFile::ControlFile(const char *file, Mode mode) + : _fileBacking(file, + O_NOCTTY | ( (mode == READONLY) ? O_RDONLY + : (mode == READWRITE) ? O_RDWR + : (O_RDWR | O_CREAT))), + _mode(mode), + _fileName(strdup(file)), + _prefix(0), + _mapBase(0), + _mappedSize(0), + _firstComponent(0) +{ + _fileBacking.lock(mode != READONLY); + ensureHeader(); // Create the header if it doesn't exist + ensureMapping(); // mmap the file + _fileBacking.unlock(); +} + +ControlFile::~ControlFile() +{ + freeMapping(); + free(_fileName); +} + +void +ControlFile::ensureHeader() +{ + // Make sure the file has a valid header. If it doesn't, or is + // empty, truncate it and write a new fresh header. + static const char fileHeader[] = "Vespa log control file version 1\n"; + const int fd = _fileBacking.fd(); + char buf[sizeof fileHeader]; + + int wantsLen = strlen(fileHeader); + int len = read(fd, &buf, wantsLen); + if (len != wantsLen || memcmp(fileHeader, buf, wantsLen) != 0) { + if (len) { + ftruncate(fd, 0); + } + lseek(fd, 0, SEEK_SET); + write(fd, fileHeader, strlen(fileHeader)); + + char spaces[_maxPrefix + 1]; + memset(spaces, ' ', sizeof spaces); + spaces[sizeof(spaces) - 1] = '\0'; + + char buf2[sizeof(spaces) + 100]; + snprintf(buf2, sizeof buf2, "Prefix: \n%s\n", spaces); + write(fd, buf2, strlen(buf2)); + } +} + +void +ControlFile::flush() +{ + if (_mapBase != NULL) { + if (msync(_mapBase, 0, MS_SYNC) != 0) { + LOG(warning, "msync of log control file failed: %s", + strerror(errno)); + } + } +} + +void +ControlFile::ensureMapping() +{ + // If the file is not mmaped yet, first do a huge anonymous mmap + // so that we never have to change the address, then mmap the + // start of the file. + if (!_mapBase) { + int prot = PROT_READ; + int flags = MAP_PRIVATE | MAP_ANON; + int fd = -1; + size_t length = pageAlign(_maxMapSize + 1); + + void *addr = mmap(NULL, length, prot, flags, fd, 0); + if (!addr) { + throwInvalid("Failed to get anonymous memory for control file: %s", + strerror(errno)); + } + _mapBase = static_cast<char *>(addr); + extendMapping(); + + char *s = strstr(_mapBase, "Prefix: "); + if (!s) { + throwInvalid("Bad format of mapped file. bleh."); + } + _prefix = s + strlen("Prefix: "); + _firstComponent = _mapBase + _maxPrefix + 25; + _firstComponent = strchr(_firstComponent, '\n') + 1; + } +} + +void +ControlFile::freeMapping() +{ + // If the file is mmapped, release all resource used + if (_mapBase && (munmap(_mapBase, pageAlign(_maxMapSize + 1)) < 0)) { + LOG(warning, "munmapping of loglevel settings failed: %s", + strerror(errno)); + } + _mapBase = NULL; +} + +unsigned int +ControlFile::pageAlign(unsigned int len) +{ + static const int pageMask = getpagesize() - 1; + return (len + pageMask) & ~pageMask; +} + +char * +ControlFile::alignLevels(char *addr) +{ + unsigned long x = reinterpret_cast<unsigned long>(addr); + x = (x + 3) & ~3; + return reinterpret_cast<char *>(x); +} + +bool +ControlFile::extendMapping() +{ + int fileLen = _fileBacking.size(); + + if (fileLen == -1) { + _fileBacking.unlock(); + LOG(error, "Cannot get file size of '%s': %s", _fileName, + strerror(errno)); + return false; + } + + if (fileLen >= _maxMapSize) { + _fileBacking.unlock(); + LOG(error, "Log control file is too big at %d bytes (max " + "size is %d). Ignoring it for further log components.", + fileLen, _maxMapSize - 1); + return false; + } + + off_t size = pageAlign(fileLen); + + int prot = PROT_READ | (_mode == READONLY ? 0 : PROT_WRITE); + int flags = MAP_FIXED | MAP_SHARED; + int fd = _fileBacking.fd(); + if (mmap(_mapBase, size, prot, flags, fd, 0) != _mapBase) { + _fileBacking.unlock(); + _mappedSize = -1; + LOG(error, "failed to mmap lock file: %s", strerror(errno)); + return false; + } + _mappedSize = size; + + return true; +} + +void +ControlFile::setPrefix(const char *prefix) +{ + if (prefix && !hasPrefix() && _prefix) { + char buf[_maxPrefix + 1]; + sprintf(buf, "%.*s\n", _maxPrefix - 1, prefix); + memcpy(_prefix, buf, strlen(buf)); + msync(_mapBase, pageAlign(1), MS_ASYNC | MS_INVALIDATE); + } +} + + +unsigned int * +ControlFile::getLevels(const char *name) +{ + // these comments are all wrong... + + // See if there already is some info for name. If not, + // lock the file, stat it, remap if necessary, scan again, + // and if still not there, append to it. Return pointer to mapped area. + + // ### call getComponent() instead, + // ### if it does not exist, create it and return the thing + // ### get default value from pre-existing levels, or from + // ### the default level if nothing exists. + // ### (if default level does not exist, create it?) + + _fileBacking.lock(_mode != READONLY); + static const char *padSpaces = " "; // 3 spaces + char buf[2000]; + if (strcmp(name, "") == 0) { + name = "default"; + } + // Leave space for 3 spaces and a full levels string (200 bytes should + // be enough) + snprintf(buf, sizeof buf - 200, "\n%s: ", name); + + char *levels = strstr(_mapBase, buf); + if (levels) { + _fileBacking.unlock(); + char *addr = levels + strlen(buf); + addr = alignLevels(addr); + return reinterpret_cast<unsigned int *>(addr); + } + + char *inheritLevels = reinterpret_cast<char *>(defaultLevels()); + const char *chop = strrchr(name, '.'); + if (chop != NULL) { + char shorterName[2000]; + strncpy(shorterName, name, chop - name); + shorterName[chop-name] = '\0'; + unsigned int *inherit = getLevels(shorterName); + if (inherit != NULL) { + inheritLevels = reinterpret_cast<char *>(inherit); + } + } + + // Append whatever is in buf, excluding the initial newline, and + // up to 3 more spaces to get the entire file length to be aligned. + int fileLength = _fileBacking.size(); + char *appendedString = buf + 1; + int newLength = fileLength + strlen(appendedString); + unsigned int padding = static_cast<unsigned int>(-newLength) & 3u; + strcat(appendedString, &padSpaces[3 - padding]); + + char *baseAddr = _mapBase + fileLength + strlen(appendedString); + + strncat(appendedString, inheritLevels, Logger::NUM_LOGLEVELS*sizeof(int)); + strcat(appendedString, "\n"); + + int len = strlen(appendedString); + + int fd = _fileBacking.fd(); + lseek(fd, (off_t)fileLength, SEEK_SET); + int wlen = write(_fileBacking.fd(), appendedString, len); + if (wlen != len) { + _fileBacking.unlock(); + LOG(error, "Writing to control file '%s' fails (%d/%d bytes): %s", + _fileName, wlen, len, strerror(errno)); + return reinterpret_cast<unsigned int *>(inheritLevels); + } else { + } + + if (fileLength + wlen > _mappedSize) { + if (!extendMapping()) { + _fileBacking.unlock(); // just for sure + LOG(error, "Failed to extend mapping of '%s', losing runtime " + "configurability of component '%s'", _fileName, name); + return defaultLevels(); + } + } + + _fileBacking.unlock(); + + return reinterpret_cast<unsigned int *>(baseAddr); +} + + +unsigned int * +ControlFile::defaultLevels() +{ + static unsigned int levels[Logger::NUM_LOGLEVELS + 1]; + if (levels[0] == 0) { + const char *env = getenv("VESPA_LOG_LEVEL"); + if (!env) { + env = "all -debug -spam"; + } + makeLogLevelArray(levels, sizeof levels, env); + levels[Logger::NUM_LOGLEVELS] = 0; + } + return levels; +} + + +unsigned int +ControlFile::findOnOffStatus(Logger::LogLevel level, const char *levelsString) +{ + const char *name = Logger::levelName(level); + if (hasWord(name, levelsString) + || (!hasNegWord(name, levelsString) + && hasWord("all", levelsString))) { + return CHARS_TO_UINT(' ', ' ', 'O', 'N'); + } else { + return CHARS_TO_UINT(' ', 'O', 'F', 'F'); + } +} + +void +ControlFile::makeLogLevelArray(unsigned int *levels, unsigned int size, + const char *env) +{ + int n; + for (n = 0; n < Logger::NUM_LOGLEVELS + && (n * sizeof levels[0] < size); ++n) { + levels[n] = findOnOffStatus(static_cast<Logger::LogLevel>(n), env); + } +} + +bool +ControlFile::hasWord(const char *word, const char *haystack) +{ + int len = strlen(word); + const char *start = strstr(haystack, word); + const char *end = start + len; + if (!start) + return false; + + return ((start == haystack) || isspace(start[-1])) + && (!*end || isspace(*end)); +} + +bool +ControlFile::hasNegWord(const char *word, const char *haystack) +{ + int len = strlen(word); + const char *start = strstr(haystack, word); + const char *end = start + len; + if (!start || start == haystack) + return false; + return start[-1] == '-' && (!*end || isspace(*end)); +} + +void +ControlFile::ensureComponent(const char *pattern) +{ + // Make sure at least one entry exists matching a pattern, if not, + // create it. Wildcard patterns cannot be created though! + ComponentIterator iter(getComponentIterator()); + bool wasSeen = false; + Component *c; + while ((c = iter.next()) != NULL) { + std::unique_ptr<Component> component(c); + if (c->matches(pattern)) { + wasSeen = true; + break; + } + } + if (!wasSeen) { + (void) getLevels(pattern); // Creates it ((### ugly ###)) + } +} + +bool +ControlFile::makeName(const char *service, char *buf, int bufLen) +{ + static const char *file = getenv("VESPA_LOG_CONTROL_FILE"); + static const char *dir = getenv("VESPA_LOG_CONTROL_DIR"); + + bool result; + + if (file) { + // mostly used for manual testing, so allow this even + // with empty service name + result = (snprintf(buf, bufLen, "%s", file) < bufLen); + } else if (dir) { + // We can't make control files for empty services here + if (!*service || strcmp(service, "-") == 0) { + return false; + } + if (strchr(service, '/') != NULL) { + LOG(debug, "service name '%s' should not contain /", service); + return false; + } + int req = snprintf(buf, bufLen, "%s/%s.logcontrol", dir, service); + result = (req < bufLen); + } else { + result = false; + } + return result; +} + + + +ComponentIterator +ControlFile::getComponentIterator() +{ + return ComponentIterator(this); +} + +ComponentIterator::ComponentIterator(const ComponentIterator& ci) + : _cf(ci._cf), + _next(ci._next) +{ +} + +ComponentIterator::ComponentIterator(ControlFile *cf) + : _cf(cf), + _next(cf->_firstComponent) +{ +} + +Component * +ComponentIterator::next() +{ + Component *ret = NULL; + if (_next && _next[0]) { + ret = new Component(_next); + _next = strchr(_next, '\n'); + if (_next) { + ++_next; + } + } + return ret; +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/control-file.h b/vespalog/src/vespa/log/control-file.h new file mode 100644 index 00000000000..d0fb211e202 --- /dev/null +++ b/vespalog/src/vespa/log/control-file.h @@ -0,0 +1,86 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "log.h" +#include "control-file.h" +#include "lock.h" + +namespace ns_log { + +class Component; +class ComponentIterator; + +class ControlFile { + friend class ComponentIterator; + friend class Component; +public: + enum Mode { READONLY, READWRITE, CREATE }; +private: + Lock _fileBacking; + enum Mode _mode; + char *_fileName; + void ensureHeader(); + bool hasPrefix() { return (_prefix != NULL && + _prefix[0] != '\0' && + _prefix[0] != ' ' && + _prefix[0] != '\n'); } + void ensureMapping(); + void freeMapping(); + bool extendMapping(); + static unsigned int pageAlign(unsigned int len); + + char *_prefix; + char *_mapBase; + int _mappedSize; + char *_firstComponent; + + static const int _maxMapSize = 200000; + static const int _maxPrefix = 64; + + ControlFile(const ControlFile &); + ControlFile& operator = (const ControlFile &); + + static bool hasWord(const char *word, const char *haystack); + static bool hasNegWord(const char *word, const char *haystack); + + static void makeLogLevelArray(unsigned int *levels, unsigned int size, + const char *env); + static char *alignLevels(char *addr); + static unsigned int findOnOffStatus(Logger::LogLevel level, + const char *levelsString); + +public: + ComponentIterator getComponentIterator(); + explicit ControlFile(const char *file, enum Mode); + void setPrefix(const char *prefix); + ~ControlFile(); + unsigned int *getLevels(const char *name); + void ensureComponent(const char *pattern); + + static unsigned int *defaultLevels(); + + // make sure in-memory changes are synchronized to disk + void flush(); + + // Construct the name of the control file for <serviceName> + // returns true if successful, false if not. + // Makes the name into <buf> which is of length <bufLen>. + static bool makeName(const char *serviceName, char *buf, int bufLen); +}; + +class ComponentIterator { +private: + ControlFile *_cf; + char *_next; + + ComponentIterator& operator = (const ComponentIterator &); + +public: + ComponentIterator(const ComponentIterator &ci); + ComponentIterator(ControlFile *cf); + ~ComponentIterator() {} + Component *next(); +}; + +} // end namespace ns_log + diff --git a/vespalog/src/vespa/log/create-multiarg-file.pl b/vespalog/src/vespa/log/create-multiarg-file.pl new file mode 100755 index 00000000000..3a937e3e0d1 --- /dev/null +++ b/vespalog/src/vespa/log/create-multiarg-file.pl @@ -0,0 +1,104 @@ +#!/usr/bin/env perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +# This script generates multi-argument functions for sending some events. + +require 5.006_001; +use warnings; +use strict; + +die "Usage: $0 <event-name> <event-version> <number of functions> <name>...\n" + unless $#ARGV > 2; + +my $event = $ARGV[0]; +my $event_version = $ARGV[1]; +my $noof_fns = $ARGV[2]; + +my @variables = @ARGV; +splice(@variables, 0, 3); + + +open(H, ">event-${event}-multiarg.h") + or die "Cannot open event-${event}-multiarg.h: $!\n"; +open(CPP, ">event-${event}-multiarg.cpp") + or die "Cannot open event-${event}-multiarg.cpp: $!\n"; + +my $called_as = "$0 $ARGV[0] $ARGV[1] $ARGV[2]"; +foreach my $x (@variables) { + $called_as .= " \"$x\""; +} + +print H << "EOF"; +// This file was generated like this: +// $called_as +// Don not modify this file manually! +EOF + +print CPP << "EOF"; +// This file was generated like this: +// $called_as +// Do not modify this file manually! +EOF + + +my $i; + +for ($i = 2; $i <= $noof_fns; $i++) { + print H "void doEvent\u${event}("; + print CPP "void\nLogger::doEvent\u${event}("; + my $n; + for ($n = 1; $n <= $i; $n++) { + print H "," unless $n == 1; + print H "\n\t"; + print CPP "," unless $n == 1; + print CPP "\n\t"; + my $first = 1; + foreach my $var (@variables) { + print H ", " unless $first; + print H "${var}${n}"; + print CPP ", " unless $first; + print CPP "${var}${n}"; + $first = 0; + } + } + print H ");\n\n"; + print CPP ")\n{\n"; + + print CPP "\tdoLog(event, \"\", 0, \"${event}/${event_version}\""; + for ($n = 1; $n <= $i; $n++) { + foreach my $var (@variables) { + my $type; + my $quot = ""; + if ($var =~ m=double =) { + $type = "%lf"; + } elsif ($var =~ m=int =) { + $type = "%d"; + } elsif ($var =~ m=const char ?\*=) { + $type = "%s"; + $quot = "\\\""; + } else { + die "Don't know printf format for variable $var\n"; + } + my $name = $var; + $name =~ s=.*[ *&]==; + print CPP "\n\t\t\" ${name}=${quot}${type}${quot}\""; + } + } + print CPP ","; + for ($n = 1; $n <= $i; $n++) { + my $first = 1; + print CPP "," unless $n == 1; + print CPP "\n\t\t"; + foreach my $var (@variables) { + print CPP ", " unless $first; + $first = 0; + my $name = $var; + $name =~ s=.*[ *&]==; + print CPP "${name}${n}"; + } + } + + print CPP ");\n"; + + print CPP "}\n\n"; +} diff --git a/vespalog/src/vespa/log/event-count-multiarg.cpp b/vespalog/src/vespa/log/event-count-multiarg.cpp new file mode 100644 index 00000000000..9a8d176bcf3 --- /dev/null +++ b/vespalog/src/vespa/log/event-count-multiarg.cpp @@ -0,0 +1,274 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// This file was generated like this: +// ./create-multiarg-file.pl count 2 10 "const char *name" "double value" +// Do not modify this file manually! +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7, + name8, value8); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7, + name8, value8, + name9, value9); +} + +void +Logger::doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9, + const char *name10, double value10) +{ + doLog(event, "", 0, "count/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7, + name8, value8, + name9, value9, + name10, value10); +} + diff --git a/vespalog/src/vespa/log/event-count-multiarg.h b/vespalog/src/vespa/log/event-count-multiarg.h new file mode 100644 index 00000000000..d2c7f60ec4a --- /dev/null +++ b/vespalog/src/vespa/log/event-count-multiarg.h @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// This file was generated like this: +// ./create-multiarg-file.pl count 2 10 "const char *name" "double value" +// Don not modify this file manually! +void doEventCount( + const char *name1, double value1, + const char *name2, double value2); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9); + +void doEventCount( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9, + const char *name10, double value10); + diff --git a/vespalog/src/vespa/log/event-value-multiarg.cpp b/vespalog/src/vespa/log/event-value-multiarg.cpp new file mode 100644 index 00000000000..c676ba44efd --- /dev/null +++ b/vespalog/src/vespa/log/event-value-multiarg.cpp @@ -0,0 +1,274 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// This file was generated like this: +// ./create-multiarg-file.pl value 2 10 "const char *name" "double value" +// Do not modify this file manually! +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7, + name8, value8); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7, + name8, value8, + name9, value9); +} + +void +Logger::doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9, + const char *name10, double value10) +{ + doLog(event, "", 0, "value/2" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf" + " name=\"%s\"" + " value=%lf", + name1, value1, + name2, value2, + name3, value3, + name4, value4, + name5, value5, + name6, value6, + name7, value7, + name8, value8, + name9, value9, + name10, value10); +} + diff --git a/vespalog/src/vespa/log/event-value-multiarg.h b/vespalog/src/vespa/log/event-value-multiarg.h new file mode 100644 index 00000000000..baf68683347 --- /dev/null +++ b/vespalog/src/vespa/log/event-value-multiarg.h @@ -0,0 +1,76 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// This file was generated like this: +// ./create-multiarg-file.pl value 2 10 "const char *name" "double value" +// Don not modify this file manually! +void doEventValue( + const char *name1, double value1, + const char *name2, double value2); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9); + +void doEventValue( + const char *name1, double value1, + const char *name2, double value2, + const char *name3, double value3, + const char *name4, double value4, + const char *name5, double value5, + const char *name6, double value6, + const char *name7, double value7, + const char *name8, double value8, + const char *name9, double value9, + const char *name10, double value10); + diff --git a/vespalog/src/vespa/log/internal.cpp b/vespalog/src/vespa/log/internal.cpp new file mode 100644 index 00000000000..3ff4c9cbec1 --- /dev/null +++ b/vespalog/src/vespa/log/internal.cpp @@ -0,0 +1,23 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <stdarg.h> +#include <cstdio> + +#include "internal.h" + +namespace ns_log { + +void +throwInvalid(const char *format, ...) +{ + char buf[4000]; + va_list args; + + va_start(args, format); + vsnprintf(buf, sizeof buf, format, args); + va_end(args); + + throw InvalidLogException(buf); +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/internal.h b/vespalog/src/vespa/log/internal.h new file mode 100644 index 00000000000..5b1e4b2deaf --- /dev/null +++ b/vespalog/src/vespa/log/internal.h @@ -0,0 +1,30 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <cstring> +#include <cstdlib> + +#if !__GNUC__ && !defined(__attribute__) +#define __attribute__(x) +#endif + +namespace ns_log { + +void throwInvalid(const char *fmt, ...) + __attribute__((format(printf, 1, 2))) __attribute__((noreturn)); + +class InvalidLogException { +private: + char *_what; + InvalidLogException& operator = (const InvalidLogException&); + +public: + InvalidLogException(const InvalidLogException &x) : + _what(strdup(x._what)) {} + InvalidLogException(const char *s) : _what(strdup(s)) {} + ~InvalidLogException() { free(_what); } + const char *what() const { return _what; } +}; + +} // end namespace ns_log + diff --git a/vespalog/src/vespa/log/llparser.cpp b/vespalog/src/vespa/log/llparser.cpp new file mode 100644 index 00000000000..88d181e2b2c --- /dev/null +++ b/vespalog/src/vespa/log/llparser.cpp @@ -0,0 +1,410 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <fcntl.h> +#include <errno.h> +#include <unistd.h> +#include <sys/time.h> + +#include "log-target.h" +#include "llparser.h" +#include "internal.h" + +namespace ns_log { + +static const char *defcomponent = "logger"; +static const char *defservice = "-"; +static char defhostname[256] = "localhost"; + +LLParser::LLParser() + : _defHostname(defhostname), + _defService(defservice), + _defComponent(defcomponent), + _defLevel(Logger::info), + _target(LogTarget::defaultTarget()), + _rejectFilter(RejectFilter::createDefaultFilter()) +{ + const char *envServ = getenv("VESPA_SERVICE_NAME"); + if (envServ != NULL) { + _defService = envServ; + } + char *name = getenv("VESPA_LOG_TARGET"); + if (name) { + LogTarget *target; + try { + target = LogTarget::makeTarget(name); + } catch (InvalidLogException& ex) { + // If we catch an exception, use default target + target = LogTarget::defaultTarget(); + } + delete _target; + _target = target; + } + if (gethostname(defhostname, 256) == 0) { + _defHostname = defhostname; + } + snprintf(_defPid, 10, "%d", (int)getpid()); +} + +LLParser::~LLParser() +{ + delete _target; +} + +const char LLParser::_hexdigit[17] = "0123456789abcdef"; + +void +LLParser::sendMessage(const char *totalMessage) +{ + _target->write(totalMessage, strlen(totalMessage)); +} + +static inline bool validLevel(Logger::LogLevel level) +{ + return (level >= 0 && level < Logger::NUM_LOGLEVELS); +} + +static bool isValidPid(const char *field) +{ + char *eol; + long pidnum = strtol(field, &eol, 10); + if (pidnum > 0 && pidnum < 18*1000*1000) { + char endbyte = *eol; + if (endbyte == '\0' || endbyte == '\t' || endbyte == '/') { + return true; + } + return false; + } + // too big to be a valid pid, maybe a timestamp? + if (pidnum >= 18*1000*1000) + return false; + + // stupid java logging... + if (field[0] == '-' && field[1] == '\0') { + return true; + } + if (field[0] == '-' && field[1] == '/') { + if (field[2] == '-' && field[3] == '\0') { + return true; + } + // thread id + pidnum = strtol(field+2, &eol, 10); + if (eol > field+2) { + char endbyte = *eol; + if (endbyte == '\0' || endbyte =='\t') { + return true; + } + } + } + return false; +} + +void +LLParser::doInput(char *line) +{ + double logTime = 0.0; + bool timefield = false; + int pidfield = 0; + char *eod = NULL; + + char *first = line; + char *tab = strchr(first, '\t'); // time? + char empty[1] = ""; + + if (tab) { + *tab = '\0'; + logTime = strtod(first, &eod); + if (eod == tab && logTime > 900*1000*1000) { + timefield = true; + } else if (isValidPid(first)) { + pidfield = 1; + } + char *second = tab+1; + tab = strchr(second, '\t'); // host? + if (tab) { + *tab = '\0'; + if (pidfield == 0) { + if (isValidPid(second)) { + pidfield = 2; + } + } + char *third = tab+1; + tab = strchr(third, '\t'); // pid? + if (tab) { + *tab = '\0'; + if (pidfield == 0) { + if (isValidPid(third)) { + pidfield = 3; + } + } + char *fourth = tab+1; + tab = strchr(fourth, '\t'); // service ? + if (tab) { + *tab = '\0'; + char *fifth = tab+1; + tab = strchr(fifth, '\t'); // component ? + if (tab) { + *tab = '\0'; + char *sixth = tab+1; + tab = strchr(sixth, '\t'); // level? + if (tab && timefield) { + *tab = '\0'; + char *seventh = tab+1; // message + Logger::LogLevel l = Logger::parseLevel(sixth); + if (validLevel(l)) { + makeMessage(first, second, third, fourth, + fifth, l, seventh); + return; + } + // pretend 6 fields + *tab = '\t'; + } + Logger::LogLevel l = Logger::parseLevel(fifth); + if (validLevel(l)) { + // missing one field - which one? + if (timefield && pidfield == 2) { + // missing host + makeMessage(first, empty, second, third, + fourth, l, sixth); + return; + } + if (timefield && pidfield == 3) { + // missing service + makeMessage(first, second, third, empty, + fourth, l, sixth); + return; + } + if (!timefield && pidfield == 2) { + // missing time + makeMessage(empty, first, second, third, + fourth, l, sixth); + return; + } + if (timefield && pidfield == 0) { + // missing pid + makeMessage(first, second, empty, third, + fourth, l, sixth); + return; + } + // fprintf(stderr, "bad 6-field\n"); + // no idea what is going on + } + // pretend 5 fields + tab = sixth-1; + *tab = '\t'; + } + Logger::LogLevel l = Logger::parseLevel(fourth); + if (validLevel(l)) { + // missing two fields + if (!timefield && pidfield == 0) { + // missing time and pid + makeMessage(empty, first, empty, second, + third, l, fifth); + return; + } + if (!timefield && pidfield == 1) { + // missing time and host + makeMessage(empty, empty, first, second, + third, l, fifth); + return; + } + if (!timefield && pidfield == 2) { + // missing time and service + makeMessage(empty, first, second, empty, + third, l, fifth); + return; + } + if (timefield && pidfield == 2) { + // missing host and service + makeMessage(first, empty, second, empty, + third, l, fifth); + return; + } + if (timefield && pidfield == 3) { + // missing service and component + makeMessage(first, second, third, empty, + empty, l, fifth); + return; + } + if (timefield && pidfield == 0) { + // missing pid and (hostname or service) + if (_defService == second) { + // it's the service, assume missing hostname + makeMessage(first, empty, empty, second, + third, l, fifth); + return; + } + makeMessage(first, second, empty, empty, + third, l, fifth); + return; + } + // fprintf(stderr, "bad 5-field\n"); + // no idea what is going on + } + // pretend 4 fields + tab = fifth-1; + *tab = '\t'; + } + Logger::LogLevel l = Logger::parseLevel(third); + if (validLevel(l)) { + // three fields + message + if (timefield && pidfield == 2) { + // missing host, service, component + makeMessage(first, empty, second, empty, + empty, l, fourth); + return; + } + if (timefield && pidfield == 0) { + // missing host, pid, service + makeMessage(first, empty, empty, empty, + second, l, fourth); + return; + } + if (!timefield && pidfield == 1) { + // missing time, host, service + makeMessage(empty, empty, first, empty, + second, l, fourth); + return; + } + if (!timefield && pidfield == 0) { + // missing time, pid, and (host or service) + if (_defService == first) { + // it's the service, assume missing hostname + makeMessage(empty, empty, empty, first, + second, l, fourth); + return; + } + // missing service + makeMessage(empty, first, empty, empty, + second, l, fourth); + return; + } + // fprintf(stderr, "bad 4-field\n"); + // no idea what is going on + + } + // pretend 3 fields + tab = fourth-1; + *tab = '\t'; + } + Logger::LogLevel l = Logger::parseLevel(second); + if (validLevel(l)) { + if (timefield) { + // time, level, message + makeMessage(first, empty, empty, empty, + empty, l, third); + return; + } + if (pidfield) { + // pid, level, message + makeMessage(empty, empty, first, empty, + empty, l, third); + return; + } + // component, level, message + makeMessage(empty, empty, empty, empty, + first, l, third); + return; + } + // pretend 2 fields + tab = third-1; + *tab = '\t'; + } + Logger::LogLevel l = Logger::parseLevel(first); + + if (validLevel(l)) { + makeMessage(empty, empty, empty, empty, + empty, l, second); + return; + } + // pretend 1 field + tab = second-1; + *tab = '\t'; + } + makeMessage(empty, empty, empty, empty, + empty, _defLevel, line); +} + + +static char escaped[16000]; +static char totalMessage[17000]; + +void +LLParser::makeMessage(const char *tmf, const char *hsf, const char *pdf, + const char *svf, const char *cmf, Logger::LogLevel level, + char *src) +{ + char tmbuffer[24]; + if (tmf[0] == '\0') { + struct timeval tv; + gettimeofday(&tv, NULL); + snprintf(tmbuffer, 24, "%u.%u", + static_cast<unsigned int>(tv.tv_sec), + static_cast<unsigned int>(tv.tv_usec)); + tmf = tmbuffer; + } + + if (hsf[0] == '\0') hsf = _defHostname.c_str(); + + if (pdf[0] == '\0') pdf = _defPid; + + if (svf[0] == '\0') svf = _defService.c_str(); + + if (cmf[0] == '\0') cmf = _defComponent.c_str(); + + char *dst = escaped; + unsigned char c; + + int len = strlen(src); + if (len > 3999) { + src[3997] = '.'; + src[3998] = '.'; + src[3999] = '.'; + src[4000] = '\0'; + } + do { + c = static_cast<unsigned char>(*src++); + if ((c == '\\' && src[0] == 't') + || (c >= 32 && c < '\\') + || (c > '\\' && c < 128) + || c == 0 + || c > 160) + { + *dst++ = static_cast<char>(c); + } else { + *dst++ = '\\'; + if (c == '\\') { + *dst++ = '\\'; + } else if (c == '\r') { + *dst++ = 'r'; + } else if (c == '\n') { + *dst++ = 'n'; + } else if (c == '\t') { + *dst++ = 't'; + } else { + *dst++ = 'x'; + *dst++ = _hexdigit[c >> 4]; + *dst++ = _hexdigit[c & 0xf]; + } + } + } while (c); + + snprintf(totalMessage, sizeof totalMessage, + "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", + tmf, hsf, pdf, svf, cmf, + Logger::logLevelNames[level], escaped); + + if (_rejectFilter.shouldReject(level, escaped)) + return; + sendMessage(totalMessage); +} + + +void +LLParser::setPid(int p) +{ + snprintf(_defPid, 10, "%d", p); +} + +} // namespace diff --git a/vespalog/src/vespa/log/llparser.h b/vespalog/src/vespa/log/llparser.h new file mode 100644 index 00000000000..e8d35bc9777 --- /dev/null +++ b/vespalog/src/vespa/log/llparser.h @@ -0,0 +1,44 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/log/log.h> +#include <vespa/log/reject-filter.h> +#include <string> + +namespace ns_log { + +class LogTarget; +class LLParser +{ +private: + static const char _hexdigit[17]; + + char _defPid[10]; + std::string _defHostname; + std::string _defService; + std::string _defComponent; + Logger::LogLevel _defLevel; + + LogTarget *_target; + RejectFilter _rejectFilter; + void makeMessage(Logger::LogLevel l, char *msg); + void makeMessage(const char *tmf, const char *hsf, const char *pdf, + const char *svf, const char *cmf, Logger::LogLevel l, + char *msg); + void sendMessage(const char *msg); + + LLParser(const LLParser &); + LLParser& operator = (const LLParser &); + +public: + void doInput(char *line); + LLParser(); + ~LLParser(); + void setService(const char *s) { _defService = s; } + void setComponent(const char *c) { _defComponent = c; } + void setPid(int p); + void setDefaultLevel(Logger::LogLevel level) { _defLevel = level; } +}; + +} // namespace + diff --git a/vespalog/src/vespa/log/lock.cpp b/vespalog/src/vespa/log/lock.cpp new file mode 100644 index 00000000000..be83a745fd7 --- /dev/null +++ b/vespalog/src/vespa/log/lock.cpp @@ -0,0 +1,89 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <errno.h> +#include <sys/file.h> +#include <cstring> +#include <unistd.h> +#include <limits.h> +#include <cstdio> + +#include "log.h" +LOG_SETUP_INDIRECT(".log.lock", "$Id$"); +#include "lock.h" +#include "internal.h" + +#undef LOG +#define LOG LOG_INDIRECT + +namespace ns_log { + +Lock::Lock(int arg_fd) + : _fd(dup(arg_fd)), + _isLocked(false) +{ + if (_fd == -1) { + throwInvalid("Cannot duplicate fd %d for lock: %s", arg_fd, + strerror(errno)); + } +} + + +Lock::Lock(const char *filename, int mode) + : _fd(open(filename, mode, 0666)), + _isLocked(false) +{ + if (_fd == -1) { + if (mode & O_CREAT) { + unlink(filename); + _fd = open(filename, mode, 0666); + } + if (_fd == -1) { + throwInvalid("Cannot open lockfile '%s': %s", filename, + strerror(errno)); + } + } + fcntl(_fd, F_SETFD, FD_CLOEXEC); +} + +void +Lock::lock(bool exclusive) +{ + // Early escape to avoid double locking which would of course block forever + if (_isLocked) return; + + if (flock(_fd, exclusive ? LOCK_EX : LOCK_SH) == -1) { + int err = errno; + close(_fd); + LOG(error, "Cannot lock fd %d: %s", _fd, strerror(err)); + throwInvalid("Cannot lock fd %d: %s", _fd, strerror(err)); + } + _isLocked = true; +} + +void +Lock::unlock() { + if (_isLocked) { + flock(_fd, LOCK_UN); + _isLocked = false; + } +} + +int Lock::size() +{ + struct stat st; + int err = fstat(_fd, &st); + if (err == -1) { + return -1; + } + return st.st_size; +} + +Lock::~Lock() +{ + unlock(); + close(_fd); +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/lock.h b/vespalog/src/vespa/log/lock.h new file mode 100644 index 00000000000..48af938fc53 --- /dev/null +++ b/vespalog/src/vespa/log/lock.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <sys/types.h> +#include <fcntl.h> +#include <limits.h> + +namespace ns_log { + +class Lock { +private: + Lock(); + Lock(const Lock &); + Lock& operator =(const Lock &); + int _fd; + bool _isLocked; + + +public: + int fd() const { return _fd; } + int size(); + explicit Lock(const char *filename, int mode = O_RDONLY | O_NOCTTY); + explicit Lock(int fd); + ~Lock(); + void lock(bool isExclusive); + void unlock(); +}; + + +} // end namespace ns_log + + diff --git a/vespalog/src/vespa/log/log-assert.cpp b/vespalog/src/vespa/log/log-assert.cpp new file mode 100644 index 00000000000..2c4106816a1 --- /dev/null +++ b/vespalog/src/vespa/log/log-assert.cpp @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <sys/types.h> +#include <cstring> +#include <cstdlib> +#include <cstdio> +#include <unistd.h> + +#include "log.h" +LOG_SETUP(""); + +namespace ns_log { + +void log_assert_fail(const char *assertion, + const char *file, + uint32_t line) +{ + LOG(error, "%s:%d: Failed assertion: '%s'", + file, line, assertion); + fprintf(stderr, "%s:%d: Failed assertion: '%s'\n", + file, line, assertion); + abort(); +} + + +void log_abort(const char *message, + const char *file, + unsigned int line) +{ + LOG(error, "%s:%d: Abort called. Reason: %s", + file, line, message); + fprintf(stderr, "%s:%d: Abort called. Reason: %s\n", + file, line, message); + abort(); +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/log-target-fd.cpp b/vespalog/src/vespa/log/log-target-fd.cpp new file mode 100644 index 00000000000..ac28293d3bc --- /dev/null +++ b/vespalog/src/vespa/log/log-target-fd.cpp @@ -0,0 +1,42 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <unistd.h> +#include <cstring> +#include <cstdlib> + +#include "log.h" +LOG_SETUP(".log"); +#include "log-target-fd.h" +#include "internal.h" + +namespace ns_log { + +LogTargetFd::LogTargetFd(const char *target) + : LogTarget(target), + _fd(-1), _istty(false) +{ + if (strncmp(target, "fd:", 3) != 0) { + throwInvalid("Bad target for LogTargetFd: '%s'", target); + } + _fd = strtol(target + 3, NULL, 0); + if (isatty(_fd) == 1) { + _istty = true; + } +} + +LogTargetFd::~LogTargetFd() +{ + // Must not close _fd, we did not open it! +} + + +// When writing to file descriptors, there is really not much to do. +// No log rotation is supported (at least not directly). +int +LogTargetFd::write(const char *buf, int bufLen) +{ + return ::write(_fd, buf, bufLen); +} + +} // end namespace ns_log + diff --git a/vespalog/src/vespa/log/log-target-fd.h b/vespalog/src/vespa/log/log-target-fd.h new file mode 100644 index 00000000000..f75ff9b44e1 --- /dev/null +++ b/vespalog/src/vespa/log/log-target-fd.h @@ -0,0 +1,25 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "log-target.h" + +namespace ns_log { + +class LogTargetFd : public LogTarget { +private: + int _fd; + bool _istty; + LogTargetFd(); + LogTargetFd(const LogTargetFd&); + LogTargetFd& operator = (const LogTargetFd); + +public: + explicit LogTargetFd(const char *target); + virtual int write(const char *buf, int len); + virtual ~LogTargetFd(); + virtual bool makeHumanReadable() const { return _istty; } +}; + + +} // end namespace log + diff --git a/vespalog/src/vespa/log/log-target-file.cpp b/vespalog/src/vespa/log/log-target-file.cpp new file mode 100644 index 00000000000..07b638d7f84 --- /dev/null +++ b/vespalog/src/vespa/log/log-target-file.cpp @@ -0,0 +1,72 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <unistd.h> +#include <cstring> +#include <sys/stat.h> +#include <fcntl.h> +#include <cstdlib> +#include <errno.h> +#include <cstdio> + +#include "log.h" +LOG_SETUP(".log"); +LOG_RCSID("$Id$"); +#include "log-target-file.h" +#include "internal.h" + +namespace ns_log { + +#ifndef O_LARGEFILE +#define O_LARGEFILE 0 +#endif + +LogTargetFile::LogTargetFile(const char *target) + : LogTarget(target), + _failstate(FS_OK) +{ + memset(_fname, 0, sizeof(_fname)); + const char *fname = target + strlen("file:"); + assert(strlen(fname) < sizeof(_fname)); + // NOTE: This function cannot LOG() + if (strncmp(target, "file:", strlen("file:")) != 0) { + throwInvalid("Illegal log target '%s'", target); + } + strcpy(_fname, fname); + int fd = open(_fname, O_WRONLY | O_CREAT | O_APPEND | O_NOCTTY | O_LARGEFILE, + 0666); + if (fd == -1) { + throwInvalid("Cannot open log target file '%s': %s", + target + strlen("file:"), strerror(errno)); + } + close(fd); +} + +LogTargetFile::~LogTargetFile() +{ +} + +// Here we must support log rotation. We do this by reopening the filename. +int +LogTargetFile::write(const char *buf, int bufLen) +{ + int fd = open(_fname, + O_WRONLY | O_CREAT | O_APPEND | O_NOCTTY | O_LARGEFILE, + 0666); + if (fd < 0) { + if (_failstate == FS_OK) { + _failstate = FS_FAILED; + LOG(warning, "cannot create %s: %s", _fname, strerror(errno)); + LOG(warning, "Log file failed, falling back to stderr logging"); + } + fd = dup(STDERR_FILENO); + } else { + if (_failstate != FS_OK) { + _failstate = FS_OK; + } + } + int retVal = ::write(fd, buf, bufLen); + close(fd); + return retVal; +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/log-target-file.h b/vespalog/src/vespa/log/log-target-file.h new file mode 100644 index 00000000000..4ac71ff65ec --- /dev/null +++ b/vespalog/src/vespa/log/log-target-file.h @@ -0,0 +1,32 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <sys/types.h> +#include <sys/stat.h> + +#include <vespa/fastos/fastos.h> + +#include "log-target.h" + +namespace ns_log { + +class LogTargetFile : public LogTarget { +private: + char _fname[256]; + enum FailState { + FS_OK, FS_CHECKING, FS_ROTATING, FS_FAILED + } _failstate; + + LogTargetFile(); + LogTargetFile(const LogTargetFile&); + LogTargetFile& operator =(const LogTargetFile&); + +public: + explicit LogTargetFile(const char *target); + virtual ~LogTargetFile(); + virtual int write(const char *buf, int len); +}; + + +} // end namespace log + diff --git a/vespalog/src/vespa/log/log-target.cpp b/vespalog/src/vespa/log/log-target.cpp new file mode 100644 index 00000000000..d34e0067b04 --- /dev/null +++ b/vespalog/src/vespa/log/log-target.cpp @@ -0,0 +1,48 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <cstring> +#include <cstdlib> + +#include "log.h" +LOG_SETUP(".log"); +LOG_RCSID("$Id$"); + +#include "log-target-fd.h" +#include "log-target-file.h" +#include "internal.h" + +namespace ns_log { + +LogTarget::LogTarget(const char *target) +{ + memset(_name, 0, sizeof(_name)); + assert(strlen(target) < sizeof(_name)); + strcpy(_name, target); +} + +LogTarget::~LogTarget() { +} + +LogTarget * +LogTarget::defaultTarget() +{ + // Note! This function cannot LOG(). + return new LogTargetFd("fd:2"); +} + +LogTarget * +LogTarget::makeTarget(const char *const target) +{ + LogTarget *res = NULL; + if (strncmp(target, "fd:", 3) == 0) { + res = new LogTargetFd(target); + } else if (strncmp(target, "file:", 5) == 0) { + res = new LogTargetFile(target); + } else { + throwInvalid("Log target '%s' is invalid.", target); + } + return res; +} + + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/log-target.h b/vespalog/src/vespa/log/log-target.h new file mode 100644 index 00000000000..5a27bec7e4d --- /dev/null +++ b/vespalog/src/vespa/log/log-target.h @@ -0,0 +1,28 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +/* This is a virtual base class for all log targets. + A log target has a constructor with the log target in string form, + and a write() method to write log messages. that's it. */ + +namespace ns_log { + +class LogTarget { +private: + char _name[256]; + LogTarget(); + LogTarget& operator =(const LogTarget &); + LogTarget(const LogTarget&); + +public: + LogTarget(const char *name); + virtual int write(const char *buf, int bufLen) = 0; + virtual ~LogTarget(); + static LogTarget *makeTarget(const char *target); + static LogTarget *defaultTarget(); + virtual const char *name() const { return _name; } + virtual bool makeHumanReadable() const { return false; } +}; + +} // end namespace ns_log + diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp new file mode 100644 index 00000000000..6ba386ba04d --- /dev/null +++ b/vespalog/src/vespa/log/log.cpp @@ -0,0 +1,409 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/fastos/fastos.h> +#include <sys/types.h> +#include <cstring> +#include <cstdlib> +#include <stdarg.h> +#include <cstdio> +#include <sys/time.h> +#include <unistd.h> +#include <errno.h> +#include <memory> + +#include <algorithm> + +#include "log.h" +LOG_SETUP_INDIRECT(".log", "$Id$"); +#undef LOG +#define LOG LOG_INDIRECT + +#include "lock.h" +#include "log-target.h" +#include "internal.h" +#include "control-file.h" + +namespace ns_log { + +uint64_t Timer::getTimestamp() const { + struct timeval tv; + gettimeofday(&tv, NULL); + uint64_t timestamp = tv.tv_sec; + timestamp *= 1000000; + timestamp += tv.tv_usec; + return timestamp; +} + +LogTarget *Logger::_target = 0; +int Logger::_numInstances = 0; +bool Logger::fakePid = false; +char Logger::_prefix[64] = { '\0' }; +const char Logger::_hexdigit[17] = "0123456789abcdef"; +char Logger::_controlName[1024] = { '\0' }; +char Logger::_hostname[1024] = { '\0'}; +char Logger::_serviceName[1024] = {'\0' }; +ControlFile *Logger::_controlFile = 0; + +void +Logger::ensureControlName() +{ + if (_controlName[0] == '\0') { + if (!ControlFile::makeName(_serviceName, _controlName, + sizeof _controlName)) { + LOG(spam, "Neither $VESPA_LOG_CONTROL_FILE nor " + "$VESPA_LOG_CONTROL_DIR + $VESPA_SERVICE_NAME are set, " + "runtime log-control is therefore disabled."); + strcpy(_controlName, "///undefined///"); + } + } +} + + +void +Logger::ensureServiceName() +{ + if (_serviceName[0] == '\0') { + const char *name = getenv("VESPA_SERVICE_NAME"); + snprintf(_serviceName, sizeof _serviceName, "%s", name ? name : "-"); + } +} + +void +Logger::setTarget() +{ + try { + char *name = getenv("VESPA_LOG_TARGET"); + if (name) { + LogTarget *target = LogTarget::makeTarget(name); + delete _target; + _target = target; + } else { + LOG(spam, "$VESPA_LOG_TARGET is not set, logging to stderr"); + } + } catch (InvalidLogException& x) { + LOG(error, "Log target problem: %s. Logging to stderr. " + "($VESPA_LOG_TARGET=\"%s\")", + x.what(), getenv("VESPA_LOG_TARGET")); + } +} + + +void +Logger::ensurePrefix(const char *name) +{ + const char *start = name; + if (name[0] != '\0' && name[0] != '.') { + const char *end = strchr(name, '.'); + int len = end ? end - name : strlen(name); + start += len; + + if (_prefix[0]) { + // Make sure the prefix already set is identical to this one + if ((len != int(strlen(_prefix))) + || memcmp(name, _prefix, len) != 0) { + LOG(error, "Fatal: Tried to set log component name '%s' which " + "conflicts with existing root component '%s'. ABORTING", + name, _prefix); + throwInvalid("Bad config component name '%s' conflicts " + "with existing name '%s'", name, _prefix); + } + } else { + // No one has set the prefix yet, so we do it + snprintf(_prefix, sizeof _prefix, "%.*s", len, name); + LOG(debug, "prefix was set to '%s'", _prefix); + } + } +} + +void +Logger::ensureHostname() +{ + if (_hostname[0] == '\0') { + char hn[257]; + snprintf(_hostname, sizeof _hostname, "%s", + (gethostname(hn, sizeof hn) == 0) ? hn : "-"); + } +} + + +Logger::Logger(const char *name, const char *rcsId) + : _logLevels(ControlFile::defaultLevels()), + _timer(new Timer()) +{ + _numInstances++; + memset(_rcsId, 0, sizeof(_rcsId)); + memset(_appendix, 0, sizeof(_appendix)); + const char *app(strchr(name, '.') ? strchr(name, '.') : ""); + assert(strlen(app) < sizeof(_appendix)); + strncpy(_appendix, app, sizeof(_appendix)); + if (!_target) { + // Set up stderr first as a target so we can log even if target + // cannot be found + _target = LogTarget::defaultTarget(); + setTarget(); + } + ensureServiceName(); + if (rcsId) { + setRcsId(rcsId); + } + ensureControlName(); + ensurePrefix(name); + ensureHostname(); + + // Only read log levels from a file if we are using a file! + if (strcmp(_controlName, "///undefined///") != 0) { + try { + if (!_controlFile) { + _controlFile = new ControlFile(_controlName, ControlFile::CREATE); + } + _logLevels = _controlFile->getLevels(_appendix); + _controlFile->setPrefix(_prefix); + } catch (InvalidLogException& x) { + LOG(error, "Problems initialising logging: %s.", x.what()); + LOG(warning, "Log control disabled, using default levels."); + } + } +} + + +Logger::~Logger() +{ + _numInstances--; + if ((_numInstances == 1)) { + if (logger != NULL) { + logger->~Logger(); + free(logger); + logger = NULL; + } + } else if (_numInstances == 0) { + delete _controlFile; + logInitialised = false; + delete _target; + _target = NULL; + } +} + + +int +Logger::setRcsId(const char *id) +{ + const char *start = strchr(id, ','); + if (start) { + start += std::min((size_t)3, strlen(start)); // Skip 3 chars + } else { + start = id; + } + + int len = strlen(start); + const char *end = strchr(start, ' '); + if (!end) { + end = start + len; + } + + assert(size_t(len + 8) < sizeof(_rcsId)); + sprintf(_rcsId, "(%.*s): ", (int)(end - start), start); + LOG(spam, "rcs id was set to '%s'", _rcsId); + return 0; +} + +int +Logger::tryLog(int sizeofPayload, LogLevel level, const char *file, int line, const char *fmt, va_list args) +{ + char * payload(new char[sizeofPayload]); + const int actualSize = vsnprintf(payload, sizeofPayload, fmt, args); + + if (actualSize < sizeofPayload) { + uint64_t timestamp = _timer->getTimestamp(); + doLogCore(timestamp, level, file, line, payload, actualSize); + } + delete[] payload; + return actualSize; +} + +void +Logger::doLog(LogLevel level, const char *file, int line, const char *fmt, ...) +{ + int sizeofPayload(0x400); + int actualSize(sizeofPayload-1); + do { + sizeofPayload = actualSize+1; + va_list args; + va_start(args, fmt); + actualSize = tryLog(sizeofPayload, level, file, line, fmt, args); + va_end(args); + } while (sizeofPayload < actualSize); +} + +void Logger::doLogCore(uint64_t timestamp, LogLevel level, + const char *file, int line, const char *msg, size_t msgSize) +{ + const size_t sizeofEscapedPayload(msgSize*4+1); + const size_t sizeofTotalMessage(sizeofEscapedPayload + 1000); + char * bigBuffer(new char[sizeofEscapedPayload+sizeofTotalMessage]); + char * escapedPayload(bigBuffer); + char * totalMessage(escapedPayload + sizeofEscapedPayload); + + char *dst = escapedPayload; + for (size_t i(0); (i < msgSize) && msg[i]; i++) { + unsigned char c = static_cast<unsigned char>(msg[i]); + if ((c >= 32) && (c != '\\') && (c != 127)) { + *dst++ = static_cast<char>(c); + } else { + *dst++ = '\\'; + if (c == '\\') { + *dst++ = '\\'; + } else if (c == '\r') { + *dst++ = 'r'; + } else if (c == '\n') { + *dst++ = 'n'; + } else if (c == '\t') { + *dst++ = 't'; + } else { + *dst++ = 'x'; + *dst++ = _hexdigit[c >> 4]; + *dst++ = _hexdigit[c & 0xf]; + } + } + } + *dst = 0; + + // The only point of tracking thread id is to be able to distinguish log + // from multiple threads from each other. As one aren't using that many + // threads, only showing the least significant bits will hopefully + // distinguish between all threads in your application. Alter later if + // found to be too inaccurate. + int32_t tid = (fakePid ? -1 : FastOS_Thread::GetCurrentThreadId() % 0xffff); + + if (_target->makeHumanReadable()) { + time_t secs = static_cast<time_t>(timestamp / 1000000); + struct tm tmbuf; + localtime_r(&secs, &tmbuf); + char timebuf[100]; + strftime(timebuf, 100, "%Y-%m-%d %H:%M:%S", &tmbuf); + snprintf(totalMessage, sizeofTotalMessage, + "[%s.%06u] %d/%d (%s%s) %s: %s\n", + timebuf, static_cast<unsigned int>(timestamp % 1000000), + fakePid ? -1 : getpid(), tid, + _prefix, _appendix, + levelName(level), msg); + } else if (level == debug || level == spam) { + snprintf(totalMessage, sizeofTotalMessage, + "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n", + static_cast<unsigned int>(timestamp / 1000000), + static_cast<unsigned int>(timestamp % 1000000), + _hostname, fakePid ? -1 : getpid(), tid, + _serviceName, _prefix, + _appendix, levelName(level), file, line, + _rcsId, + escapedPayload); + } else { + snprintf(totalMessage, sizeofTotalMessage, + "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n", + static_cast<unsigned int>(timestamp / 1000000), + static_cast<unsigned int>(timestamp % 1000000), + _hostname, fakePid ? -1 : getpid(), tid, + _serviceName, _prefix, + _appendix, levelName(level), escapedPayload); + } + + _target->write(totalMessage, strlen(totalMessage)); + delete [] bigBuffer; +} + +const char * +Logger::levelName(LogLevel level) +{ + switch (level) { + case fatal: return "fatal"; // Deprecated, remove this later. + case error: return "error"; + case warning: return "warning"; + case event: return "event"; + case config: return "config"; + case info: return "info"; + case debug: return "debug"; + case spam: return "spam"; + case NUM_LOGLEVELS: break; + } + return "--unknown--"; +} + +void +Logger::doEventStarting(const char *name) +{ + doLog(event, "", 0, "starting/1 name=\"%s\"", name); +} + +void +Logger::doEventStopping(const char *name, const char *why) +{ + doLog(event, "", 0, "stopping/1 name=\"%s\" why=\"%s\"", name, why); +} + +void +Logger::doEventStarted(const char *name) +{ + doLog(event, "", 0, "started/1 name=\"%s\"", name); +} + +void +Logger::doEventStopped(const char *name, pid_t pid, int exitCode) +{ + doLog(event, "", 0, "stopped/1 name=\"%s\" pid=%d exitcode=%d", name, + static_cast<int>(pid), exitCode); +} + +void +Logger::doEventReloading(const char *name) +{ + doLog(event, "", 0, "reloading/1 name=\"%s\"", name); +} + +void +Logger::doEventReloaded(const char *name) +{ + doLog(event, "", 0, "reloaded/1 name=\"%s\"", name); +} + +void +Logger::doEventCrash(const char *name, pid_t pid, int signal) +{ + doLog(event, "", 0, "crash/1 name=\"%s\" pid=%d signal=\"%s\"", name, pid, + strsignal(signal)); +} + +void +Logger::doEventProgress(const char *name, double value, double total) +{ + if (total > 0) { + doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g total=%.18g", + name, value, total); + } else { + doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g", name, value); + } +} + +void +Logger::doEventCount(const char *name, uint64_t value) +{ + doLog(event, "", 0, "count/1 name=\"%s\" value=%" PRIu64, name, value); +} + +void +Logger::doEventValue(const char *name, double value) +{ + doLog(event, "", 0, "value/1 name=\"%s\" value=%.18g", name, value); +} + +void +Logger::doEventCollection(uint64_t collectionId, const char* name, const char* params) +{ + doLog(event, "", 0, "collection/1 collectionId=%" PRIu64 " name=\"%s\" %s", + collectionId, name, params); +} + +void +Logger::doEventState(const char *name, const char *value) +{ + doLog(event, "", 0, "state/1 name=\"%s\" value=\"%s\"", name, value); +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h new file mode 100644 index 00000000000..827d615c4e8 --- /dev/null +++ b/vespalog/src/vespa/log/log.h @@ -0,0 +1,308 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <memory> +#include <inttypes.h> +#include <sys/types.h> // for pid_t +#include <new> // for placement new +#include <cstdlib> // for malloc +#include <cstring> // for memset + +/** + * If this macro is defined, the regular LOG calls will go through the + * buffered logger, using the whole messages as tokens. + * + * If this macro is not defined, only LOGB calls will go through log buffer + */ +//#define VESPA_LOG_USELOGBUFFERFORREGULARLOG 1 + +// Used to use anonymous namespaces, but they fail miserably in gdb 5.3 + +#define LOG_SETUP(...) \ +static ns_log::Logger logger(__VA_ARGS__) + +#define LOG_SETUP_INDIRECT(x, id) \ +static ns_log::Logger *logger=NULL; \ +static bool logInitialised = false; \ +static const char *logName = x; \ +static const char *rcsId = id + + +#define LOG_RCSID(x) \ +static int log_dummmy __attribute__((unused)) = logger.setRcsId(x) + + +// Define LOG if not using log buffer. Otherwise log buffer will define them +#ifndef VESPA_LOG_USELOGBUFFERFORREGULARLOG +#define LOG(level, ...) \ +do { \ + if (logger.wants(ns_log::Logger::level)) { \ + logger.doLog(ns_log::Logger::level, __FILE__, __LINE__, __VA_ARGS__); \ + ns_log::BufferedLogger::logger.trimCache(); \ + } \ +} while (0) +#define VLOG(level, ...) \ +do { \ + if (logger.wants(level)) { \ + logger.doLog(level, __FILE__, __LINE__, __VA_ARGS__); \ + ns_log::BufferedLogger::logger.trimCache(); \ + } \ +} while (0) +#endif + +// Must use placement new in the following definition, since the variable +// "logger" must be a valid logger object DURING the construction of the +// logger object itself. +#define LOG_INDIRECT_MUST \ + if (!logInitialised) { \ + logInitialised = true; \ + logger = static_cast<Logger *>(malloc(sizeof *logger)); \ + memset(logger, 0, sizeof *logger); \ + new (logger) Logger(logName, rcsId); \ + } +#define LOG_INDIRECT(level, ...) \ +do { \ + LOG_INDIRECT_MUST \ + if (logger->wants(ns_log::Logger::level)) { \ + logger->doLog(ns_log::Logger::level, __FILE__, __LINE__, __VA_ARGS__); \ + } \ +} while (0) + +#define LOG_WOULD_LOG(level) logger.wants(ns_log::Logger::level) +#define LOG_WOULD_VLOG(level) logger.wants(level) + +#define EV_STARTING(name) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventStarting(name); \ + } \ +} while (0) + +#define EV_STOPPING(name,why) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventStopping(name, why); \ + } \ +} while (0) + +#define EV_STARTED(name) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventStarted(name); \ + } \ +} while (0) + +#define EV_STOPPED(name,pid,exitcode) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventStopped(name, pid, exitcode); \ + } \ +} while (0) + +#define EV_RELOADING(name) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventReloading(name); \ + } \ +} while (0) + +#define EV_RELOADED(name) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventReloaded(name); \ + } \ +} while (0) + +#define EV_CRASH(name,pid,signal) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventCrash(name, pid, signal); \ + } \ +} while (0) + +#define EV_PROGRESS(name, ...) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventProgress(name, __VA_ARGS__); \ + } \ +} while (0) + +#define EV_COUNT(name,value) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventCount(name, value); \ + } \ +} while (0) + +#define EV_VALUE(name,value) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventValue(name, value); \ + } \ +} while (0) + +#define EV_COLLECTION(collectionId,name,params) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventCollection(collectionId, name, params); \ + } \ +} while (0) + +#define EV_STATE(name,value) \ +do { \ + if (logger.wants(ns_log::Logger::event)) { \ + logger.doEventState(name, value); \ + } \ +} while (0) + +namespace ns_log { + +class LogTarget; +class ControlFile; + +// XXX this is way too complicated, must be some simpler way to do this +/** Timer class used to retrieve timestamp, such that we can override in test */ +struct Timer { + virtual ~Timer() {} + virtual uint64_t getTimestamp() const; +}; + +/** Test timer returning just a given time. Used in tests to fake time. */ +struct TestTimer : public Timer { + uint64_t & _time; + TestTimer(uint64_t & timeVar) : _time(timeVar) { } + virtual ~TestTimer() {} + virtual uint64_t getTimestamp() const { return _time; } +}; + +class Logger { +public: + enum LogLevel { + fatal, error, warning, config, info, event, debug, spam, NUM_LOGLEVELS + }; + static const char *logLevelNames[]; + static enum LogLevel parseLevel(const char *lname); + + /** + * Set by unit tests to avoid needing to match different pids. Will be + * false by default. + */ + static bool fakePid; + +private: + Logger(); + Logger(const Logger &); + Logger& operator =(const Logger &); + + unsigned int *_logLevels; + + static char _prefix[64]; + static LogTarget *_target; + char _rcsId[256]; + static int _numInstances; + + static char _controlName[1024]; + static char _hostname[1024]; + static char _serviceName[1024]; + static const char _hexdigit[17]; + static ControlFile *_controlFile; + + static void setTarget(); + void makeLockFile(); + + char _appendix[256]; + + std::unique_ptr<Timer> _timer; + + void ensurePrefix(const char *name); + void ensureHostname(); + void ensureControlName(); + void ensureServiceName(); + + int tryLog(int sizeofPayload, LogLevel level, const char *file, int line, + const char *fmt, va_list args); +public: + ~Logger(); + explicit Logger(const char *name, const char *rcsId = 0); + static Logger& getLogger(const char *name); + int setRcsId(const char *rcsId); + static const char *levelName(LogLevel level); + + inline bool wants(LogLevel level); + void doLog(LogLevel level, const char *file, int line, + const char *fmt, ...) __attribute__((format(printf,5,6))); + /** + * The log buffer creates timestamp and creates the log message itself. + * Thus the core log functionality has been moved here to avoid doing it + * twice. doLogCore is called from doLog and from the log buffer. + * + * @param timestamp Time in microseconds. + */ + void doLogCore(uint64_t timestamp, LogLevel level, + const char *file, int line, const char *msg, size_t msgSize); + void doEventStarting(const char *name); + void doEventStopping(const char *name, const char *why); + void doEventStarted(const char *name); + void doEventStopped(const char *name, pid_t pid, int exitCode); + void doEventReloading(const char *name); + void doEventReloaded(const char *name); + void doEventCrash(const char *name, pid_t pid, int signal); + void doEventProgress(const char *name, double value, double total = 0); + void doEventCount(const char *name, uint64_t value); + void doEventValue(const char *name, double value); + void doEventCollection(uint64_t collectionId, const char *name, + const char *params); + void doEventState(const char *name, const char *value); + + // Only for unit testing + void setTimer(std::unique_ptr<Timer> timer) { _timer = std::move(timer); } +}; + + +#define CHARS_TO_UINT(a,b,c,d) \ +(static_cast<unsigned char>(a) \ + | static_cast<unsigned int>(static_cast<unsigned char>(b) << 8u) \ + | static_cast<unsigned int>(static_cast<unsigned char>(c) << 16u) \ + | static_cast<unsigned int>(static_cast<unsigned char>(d) << 24u)) + +inline bool Logger::wants(LogLevel level) +{ + return _logLevels[level] == CHARS_TO_UINT(' ', ' ', 'O', 'N'); +} + +#define LOG_noreturn __attribute__((__noreturn__)) + +extern void log_assert_fail(const char *assertion, + const char *file, + uint32_t line) LOG_noreturn; + +extern void log_abort(const char *message, + const char *file, + uint32_t line) LOG_noreturn; + +#undef LOG_noreturn + +} // end namespace log + +//======================================// +// LOG_ASSERT and LOG_ABORT definitions // +//======================================// + +#ifndef __STRING +#define __STRING(x) #x +#endif + +#define LOG_ABORT(msg) \ + (ns_log::log_abort(msg, __FILE__, __LINE__)) + +#ifndef NDEBUG +#define LOG_ASSERT(expr) \ + ((void) ((expr) ? 0 : \ + (ns_log::log_assert_fail(__STRING(expr), \ + __FILE__, __LINE__), 0))) +#else +#define LOG_ASSERT(expr) +#endif // #ifndef NDEBUG + +#include <vespa/log/bufferedlogger.h> + diff --git a/vespalog/src/vespa/log/mknm.pl b/vespalog/src/vespa/log/mknm.pl new file mode 100755 index 00000000000..126fe0ba6ff --- /dev/null +++ b/vespalog/src/vespa/log/mknm.pl @@ -0,0 +1,46 @@ +#!/usr/bin/perl +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +while (<>) { + if ( s/.*\benum\s+LogLevel\s*\{// ) { + chomp; + $t = $_; + while (<>) { + if ( s/\}.*// ) { + $t .= $_; + $t =~ s/,/ /g; + @t = split(" ", $t); + if ( $t[$#t] ne "NUM_LOGLEVELS" ) { + die "expected NUM_LOGLEVELS got '$t[$#t]'\n"; + } + pop @t; + makecpp(); + } + $t .= $_; + } + } +} +die "did not find enum\n"; + +sub makecpp +{ + print "#include <string.h>\n"; + print '#include <vespa/log/log.h>'; + print "\n\n" . "namespace ns_log {" . "\n\n"; + + print "enum Logger::LogLevel\n"; + print "Logger::parseLevel(const char *lname)\n{\n"; + foreach $l ( @t ) { + print " if (strcmp(lname, \"$l\") == 0) return $l;\n"; + } + print " // bad level name signaled by NUM_LOGLEVELS\n"; + print " return NUM_LOGLEVELS;\n"; + print "}\n\n"; + + print "const char *Logger::logLevelNames[] = {" . "\n "; + foreach $l ( @t ) { $l = "\"$l\""; } + push @t, "0 // converting NUM_LOGLEVELS gives null pointer\n"; + print join(",\n ", @t); + print "};\n\n} // namespace\n"; + exit(0); +} diff --git a/vespalog/src/vespa/log/reject-filter.cpp b/vespalog/src/vespa/log/reject-filter.cpp new file mode 100644 index 00000000000..59155a44a6e --- /dev/null +++ b/vespalog/src/vespa/log/reject-filter.cpp @@ -0,0 +1,65 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <sys/types.h> +#include <cstring> +#include <cstdlib> + +#include "reject-filter.h" + +namespace ns_log { + +void +RejectFilter::addRejectRule(Logger::LogLevel level, const std::string & message) +{ + _rejectRules.push_back(RejectRule(level, message, false)); +} + +void +RejectFilter::addExactRejectRule(Logger::LogLevel level, const std::string & message) +{ + _rejectRules.push_back(RejectRule(level, message, true)); +} + +bool +RejectFilter::shouldReject(Logger::LogLevel level, const char * message) +{ + if (message == NULL) { + return false; + } + for (size_t i = 0; i < _rejectRules.size(); i++) { + if (_rejectRules[i].shouldReject(level, message)) { + return true; + } + } + + + return false; +} + +bool +RejectFilter::RejectRule::shouldReject(Logger::LogLevel level, const char * message) +{ + if (_level == level) { + if (_exact) { + if (strlen(message) == _message.length() && _message.compare(message) == 0) { + return true; + } + } else { + if (strstr(message, _message.c_str()) != NULL) { + return true; + } + } + } + return false; +} + +RejectFilter +RejectFilter::createDefaultFilter() +{ + RejectFilter filter; + filter.addRejectRule(Logger::warning, "Using FILTER_NONE: This must be paranoid approved, and since you are using FILTER_NONE you must live with this error."); + filter.addExactRejectRule(Logger::warning, ""); + filter.addRejectRule(Logger::warning, "yjava_preload.so: [preload.c:670] Accept failed: -1 (4)"); + return filter; +} + +} // end namespace ns_log diff --git a/vespalog/src/vespa/log/reject-filter.h b/vespalog/src/vespa/log/reject-filter.h new file mode 100644 index 00000000000..82c6d975d6d --- /dev/null +++ b/vespalog/src/vespa/log/reject-filter.h @@ -0,0 +1,37 @@ +// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/log/log.h> + +#include <vector> +#include <string> + +namespace ns_log { + +/** + * A reject filter is simply a list of log messages that should be rejected and + * not logged. + */ +class RejectFilter { +private: + class RejectRule { + private: + Logger::LogLevel _level; + std::string _message; + bool _exact; + public: + RejectRule(Logger::LogLevel level, const std::string & message, bool exact) + : _level(level), _message(message), _exact(exact) + { } + bool shouldReject(Logger::LogLevel level, const char * message); + }; + std::vector<RejectRule> _rejectRules; +public: + void addRejectRule(Logger::LogLevel level, const std::string & rejectedMessage); + void addExactRejectRule(Logger::LogLevel level, const std::string & rejectedMessage); + bool shouldReject(Logger::LogLevel level, const char * message); + static RejectFilter createDefaultFilter(); +}; + +} // end namespace ns_log + |