Rename vespalog binaries to have vespa- prefix.

Temporarily add symlinks from old name to new name.

4 files changed, 624 insertions, 0 deletions

diff --git a/vespalog/src/vespa-logfmt/.gitignore b/vespalog/src/vespa-logfmt/.gitignore
new file mode 100644
index 00000000000..568d69eee3e
--- /dev/null
+++ b/vespalog/src/vespa-logfmt/.gitignore
@@ -0,0 +1 @@
+logfilter
diff --git a/vespalog/src/vespa-logfmt/logfilter.c b/vespalog/src/vespa-logfmt/logfilter.c
new file mode 100644
index 00000000000..709ce2c016e
--- /dev/null
+++ b/vespalog/src/vespa-logfmt/logfilter.c
@@ -0,0 +1,78 @@
+// Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <signal.h>
+
+int main(int argc, char *argv[])
+{
+    int   lfpipe[2];
+    pid_t lfmtpid;
+    pid_t progpid;
+    int   wstat;
+
+    if (argc < 3) {
+        fprintf(stderr, "Usage: %s logfmt prog [...]\n", argv[0]);
+        exit(1);
+    }
+
+    pipe(lfpipe);
+    lfmtpid = fork();
+    if (lfmtpid == 0) {
+        close(lfpipe[1]);
+        if (lfpipe[0] != 0) {
+            close(0);
+            dup(lfpipe[0]);
+            close(lfpipe[0]);
+        }
+        execlp(argv[1], argv[1], "-", (const char *)NULL);
+        perror("exec logfmt failed");
+        exit(1);
+    }
+    if (lfmtpid < 0) {
+        perror("fork failed");
+        exit(1);
+    }
+    close(lfpipe[0]);
+
+    progpid = fork();
+    if (progpid == 0) {
+        char buf[20];
+        int i=2;
+
+        sprintf(buf, "fd:%d", lfpipe[1]);
+        setenv("VESPA_LOG_TARGET", buf, 1);
+        for (; i<argc; i++) {
+            argv[i-2] = argv[i];
+        }
+        argv[i-2] = NULL;
+        execvp(argv[0], argv);
+        perror("exec program failed");
+        exit(1);
+    }
+    if (progpid < 0) {
+        perror("fork failed");
+        exit(1);
+    }
+    close(lfpipe[1]);
+
+    if (waitpid(lfmtpid, &wstat, 0) != lfmtpid) {
+        perror("bad waitpid for logfmt");
+    }
+    if (waitpid(progpid, &wstat, 0) != progpid) {
+        perror("bad waitpid for program");
+    }
+
+    if (WIFEXITED(wstat)) {
+        return WEXITSTATUS(wstat);
+    }
+    if (WIFSIGNALED(wstat)) {
+        int sig = WTERMSIG(wstat);
+        signal(sig, SIG_DFL);
+        kill(getpid(), sig);
+    }
+    /* should not get here */
+    exit(1);
+}
diff --git a/vespalog/src/vespa-logfmt/vespa-logfmt.1 b/vespalog/src/vespa-logfmt/vespa-logfmt.1
new file mode 100644
index 00000000000..57c8d46370e
--- /dev/null
+++ b/vespalog/src/vespa-logfmt/vespa-logfmt.1
@@ -0,0 +1,218 @@
+.\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $
+.\"
+.Dd Jun 7, 2017
+.Dt vespa-logfmt \&1 "Vespa documentation"
+.Os "Yahoo! Vespa" "6.0"
+.Os
+.Sh NAME
+.Nm vespa-logfmt
+.Nd select and format messages from Vespa log files
+.Sh SYNOPSIS
+.Nm
+.Op Fl l Ar levellist
+.Op Fl s Ar fieldlist
+.Op Fl p Ar pid
+.Op Fl S Ar service
+.Op Fl H Ar host
+.Op Fl c Ar regex
+.Op Fl m Ar regex
+.Op Fl t
+.Op Fl f
+.Op Fl L
+.Op Fl N
+.Op Fl ts
+.Op Ar
+.Sh DESCRIPTION
+The
+.Nm
+utility reads Vespa log files, select messages and writes a formatted
+version of selected messages to the standard output.
+.Pp
+The options are as follows:
+.Bl -tag -width ".It Fl l Ar levellist"
+.It Fl l Ar levellist
+Select which log levels to select.
+The default is to select "fatal", "error", "warning" and "info"
+levels, and suppress "config", "event", "debug" and "spam" levels;
+but when using this option, only the named levels will be selected.
+The
+.Ar levellist
+is a comma separated list of level names.
+The name
+.Em all
+may be used to add all known levels.
+Prepending a minus sign will deselect the level named.
+Starting the list with a plus sign will add and remove levels
+from the current (or default) list of levels instead
+of replacing it.
+.It Fl s Ar fieldlist
+Select which fields of log messages to show.
+The order of the actual output fields is fixed.
+When using this option, only the named fields will be shown.  The
+fieldlist is a comma separated list of field names.  The name
+.Em all
+may be used to add all possible fields.
+Prepending a minus sign will turn off display of the named field.
+Starting the list with a plus sign will add and remove fields
+from the current (or default) list of fields instead
+of replacing it.
+.Pp
+The fields which may be named are:
+.Bl -tag -width component
+.It time
+Print the time in seconds since the epoch.
+Ignored if
+.Em fmttime
+is shown.
+.It fmttime
+Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format.
+Note that the time is printed in the local timezone; to get GMT
+output use
+.Nm "\*[q]env TZ=GMT vespa-logfmt\*[q]"
+as your command.
+.It msecs
+Add milliseconds after the seconds in
+.Em time
+and
+.Em fmttime
+output.  Ignored if
+.Em usecs
+is in effect.
+.It usecs
+Add microseconds after the seconds in
+.Em time
+and
+.Em fmttime
+output.
+.It host
+Print the hostname field.
+.It level
+Print the level field (uppercased).
+.It pid
+Print the pid field.
+.It service
+Print the service field.
+.It component
+Print the component field.
+.It message
+Print the message text field.
+You probably always want to add this.
+.El
+.Pp
+Using this option several times works as if the given
+.Ar fieldlist
+arguments had been concatenated into one comma-separated list.
+The default fields to show are as if
+.Bk
+.Op Fl s Ar fmttime,msecs,level,service,component,message
+.Ek
+had been given.
+.It Fl p Ar pid
+Select only messages where the pid field matches the
+.Ar pid
+string exactly.
+.It Fl S Ar service
+Select only messages where the service field matches the
+.Ar service
+string exactly.
+.It Fl H Ar host
+Select only messages where the hostname field matches the
+.Ar host
+string exactly.
+.It Fl c Ar regex
+Select only messages where the component field matches the
+.Ar regex
+given, using
+.Xr perlre
+regular expression matching.
+.It Fl m Ar regex
+Select only messages where the message text field matches the
+.Ar regex
+given, using
+.Xr perlre
+regular expression matching.
+.It Fl f
+Invoke tail -F to follow input file
+.It Fl L
+.It Fl -livestream
+Only useful for multi-machine setups:
+Invoke vespa-replicate-log-stream to follow live stream from log server,
+instead of reading any input file.  Also implies
+.Bk
+.Fl "s +host"
+.Ek
+as default.
+.It Fl N
+Dequote quoted newlines in the message text field to an actual newline plus tab.
+.It Fl t
+Format the component field (if shown) as a fixed-with string,
+truncating if necessary.
+.It Fl ts
+Format the service field (if shown) as a fixed-with string,
+truncating if necessary.
+.El
+.Sh EXAMPLES
+The command:
+.Pp
+.Bd -literal -offset indent
+vespa-logfmt -l event -s service,message,fmttime,message
+.Ed
+.Pp
+will display only messages with log level "event",
+printing a human-readable time (without any fractional seconds),
+the service generating the event and the event message, like this:
+.Bd -literal -offset indent
+[2004-12-07 18:43:01] config-sentinel  starting/1 name="logd"
+[2004-12-07 18:43:01] logd             started/1 name="logdemon"
+[2004-12-07 18:45:51] rtc              starting/1 name="rtc.index0"
+[2004-12-07 18:45:51] rtc.index0       started/1 name="flexindexer.index"
+[2004-12-07 18:45:51] rtc.index0       stopping/1 name="flexindexer.index" why="done"
+[2004-12-07 18:45:53] rtc              stopped/1 name="rtc.index0" pid=50600 exitcode=0
+[2004-12-07 18:46:13] logd             stopping/1 name="logdemon" why="done ok."
+[2004-12-07 18:46:13] config-sentinel  stopped/1 name="logd" pid=49633 exitcode=0
+.Ed
+.Pp
+Note that the second "message" item in the fieldlist is redundant,
+and that order of printed field is fixed no matter what the fieldlist
+order is.
+.Pp
+The command:
+.Pp
+.Bd -literal -offset indent
+vespa-logfmt -l all-info,-debug -s level \e
+       -s time,usecs,component,message -t -l -event
+.Ed
+.Pp
+will display messages with log levels that are
+.Em not
+any of
+.Em info, debug,
+or
+.Em event,
+printing the time in seconds and microseconds, the log level, the
+component name, and the message text, possibly somewhat like this:
+.Bd -literal -offset indent
+1102441382.530423 CONFIG  : nc              Config handle: 'pandora.0-rtx'
+1102441551.471568 CONFIG  : flexindexer.doc Adding document type typetest-0
+1102441573.148211 WARNING : logdemon        stopping on signal 15
+1102441887.158000 WARNING : com.yahoo.fs4.m read exception
+1102441935.569567 WARNING : rtc             Dispatch inherited job failed for dir dispatch0
+1102442115.746001 WARNING : fdispatch       Search node 172.24.94.75:10124 down
+1102442474.205920 WARNING : rtx             RTC (tcp/172.24.94.75:10161) : DOWN
+1102442474.515877 WARNING : fdispatch       Search node localhost:10128 down
+1102442983.075669 ERROR   : flexindexer.std Unable to find cluster map defaultcluster
+.Ed
+.Sh FILES
+If no file argument is given,
+.Nm
+will read the last vespa log file $VESPA_HOME/logs/vespa/vespa.log (this also works with the
+.Fl f
+option).
+Otherwise, reads only the files given as arguments.
+To read standard input, supply a single dash '-' as a file argument.
+.Sh SEE ALSO
+Documentation in the "log" module for input file format.
+.Sh HISTORY
+Developed as part of Vespa 1.1.  The default output format
+reflects the old "fastlib" log formatting, with minor differences
+and is intended to be human-readable, not parsed.
diff --git a/vespalog/src/vespa-logfmt/vespa-logfmt.pl b/vespalog/src/vespa-logfmt/vespa-logfmt.pl
new file mode 100755
index 00000000000..8d01feb3a6b
--- /dev/null
+++ b/vespalog/src/vespa-logfmt/vespa-logfmt.pl
@@ -0,0 +1,327 @@
+#!/usr/bin/env perl
+# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+# BEGIN perl environment bootstrap section
+# Do not edit between here and END as this section should stay identical in all scripts
+
+use File::Basename;
+use File::Path;
+
+sub findpath {
+    my $myfullname = ${0};
+    my($myname, $mypath) = fileparse($myfullname);
+
+    return $mypath if ( $mypath && -d $mypath );
+    $mypath=`pwd`;
+
+    my $pwdfullname = $mypath . "/" . $myname;
+    return $mypath if ( -f $pwdfullname );
+    return 0;
+}
+
+# Returns the argument path if it seems to point to VESPA_HOME, 0 otherwise
+sub is_vespa_home {
+    my($VESPA_HOME) = shift;
+    my $COMMON_ENV="libexec/vespa/common-env.sh";
+    if ( $VESPA_HOME && -d $VESPA_HOME ) {
+        my $common_env = $VESPA_HOME . "/" . $COMMON_ENV;
+        return $VESPA_HOME if -f $common_env;
+    }
+    return 0;
+}
+
+# Returns the home of Vespa, or dies if it cannot
+sub findhome {
+    # Try the VESPA_HOME env variable
+    return $ENV{'VESPA_HOME'} if is_vespa_home($ENV{'VESPA_HOME'});
+    if ( $ENV{'VESPA_HOME'} ) { # was set, but not correctly
+        die "FATAL: bad VESPA_HOME value '" . $ENV{'VESPA_HOME'} . "'\n";
+    }
+
+    # Try the ROOT env variable
+    $ROOT = $ENV{'ROOT'};
+    return $ROOT if is_vespa_home($ROOT);
+
+    # Try the script location or current dir
+    my $mypath = findpath();
+    if ($mypath) {
+        while ( $mypath =~ s|/[^/]*$|| ) {
+            return $mypath if is_vespa_home($mypath);
+        }
+    }
+    die "FATAL: Missing VESPA_HOME environment variable\n";
+}
+
+BEGIN {
+    my $tmp = findhome();
+    if ( $tmp !~ m{[/]$} ) { $tmp .= "/"; }
+    $ENV{'VESPA_HOME'} = $tmp;
+}
+my $VESPA_HOME = $ENV{'VESPA_HOME'};
+
+# END perl environment bootstrap section
+
+use 5.006_001;
+use strict;
+use warnings;
+
+use Getopt::Long qw(:config no_ignore_case);
+
+my %showflags = (
+	time => 1,
+	fmttime => 1,
+	msecs => 1,
+	usecs => 0,
+	host => 0,
+	level => 1,
+	pid => 0,
+	service => 1,
+	component => 1,
+	message => 1
+);
+
+my %levelflags = (
+	fatal => 1,
+	error => 1,
+	warning => 1,
+	info => 1,
+	config => 0,
+	event => 0,
+	debug => 0,
+	spam => 0
+);
+
+# Do not buffer the output
+$| = 1;
+
+my $compore;
+my $msgtxre;
+my $onlypid;
+my $onlysvc;
+my $onlyhst;
+
+my $shortsvc;
+my $shortcmp;
+
+my @optlevels;
+my @optshow;
+my $optlevels;
+my $optfollow;
+my $optnldequote;
+my $optreplication;
+my $opthelp = '';
+
+my $bad = 0;
+
+GetOptions ('level|l=s' => \@optlevels,
+            'service|S=s'  => \$onlysvc,
+            'show|s=s'  => \@optshow,
+            'pid|p=s'  => \$onlypid,
+            'component|c=s'  => \$compore,
+            'message|m=s'  => \$msgtxre,
+            'help|h' => \$opthelp,
+            'follow|f' => \$optfollow,
+            'livestream|L' => \$optreplication,
+            'nldequote|N' => \$optnldequote,
+            'host|H=s' => \$onlyhst,
+            'truncateservice|ts' => \$shortsvc,
+            'truncatecomponent|tc|t' => \$shortcmp,
+) or $bad=1;
+
+if ( $optreplication ) {
+        open(STDIN, "vespa-replicate-log-stream |");
+        $showflags{'host'} = 1;
+} elsif ( @ARGV == 0 and ! -p STDIN) {
+        push(@ARGV, "$VESPA_HOME/logs/vespa/vespa.log");
+}
+
+if ( $optfollow ) {
+    my $filearg = "";
+    if ( @ARGV > 1 ) {
+	print STDERR "ERROR: Cannot follow more than one file\n\n";
+	$bad=1;
+    } else {
+	$filearg = shift @ARGV if (@ARGV > 0);
+	open(STDIN, "tail -F $filearg |")
+	    or die "cannot open 'tail -F $filearg' as input pipe\n";
+    }
+}
+
+if ( $opthelp || $bad ) {
+    print STDERR "Usage: $0 [options] [inputfile ...]\n",
+	"Options:\n",
+	"  -l LEVELLIST\t--level=LEVELLIST\tselect levels to include\n",
+	"  -s FIELDLIST\t--show=FIELDLIST\tselect fields to print\n",
+	"  -p PID\t--pid=PID\t\tselect messages from given PID\n",
+	"  -S SERVICE\t--service=SERVICE\tselect messages from given SERVICE\n",
+	"  -H HOST\t--host=HOST\t\tselect messages from given HOST\n",
+	"  -c REGEX\t--component=REGEX\tselect components matching REGEX\n",
+	"  -m REGEX\t--message=REGEX\t\tselect message text matching REGEX\n",
+	"  -f\t\t--follow\t\tinvoke tail -F to follow input file\n",
+        "  -L\t\t--livestream\t\tfollow log stream from logserver\n",
+	"  -N\t\t--nldequote\t\tdequote newlines in message text field\n",
+	"  -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n",
+	"  --ts\t\t--truncateservice\tchop service to 9 chars\n",
+	"\n",
+	"FIELDLIST is comma separated, available fields:\n",
+	"\t time fmttime msecs usecs host level pid service component message\n",
+	"Available levels for LEVELLIST:\n",
+	"\t fatal error warning info event debug spam\n",
+	"for both lists, use 'all' for all possible values, and -xxx to disable xxx.\n";
+    exit $bad;
+}
+
+
+$optlevels = join(",", @optlevels );
+if ( $optlevels ) {
+	my $k;
+	unless ( $optlevels =~ s/^\+// or $optlevels =~ m/^-/ ) {
+		$levelflags{$_} = 0 foreach ( keys %levelflags );
+	}
+	my @l = split(/,|(?=-)/, $optlevels);
+	my $l;
+	foreach $l ( @l ) {
+		my $v = 1;
+		my $minus = "";
+		if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; }
+		if ( $l eq "all" ) {
+			foreach $k ( keys %levelflags ) {
+				$levelflags{$k} = $v;
+			}
+		} elsif ( defined $levelflags{$l} ) {
+			$levelflags{$l} = $v;
+		} else {
+			print STDERR "bad level option '$minus$l'\n";
+			exit 1;
+		}
+	}
+# 	print STDERR "select level $_ => $levelflags{$_}\n"
+# 		foreach ( keys %levelflags );
+}
+
+my $optshow;
+$optshow = join(",", @optshow );
+if ( $optshow ) {
+	my $k;
+	unless ( $optshow =~ s/^\+// or $optshow =~ m/^-/ ) {
+		$showflags{$_} = 0 foreach ( keys %showflags );
+	}
+	my @l = split(/,|(?=-)/, $optshow);
+	my $l;
+	foreach $l ( @l ) {
+		my $v = 1;
+		my $minus = "";
+		if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; }
+		if ( $l eq "all" ) {
+			foreach $k ( keys %showflags ) {
+				$showflags{$k} = $v;
+			}
+		} elsif ( defined $showflags{$l} ) {
+			$showflags{$l} = $v;
+		} else {
+			print STDERR "bad show option '$minus$l'\n";
+			exit 1;
+		}
+	}
+#	print STDERR "show field $_ => $showflags{$_}\n"
+#		foreach ( keys %showflags ) ;
+}
+
+while (<>) {
+	chomp;
+	if ( /^
+		(\d+)\.?(\d*)	# seconds, optional fractional seconds
+		\t
+		([^\t]*)	# host
+		\t
+		(\d+\/?\d*|\-\/\d+)	# pid, optional tid
+		\t
+		([^\t]*)	# servicename
+		\t
+		([^\t]*)	# componentname
+		\t
+		(\w+)		# level
+		\t
+		(.*)		# message text
+             $/x )
+	{
+		my $secs = $1;
+		my $usec = $2 . "000000";  # make sure we have atleast 6 digits
+		my $host = $3;
+		my $pidn = $4;
+		my $svcn = $5;
+		my $comp = $6;
+		my $levl = $7;
+		my $msgt = $8;
+
+		if ( ! defined $levelflags{$levl} ) {
+			print STDERR "Warning: unknown level '$levl' in input\n";
+			$levelflags{$levl} = 1;
+		}
+		next unless ( $levelflags{$levl} );
+
+		if ($compore && $comp !~ m/$compore/o) { next; }
+		if ($msgtxre && $msgt !~ m/$msgtxre/o) { next; }
+		if ($onlypid && $pidn ne $onlypid) { next; }
+		if ($onlysvc && $svcn ne $onlysvc) { next; }
+		if ($onlyhst && $host ne $onlyhst) { next; }
+
+		$levl = "\U$levl";
+
+		my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday);
+		($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($secs);
+		my $datestr = sprintf("%04d-%02d-%02d",
+				      1900+$year, 1+$mon, $mday);
+		my $timestr = sprintf("%02d:%02d:%02d",
+				      $hour, $min, $sec);
+
+		if ( $showflags{"time"} || $showflags{"fmttime"} ) {
+			if ($showflags{"fmttime"} ) {
+				print "[$datestr $timestr";
+				if ( $showflags{"usecs"} ) {
+					printf ".%.6s", $usec;
+				} elsif ( $showflags{"msecs"} ) {
+					printf ".%.3s", $usec;
+				}
+				print "] ";
+			} else {
+				printf "%s.%.6s ", $secs, $usec;
+			}
+		}
+		if ( $showflags{"host"} ) {
+			printf "%-8s ", $host;
+		}
+		if ( $showflags{"level"} ) {
+			printf "%-7s : ", $levl;
+		}
+		if ( $showflags{"pid"} ) {
+			printf "%5s ", $pidn;
+		}
+		if ( $showflags{"service"} ) {
+			if ( $shortsvc ) {
+				printf "%-9.9s ", $svcn;
+			} else {
+				printf "%-16s ", $svcn;
+			}
+		}
+		if ( $showflags{"component"} ) {
+			if ( $shortcmp ) {
+				printf "%-15.15s ", $comp;
+			} else {
+				printf "%s\t", $comp;
+			}
+		}
+		if ( $showflags{"message"} ) {
+			if ( $optnldequote ) {
+				my $did_dequote_1 = ( $msgt =~ s/\\n\\t/\n\t/g );
+				my $did_dequote_2 = ( $msgt =~ s/\\n/\n\t/g );
+				$msgt = "\n\t${msgt}" if ( $did_dequote_1 || $did_dequote_2 );
+				$msgt =~ s/\\t/\t/;
+			}
+			print $msgt;
+		}
+		print "\n";
+	} else {
+		print STDERR "bad log line: '$_'\n";
+	}
+}