diff options
Diffstat (limited to 'jdisc_core/src/main/perl')
-rwxr-xr-x | jdisc_core/src/main/perl/vespa-jdisc-logfmt | 277 | ||||
-rw-r--r-- | jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 | 215 |
2 files changed, 0 insertions, 492 deletions
diff --git a/jdisc_core/src/main/perl/vespa-jdisc-logfmt b/jdisc_core/src/main/perl/vespa-jdisc-logfmt deleted file mode 100755 index 655fd8da24c..00000000000 --- a/jdisc_core/src/main/perl/vespa-jdisc-logfmt +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env perl -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -sub findhome { - # Try the VESPA_HOME env variable - return $ENV{'VESPA_HOME'} if defined $ENV{'VESPA_HOME'}; - if ( $0 =~ m{(.*)/bin[^/]*/[^/]*logfmt[^/]*$} ) { - return $1; - } - return "/opt/vespa"; -} - -my $VESPA_HOME = findhome(); - -# TODO: Remove on Vespa 7 - -use 5.006_001; -use strict; -use warnings; - -use File::Basename; -use Getopt::Long qw(:config no_ignore_case); - -my %showflags = ( - time => 1, - fmttime => 1, - msecs => 1, - usecs => 0, - host => 0, - level => 1, - pid => 0, - service => 1, - component => 1, - message => 1 - ); - -my %levelflags = ( - error => 1, - warning => 1, - info => 1, - debug => 0, - unknown => 0 - ); - -# Do not buffer the output -$| = 1; - -my $compore; -my $msgtxre; -my $onlypid; -my $onlysvc; -my $onlyhst; - -my $shortsvc; -my $shortcmp; - -my @optlevels; -my @optaddlevels; -my @optshow; -my $optaddlevels; -my $optlevels; -my $optfollow; -my $optnldequote; -my $opthelp = ''; - -my $bad = 0; - -GetOptions ('level|l=s' => \@optlevels, - 'add-level|L=s' => \@optaddlevels, - 'service|S=s' => \$onlysvc, - 'show|s=s' => \@optshow, - 'pid|p=s' => \$onlypid, - 'component|c=s' => \$compore, - 'message|m=s' => \$msgtxre, - 'help|h' => \$opthelp, - 'follow|f' => \$optfollow, - 'nldequote|N' => \$optnldequote, - 'host|H=s' => \$onlyhst, - 'truncateservice|ts' => \$shortsvc, - 'truncatecomponent|tc|t' => \$shortcmp, - ) or $bad=1; - -if ( @ARGV == 0 and ! -p STDIN) { - push(@ARGV, "$VESPA_HOME/logs/jdisc_core/jdisc_core.log"); -} - -if ( $optfollow ) { - my $filearg = ""; - if ( @ARGV > 1 ) { - print STDERR "ERROR: Cannot follow more than one file\n\n"; - $bad=1; - } else { - $filearg = shift @ARGV if (@ARGV > 0); - open(STDIN, "tail -F $filearg |") - or die "cannot open 'tail -F $filearg' as input pipe\n"; - } -} - -$optaddlevels = join(",", @optaddlevels ); -if ( $optaddlevels ) { - my @l = split(/,/, $optaddlevels); - my $l; - foreach $l ( @l ) { - $levelflags{$l} = 0; - } -} - -if ( $opthelp || $bad ) { - print STDERR "Usage: ", basename($0), " [options] [inputfile ...]\n", - "Options:\n", - " -l LEVELLIST\t--level=LEVELLIST\tselect levels to include\n", - " -L LEVELLIST\t--add-level=LEVELLIST\tdefine extra levels\n", - " -s FIELDLIST\t--show=FIELDLIST\tselect fields to print\n", - " -p PID\t--pid=PID\t\tselect messages from given PID\n", - " -S SERVICE\t--service=SERVICE\tselect messages from given SERVICE\n", - " -H HOST\t--host=HOST\t\tselect messages from given HOST\n", - " -c REGEX\t--component=REGEX\tselect components matching REGEX\n", - " -m REGEX\t--message=REGEX\t\tselect message text matching REGEX\n", - " -f\t\t--follow\t\tinvoke tail -F to follow input file\n", - " -N\t\t--nldequote\t\tdequote newlines in message text field\n", - " -t\t--tc\t--truncatecomponent\tchop component to 15 chars\n", - " \t--ts\t--truncateservice\tchop service to 9 chars\n", - "\n", - "FIELDLIST is comma separated, available fields:\n", - "\t time fmttime msecs usecs host level pid service component message\n", - "Available levels for LEVELLIST:\n", - "\t ", join(" ", sort keys(%levelflags)), "\n", - "for both lists, use 'all' for all possible values, and -xxx to disable xxx.\n"; - exit $bad; -} - -$optlevels = join(",", @optlevels ); -if ( $optlevels ) { - my $k; - unless ( $optlevels =~ s/^\+// or $optlevels =~ m/^-/ ) { - $levelflags{$_} = 0 foreach ( keys %levelflags ); - } - my @l = split(/,|(?=-)/, $optlevels); - my $l; - foreach $l ( @l ) { - my $v = 1; - my $minus = ""; - if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } - if ( $l eq "all" ) { - foreach $k ( keys %levelflags ) { - $levelflags{$k} = $v; - } - } elsif ( defined $levelflags{$l} ) { - $levelflags{$l} = $v; - } else { - print STDERR "bad level option '$minus$l'\n"; - exit 1; - } - } -} - -my $optshow; -$optshow = join(",", @optshow ); -if ( $optshow ) { - my $k; - unless ( $optshow =~ s/^\+// or $optshow =~ m/^-/ ) { - $showflags{$_} = 0 foreach ( keys %showflags ); - } - my @l = split(/,|(?=-)/, $optshow); - my $l; - foreach $l ( @l ) { - my $v = 1; - my $minus = ""; - if ( $l =~ s/^-// ) { $v = 0; $minus = "-"; } - if ( $l eq "all" ) { - foreach $k ( keys %showflags ) { - $showflags{$k} = $v; - } - } elsif ( defined $showflags{$l} ) { - $showflags{$l} = $v; - } else { - print STDERR "bad show option '$minus$l'\n"; - exit 1; - } - } -} - -while (<>) { - chomp; - if ( /^ - (\d+)\.?(\d*) # seconds, optional fractional seconds - \t - ([^\t]*) # host - \t - (\d+\/?\d*|\-\/\d+) # pid, optional tid - \t - ([^\t]*) # servicename - \t - ([^\t]*) # componentname - \t - (\w+) # level - \t - (.*) # message text - $/x ) - { - my $secs = $1; - my $usec = $2 . "000000"; # make sure we have atleast 6 digits - my $host = $3; - my $pidn = $4; - my $svcn = $5; - my $comp = $6; - my $levl = $7; - my $msgt = $8; - - if ( ! defined $levelflags{$levl} ) { - print STDERR "Warning: unknown level '$levl' in input\n"; - $levelflags{$levl} = 1; - } - next unless ( $levelflags{$levl} ); - - if ($compore && $comp !~ m/$compore/o) { next; } - if ($msgtxre && $msgt !~ m/$msgtxre/o) { next; } - if ($onlypid && $pidn ne $onlypid) { next; } - if ($onlysvc && $svcn ne $onlysvc) { next; } - if ($onlyhst && $host ne $onlyhst) { next; } - - $levl = "\U$levl"; - - my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday); - ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday)=localtime($secs); - my $datestr = sprintf("%04d-%02d-%02d", - 1900+$year, 1+$mon, $mday); - my $timestr = sprintf("%02d:%02d:%02d", - $hour, $min, $sec); - - if ( $showflags{"time"} || $showflags{"fmttime"} ) { - if ($showflags{"fmttime"} ) { - print "[$datestr $timestr"; - if ( $showflags{"usecs"} ) { - printf ".%.6s", $usec; - } elsif ( $showflags{"msecs"} ) { - printf ".%.3s", $usec; - } - print "] "; - } else { - printf "%s.%.6s ", $secs, $usec; - } - } - if ( $showflags{"host"} ) { - printf "%-8s ", $host; - } - if ( $showflags{"level"} ) { - printf "%-7s : ", $levl; - } - if ( $showflags{"pid"} ) { - printf "%5s ", $pidn; - } - if ( $showflags{"service"} ) { - if ( $shortsvc ) { - printf "%-9.9s ", $svcn; - } else { - printf "%-16s ", $svcn; - } - } - if ( $showflags{"component"} ) { - if ( $shortcmp ) { - printf "%-15.15s ", $comp; - } else { - printf "%s\t", $comp; - } - } - if ( $showflags{"message"} ) { - if ( $optnldequote ) { - $msgt = "\n\t${msgt}" if ( $msgt =~ s/\\n/\n\t/g ); - } - print $msgt; - } - print "\n"; - } else { - print STDERR "bad log line: '$_'\n"; - } -} diff --git a/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 b/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 deleted file mode 100644 index 2f855d94207..00000000000 --- a/jdisc_core/src/main/perl/vespa-jdisc-logfmt.1 +++ /dev/null @@ -1,215 +0,0 @@ -.\" Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -.\" $Id: logfmt.1,v 1.12 2007-06-19 09:37:25 daljord Exp $ -.\" -.Dd October 29, 2004 -.Dt VESPA-JDISC-LOGFMT \&1 "JDisc documentation" -.Os "Yahoo! JDisc" "2.3" -.Os -.Sh NAME -.Nm vespa-jdisc-logfmt -.Nd select and format messages from JDisc log files -.Sh SYNOPSIS -.Nm -.Op Fl L Ar levellist -.Op Fl l Ar levellist -.Op Fl s Ar fieldlist -.Op Fl p Ar pid -.Op Fl S Ar service -.Op Fl H Ar host -.Op Fl c Ar regex -.Op Fl m Ar regex -.Op Fl t -.Op Fl f -.Op Fl N -.Op Fl ts -.Op Ar -.Sh DESCRIPTION -The -.Nm -utility reads JDisc log files, select messages and writes a formatted -version of selected messages to the standard output. -.Pp -The options are as follows: -.Bl -tag -width ".It Fl l Ar levellist" -.It Fl L Ar levellist -Declares additional log levels that should be treated as known. These -levels are suppressed unless also given as argument to option -l. -.Ar levellist -is a comma separated list of level names. -.It Fl l Ar levellist -Select which log levels to select. -The default is to select "error", "warning" and "info" levels, and -suppress "debug" and "unknown" levels; but when using this option, only -the named levels will be selected. -The -.Ar levellist -is a comma separated list of level names. -The name -.Em all -may be used to add all known levels. -Prepending a minus sign will deselect the level named. -Starting the list with a plus sign will add and remove levels -from the current (or default) list of levels instead -of replacing it. -.It Fl s Ar fieldlist -Select which fields of log messages to show. -The order of the actual output fields is fixed. -When using this option, only the named fields will be shown. The -fieldlist is a comma separated list of field names. The name -.Em all -may be used to add all possible fields. -Prepending a minus sign will turn off display of the named field. -Starting the list with a plus sign will add and remove fields -from the current (or default) list of fields instead -of replacing it. -.Pp -The fields which may be named are: -.Bl -tag -width component -.It time -Print the time in seconds since the epoch. -Ignored if -.Em fmttime -is shown. -.It fmttime -Print the time in human-readable [YYYY-MM-DD HH:mm:ss] format. -Note that the time is printed in the local timezone; to get GMT -output use -.Nm "\*[q]env TZ=GMT vespa-jdisc-logfmt\*[q]" -as your command. -.It msecs -Add milliseconds after the seconds in -.Em time -and -.Em fmttime -output. Ignored if -.Em usecs -is in effect. -.It usecs -Add microseconds after the seconds in -.Em time -and -.Em fmttime -output. -.It host -Print the hostname field. -.It level -Print the level field (uppercased). -.It pid -Print the pid field. -.It service -Print the service field. -.It component -Print the component field. -.It message -Print the message text field. -You probably always want to add this. -.El -.Pp -Using this option several times works as if the given -.Ar fieldlist -arguments had been concatenated into one comma-separated list. -The default fields to show are as if -.Bk -.Op Fl s Ar fmttime,msecs,level,service,component,message -.Ek -had been given. -.It Fl p Ar pid -Select only messages where the pid field matches the -.Ar pid -string exactly. -.It Fl S Ar service -Select only messages where the service field matches the -.Ar service -string exactly. -.It Fl H Ar host -Select only messages where the hostname field matches the -.Ar host -string exactly. -.It Fl c Ar regex -Select only messages where the component field matches the -.Ar regex -given, using -.Xr perlre -regular expression matching. -.It Fl m Ar regex -Select only messages where the message text field matches the -.Ar regex -given, using -.Xr perlre -regular expression matching. -.It Fl f -Invoke tail -F to follow input file -.It Fl N -Dequote quoted newlines in the message text field to an actual newline plus tab. -.It Fl t -Format the component field (if shown) as a fixed-with string, -truncating if necessary. -.It Fl ts -Format the service field (if shown) as a fixed-with string, -truncating if necessary. -.El -.Sh EXAMPLES -The command: -.Pp -.Bd -literal -offset indent -vespa-jdisc-logfmt -l event -s service,message,fmttime,message -.Ed -.Pp -will display only messages with log level "event", -printing a human-readable time (without any fractional seconds), -the service generating the event and the event message, like this: -.Bd -literal -offset indent -[2004-12-07 18:43:01] config-sentinel starting/1 name="logd" -[2004-12-07 18:43:01] logd started/1 name="logdemon" -[2004-12-07 18:45:51] rtc starting/1 name="rtc.index0" -[2004-12-07 18:45:51] rtc.index0 started/1 name="flexindexer.index" -[2004-12-07 18:45:51] rtc.index0 stopping/1 name="flexindexer.index" why="done" -[2004-12-07 18:45:53] rtc stopped/1 name="rtc.index0" pid=50600 exitcode=0 -[2004-12-07 18:46:13] logd stopping/1 name="logdemon" why="done ok." -[2004-12-07 18:46:13] config-sentinel stopped/1 name="logd" pid=49633 exitcode=0 -.Ed -.Pp -Note that the second "message" item in the fieldlist is redundant, -and that order of printed field is fixed no matter what the fieldlist -order is. -.Pp -The command: -.Pp -.Bd -literal -offset indent -vespa-jdisc-logfmt -l all-info,-debug -s level \e - -s time,usecs,component,message -t -l -event -.Ed -.Pp -will display messages with log levels that are -.Em not -any of -.Em info, debug, -or -.Em event, -printing the time in seconds and microseconds, the log level, the -component name, and the message text, possibly somewhat like this: -.Bd -literal -offset indent -1102441382.530423 CONFIG : nc Config handle: 'pandora.0-rtx' -1102441551.471568 CONFIG : flexindexer.doc Adding document type typetest-0 -1102441573.148211 WARNING : logdemon stopping on signal 15 -1102441887.158000 WARNING : com.yahoo.fs4.m read exception -1102441935.569567 WARNING : rtc Dispatch inherited job failed for dir dispatch0 -1102442115.746001 WARNING : fdispatch Search node 172.24.94.75:10124 down -1102442474.205920 WARNING : rtx RTC (tcp/172.24.94.75:10161) : DOWN -1102442474.515877 WARNING : fdispatch Search node localhost:10128 down -1102442983.075669 ERROR : flexindexer.std Unable to find cluster map defaultcluster -.Ed -.Sh FILES -If no file argument is given, -.Nm -will read the last JDisc log file $VESPA_HOME/logs/jdisc_core/jdisc_core.log (this also works with the -.Fl f -option). -Otherwise, reads only the files given as arguments. -To read standard input, supply a single dash '-' as a file argument. -.Sh SEE ALSO -Documentation in the "log" module for input file format. -.Sh HISTORY -Developed as part of Vespa 1.1, later moved to JDisc 2.3. The default output -format reflects the old "fastlib" log formatting, with minor differences -and is intended to be human-readable, not parsed. |