From f78419ed329923a8c966a611a876bd97108afed4 Mon Sep 17 00:00:00 2001 From: Eirik Nygaard Date: Thu, 6 Oct 2022 11:34:32 +0200 Subject: Add support for different output formats in logfmt Support vespa (default, json and raw lines --- client/go/cmd/logfmt/cmd.go | 1 + client/go/cmd/logfmt/formatflags.go | 41 ++++++++++ client/go/cmd/logfmt/formatflags_test.go | 30 ++++++++ client/go/cmd/logfmt/handleline.go | 127 +++++++++++++++++++++++-------- client/go/cmd/logfmt/options.go | 1 + 5 files changed, 168 insertions(+), 32 deletions(-) create mode 100644 client/go/cmd/logfmt/formatflags.go create mode 100644 client/go/cmd/logfmt/formatflags_test.go diff --git a/client/go/cmd/logfmt/cmd.go b/client/go/cmd/logfmt/cmd.go index 54e82844a30..b549c5fa7ef 100644 --- a/client/go/cmd/logfmt/cmd.go +++ b/client/go/cmd/logfmt/cmd.go @@ -38,6 +38,7 @@ and converts it to something human-readable`, cmd.Flags().StringVarP(&curOptions.OnlyHostname, "host", "H", "", "select only one host") cmd.Flags().StringVarP(&curOptions.OnlyPid, "pid", "p", "", "select only one process ID") cmd.Flags().StringVarP(&curOptions.OnlyService, "service", "S", "", "select only one service") + cmd.Flags().VarP(&curOptions.Format, "format", "F", "select logfmt output format, vespa (default), json or raw are supported") cmd.Flags().MarkHidden("tc") cmd.Flags().MarkHidden("ts") cmd.Flags().MarkHidden("dequotenewlines") diff --git a/client/go/cmd/logfmt/formatflags.go b/client/go/cmd/logfmt/formatflags.go new file mode 100644 index 00000000000..097746d696f --- /dev/null +++ b/client/go/cmd/logfmt/formatflags.go @@ -0,0 +1,41 @@ +package logfmt + +import ( + "fmt" + "strings" +) + +type OutputFormat int + +const ( + FormatVespa OutputFormat = iota //default is vespa + FormatRaw + FormatJSON +) + +func (v *OutputFormat) Type() string { + return "output format" +} + +func (v *OutputFormat) String() string { + flagNames := []string{ + "vespa", + "raw", + "json", + } + return flagNames[*v] +} + +func (v *OutputFormat) Set(val string) error { + switch strings.ToLower(val) { + case "vespa": + *v = FormatVespa + case "raw": + *v = FormatRaw + case "json": + *v = FormatJSON + default: + return fmt.Errorf("'%s' is not a valid format argument", val) + } + return nil +} diff --git a/client/go/cmd/logfmt/formatflags_test.go b/client/go/cmd/logfmt/formatflags_test.go new file mode 100644 index 00000000000..53c47d24208 --- /dev/null +++ b/client/go/cmd/logfmt/formatflags_test.go @@ -0,0 +1,30 @@ +package logfmt + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestOutputFormat(t *testing.T) { + type args struct { + val string + } + tests := []struct { + expected OutputFormat + arg string + wantErr assert.ErrorAssertionFunc + }{ + {FormatVespa, "vespa", assert.NoError}, + {FormatRaw, "raw", assert.NoError}, + {FormatJSON, "json", assert.NoError}, + {-1, "foo", assert.Error}, + } + for _, tt := range tests { + t.Run(tt.arg, func(t *testing.T) { + var v OutputFormat = -1 + tt.wantErr(t, v.Set(tt.arg), fmt.Sprintf("Set(%v)", tt.arg)) + assert.Equal(t, v, tt.expected) + }) + } +} diff --git a/client/go/cmd/logfmt/handleline.go b/client/go/cmd/logfmt/handleline.go index 33a1a1b386b..813ca82acb4 100644 --- a/client/go/cmd/logfmt/handleline.go +++ b/client/go/cmd/logfmt/handleline.go @@ -5,58 +5,122 @@ package logfmt import ( + "bytes" + "encoding/json" "fmt" "strconv" "strings" "time" ) -// handle a line in "vespa.log" format; do filtering and formatting as specified in opts +type logFields struct { + timestamp string // seconds, optional fractional seconds + host string + pid string // pid, optional tid + service string + component string + level string + messages []string +} -func handleLine(opts *Options, line string) (output string, err error) { - fields := strings.SplitN(line, "\t", 7) - if len(fields) < 7 { +// handle a line in "vespa.log" format; do filtering and formatting as specified in opts +func handleLine(opts *Options, line string) (string, error) { + fieldStrings := strings.SplitN(line, "\t", 7) + if len(fieldStrings) < 7 { return "", fmt.Errorf("not enough fields: '%s'", line) } - timestampfield := fields[0] // seconds, optional fractional seconds - hostfield := fields[1] - pidfield := fields[2] // pid, optional tid - servicefield := fields[3] - componentfield := fields[4] - levelfield := fields[5] - messagefields := fields[6:] + fields := logFields{ + timestamp: fieldStrings[0], + host: fieldStrings[1], + pid: fieldStrings[2], + service: fieldStrings[3], + component: fieldStrings[4], + level: fieldStrings[5], + messages: fieldStrings[6:], + } - if !opts.showLevel(levelfield) { + if !opts.showLevel(fields.level) { return "", nil } - if opts.OnlyHostname != "" && opts.OnlyHostname != hostfield { + if opts.OnlyHostname != "" && opts.OnlyHostname != fields.host { return "", nil } - if opts.OnlyPid != "" && opts.OnlyPid != pidfield { + if opts.OnlyPid != "" && opts.OnlyPid != fields.pid { return "", nil } - if opts.OnlyService != "" && opts.OnlyService != servicefield { + if opts.OnlyService != "" && opts.OnlyService != fields.service { return "", nil } - if opts.OnlyInternal && !isInternal(componentfield) { + if opts.OnlyInternal && !isInternal(fields.component) { return "", nil } - if opts.ComponentFilter.unmatched(componentfield) { + if opts.ComponentFilter.unmatched(fields.component) { return "", nil } - if opts.MessageFilter.unmatched(strings.Join(messagefields, "\t")) { + if opts.MessageFilter.unmatched(strings.Join(fields.messages, "\t")) { return "", nil } + switch opts.Format { + case FormatRaw: + return line + "\n", nil + case FormatJSON: + return handleLineJson(opts, &fields) + case FormatVespa: + fallthrough + default: + return handleLineVespa(opts, &fields) + } +} + +func parseTimestamp(timestamp string) (time.Time, error) { + secs, err := strconv.ParseFloat(timestamp, 64) + if err != nil { + return time.Time{}, err + } + nsecs := int64(secs * 1e9) + return time.Unix(0, nsecs), nil +} + +type logFieldsJson struct { + Timestamp string `json:"timestamp"` + Host string `json:"host"` + Pid string `json:"pid"` + Service string `json:"service"` + Component string `json:"component"` + Level string `json:"level"` + Messages []string `json:"messages"` +} + +func handleLineJson(_ *Options, fields *logFields) (string, error) { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + outputFields := logFieldsJson{ + Timestamp: timestamp.Format(time.RFC3339Nano), + Host: fields.host, + Pid: fields.pid, + Service: fields.service, + Component: fields.component, + Level: fields.level, + Messages: fields.messages, + } + buf := bytes.Buffer{} + if err := json.NewEncoder(&buf).Encode(&outputFields); err != nil { + return "", err + } + return buf.String(), nil +} + +func handleLineVespa(opts *Options, fields *logFields) (string, error) { var buf strings.Builder if opts.showField("fmttime") { - secs, err := strconv.ParseFloat(timestampfield, 64) + timestamp, err := parseTimestamp(fields.timestamp) if err != nil { return "", err } - nsecs := int64(secs * 1e9) - timestamp := time.Unix(0, nsecs) if opts.showField("usecs") { buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000000] ")) } else if opts.showField("msecs") { @@ -65,36 +129,36 @@ func handleLine(opts *Options, line string) (output string, err error) { buf.WriteString(timestamp.Format("[2006-01-02 15:04:05] ")) } } else if opts.showField("time") { - buf.WriteString(timestampfield) + buf.WriteString(fields.timestamp) buf.WriteString(" ") } if opts.showField("host") { - buf.WriteString(fmt.Sprintf("%-8s ", hostfield)) + buf.WriteString(fmt.Sprintf("%-8s ", fields.host)) } if opts.showField("level") { - buf.WriteString(fmt.Sprintf("%-7s ", strings.ToUpper(levelfield))) + buf.WriteString(fmt.Sprintf("%-7s ", strings.ToUpper(fields.level))) } if opts.showField("pid") { // OnlyPid, _, _ := strings.Cut(pidfield, "/") - buf.WriteString(fmt.Sprintf("%6s ", pidfield)) + buf.WriteString(fmt.Sprintf("%6s ", fields.pid)) } if opts.showField("service") { if opts.TruncateService { - buf.WriteString(fmt.Sprintf("%-9.9s ", servicefield)) + buf.WriteString(fmt.Sprintf("%-9.9s ", fields.service)) } else { - buf.WriteString(fmt.Sprintf("%-16s ", servicefield)) + buf.WriteString(fmt.Sprintf("%-16s ", fields.service)) } } if opts.showField("component") { if opts.TruncateComponent { - buf.WriteString(fmt.Sprintf("%-15.15s ", componentfield)) + buf.WriteString(fmt.Sprintf("%-15.15s ", fields.component)) } else { - buf.WriteString(fmt.Sprintf("%s\t", componentfield)) + buf.WriteString(fmt.Sprintf("%s\t", fields.component)) } } if opts.showField("message") { var msgBuf strings.Builder - for idx, message := range messagefields { + for idx, message := range fields.messages { if idx > 0 { msgBuf.WriteString("\n\t") } @@ -111,6 +175,5 @@ func handleLine(opts *Options, line string) (output string, err error) { buf.WriteString(message) } buf.WriteString("\n") - output = buf.String() - return + return buf.String(), nil } diff --git a/client/go/cmd/logfmt/options.go b/client/go/cmd/logfmt/options.go index f564ffd4df0..864868d4ce5 100644 --- a/client/go/cmd/logfmt/options.go +++ b/client/go/cmd/logfmt/options.go @@ -24,6 +24,7 @@ type Options struct { TruncateComponent bool ComponentFilter regexFlag MessageFilter regexFlag + Format OutputFormat } func NewOptions() (ret Options) { -- cgit v1.2.3