From 2419c07ab4f94eb7fb97b2cd27c9900b79df0ecc Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 19 Aug 2022 11:07:12 +0200 Subject: Make vespa-logfmt build on non-unix --- client/go/cmd/logfmt/runlogfmt.go | 15 ++- client/go/cmd/logfmt/tail.go | 162 +------------------------------- client/go/cmd/logfmt/tail_not_unix.go | 15 +++ client/go/cmd/logfmt/tail_unix.go | 170 ++++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 163 deletions(-) create mode 100644 client/go/cmd/logfmt/tail_not_unix.go create mode 100644 client/go/cmd/logfmt/tail_unix.go (limited to 'client/go/cmd') diff --git a/client/go/cmd/logfmt/runlogfmt.go b/client/go/cmd/logfmt/runlogfmt.go index 9d782692f50..04b1ac96733 100644 --- a/client/go/cmd/logfmt/runlogfmt.go +++ b/client/go/cmd/logfmt/runlogfmt.go @@ -39,7 +39,10 @@ func RunLogfmt(opts *Options, args []string) { fmt.Fprintf(os.Stderr, "Must have exact 1 file for 'follow' option, got %d\n", len(args)) return } - tailFile(opts, args[0]) + if err := tailFile(opts, args[0]); err != nil { + fmt.Fprintln(os.Stderr, err) + return + } return } for _, arg := range args { @@ -62,11 +65,15 @@ func formatLine(opts *Options, line string) { } } -func tailFile(opts *Options, fn string) { - tailed := FollowFile(fn) - for line := range tailed.Lines { +func tailFile(opts *Options, fn string) error { + tailed, err := FollowFile(fn) + if err != nil { + return err + } + for line := range tailed.Lines() { formatLine(opts, line.Text) } + return nil } func formatFile(opts *Options, arg *os.File) { diff --git a/client/go/cmd/logfmt/tail.go b/client/go/cmd/logfmt/tail.go index 44674012548..75e7cbb0693 100644 --- a/client/go/cmd/logfmt/tail.go +++ b/client/go/cmd/logfmt/tail.go @@ -1,169 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa logfmt command -// Author: arnej +// Author: mpolden package logfmt -import ( - "bufio" - "fmt" - "io" - "os" - "time" - - "golang.org/x/sys/unix" -) - -const lastLinesSize = 4 * 1024 - type Line struct { Text string } -// an active "tail -f" like object - -type Tail struct { - Lines chan Line - lineBuf []byte - curFile *os.File - fn string - reader *bufio.Reader - curStat unix.Stat_t -} - -// API for starting to follow a log file - -func FollowFile(fn string) (res Tail) { - res.fn = fn - res.lineBuf = make([]byte, lastLinesSize) - res.openTail() - res.Lines = make(chan Line, 20) - res.lineBuf = res.lineBuf[:0] - go runTailWith(&res) - return res -} - -func (t *Tail) setFile(f *os.File) { - if t.curFile != nil { - t.curFile.Close() - } - t.curFile = f - if f != nil { - err := unix.Fstat(int(f.Fd()), &t.curStat) - if err != nil { - f.Close() - fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) - return - } - t.reader = bufio.NewReaderSize(f, 1024*1024) - } else { - t.reader = nil - } -} - -// open log file and seek to the start of a line near the end, if possible. -func (t *Tail) openTail() { - file, err := os.Open(t.fn) - if err != nil { - return - } - sz, err := file.Seek(0, os.SEEK_END) - if err != nil { - return - } - if sz < lastLinesSize { - sz, err = file.Seek(0, os.SEEK_SET) - if err == nil { - // just read from start of file, all OK - t.setFile(file) - } - return - } - sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) - n, err := file.Read(t.lineBuf) - if err != nil { - return - } - for i := 0; i < n; i++ { - if t.lineBuf[i] == '\n' { - sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) - if err == nil { - t.setFile(file) - } - return - } - } -} - -func (t *Tail) reopen(cur *unix.Stat_t) { - for cnt := 0; cnt < 100; cnt++ { - file, err := os.Open(t.fn) - if err != nil { - t.setFile(nil) - if cnt == 0 { - fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) - } - time.Sleep(1000 * time.Millisecond) - continue - } - var stat unix.Stat_t - err = unix.Fstat(int(file.Fd()), &stat) - if err != nil { - file.Close() - fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) - time.Sleep(5000 * time.Millisecond) - continue - } - if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { - // same file, continue following it - file.Close() - return - } - // new file, start following it - t.setFile(file) - return - } -} - -// runs as a goroutine -func runTailWith(t *Tail) { - defer t.setFile(nil) -loop: - for { - for t.curFile == nil { - t.reopen(nil) - } - bytes, err := t.reader.ReadSlice('\n') - t.lineBuf = append(t.lineBuf, bytes...) - if err == bufio.ErrBufferFull { - continue - } - if err == nil { - ll := len(t.lineBuf) - 1 - t.Lines <- Line{Text: string(t.lineBuf[:ll])} - t.lineBuf = t.lineBuf[:0] - continue - } - if err == io.EOF { - pos, _ := t.curFile.Seek(0, os.SEEK_CUR) - for cnt := 0; cnt < 100; cnt++ { - time.Sleep(10 * time.Millisecond) - sz, _ := t.curFile.Seek(0, os.SEEK_END) - if sz != pos { - if sz < pos { - // truncation case - pos = 0 - } - t.curFile.Seek(pos, os.SEEK_SET) - continue loop - } - } - // no change in file size, try reopening - t.reopen(&t.curStat) - } else { - fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) - close(t.Lines) - return - } - } +type Tail interface { + Lines() chan Line } diff --git a/client/go/cmd/logfmt/tail_not_unix.go b/client/go/cmd/logfmt/tail_not_unix.go new file mode 100644 index 00000000000..bc5d7879804 --- /dev/null +++ b/client/go/cmd/logfmt/tail_not_unix.go @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +//go:build !unix + +package logfmt + +import ( + "fmt" +) + +func FollowFile(fn string) (Tail, error) { + return nil, fmt.Errorf("tail is not supported on this platform") +} diff --git a/client/go/cmd/logfmt/tail_unix.go b/client/go/cmd/logfmt/tail_unix.go new file mode 100644 index 00000000000..f3d6a6f7bbe --- /dev/null +++ b/client/go/cmd/logfmt/tail_unix.go @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +//go:build unix + +package logfmt + +import ( + "bufio" + "fmt" + "io" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const lastLinesSize = 4 * 1024 + +// an active "tail -f" like object + +type unixTail struct { + lines chan Line + lineBuf []byte + curFile *os.File + fn string + reader *bufio.Reader + curStat unix.Stat_t +} + +func (t *unixTail) Lines() chan Line { return t.lines } + +// API for starting to follow a log file + +func FollowFile(fn string) (Tail, error) { + res := unixTail{} + res.fn = fn + res.lineBuf = make([]byte, lastLinesSize) + res.openTail() + res.lines = make(chan Line, 20) + res.lineBuf = res.lineBuf[:0] + go runTailWith(&res) + return &res, nil +} + +func (t *unixTail) setFile(f *os.File) { + if t.curFile != nil { + t.curFile.Close() + } + t.curFile = f + if f != nil { + err := unix.Fstat(int(f.Fd()), &t.curStat) + if err != nil { + f.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + return + } + t.reader = bufio.NewReaderSize(f, 1024*1024) + } else { + t.reader = nil + } +} + +// open log file and seek to the start of a line near the end, if possible. +func (t *unixTail) openTail() { + file, err := os.Open(t.fn) + if err != nil { + return + } + sz, err := file.Seek(0, os.SEEK_END) + if err != nil { + return + } + if sz < lastLinesSize { + sz, err = file.Seek(0, os.SEEK_SET) + if err == nil { + // just read from start of file, all OK + t.setFile(file) + } + return + } + sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) + n, err := file.Read(t.lineBuf) + if err != nil { + return + } + for i := 0; i < n; i++ { + if t.lineBuf[i] == '\n' { + sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) + if err == nil { + t.setFile(file) + } + return + } + } +} + +func (t *unixTail) reopen(cur *unix.Stat_t) { + for cnt := 0; cnt < 100; cnt++ { + file, err := os.Open(t.fn) + if err != nil { + t.setFile(nil) + if cnt == 0 { + fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) + } + time.Sleep(1000 * time.Millisecond) + continue + } + var stat unix.Stat_t + err = unix.Fstat(int(file.Fd()), &stat) + if err != nil { + file.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + time.Sleep(5000 * time.Millisecond) + continue + } + if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { + // same file, continue following it + file.Close() + return + } + // new file, start following it + t.setFile(file) + return + } +} + +// runs as a goroutine +func runTailWith(t *unixTail) { + defer t.setFile(nil) +loop: + for { + for t.curFile == nil { + t.reopen(nil) + } + bytes, err := t.reader.ReadSlice('\n') + t.lineBuf = append(t.lineBuf, bytes...) + if err == bufio.ErrBufferFull { + continue + } + if err == nil { + ll := len(t.lineBuf) - 1 + t.lines <- Line{Text: string(t.lineBuf[:ll])} + t.lineBuf = t.lineBuf[:0] + continue + } + if err == io.EOF { + pos, _ := t.curFile.Seek(0, os.SEEK_CUR) + for cnt := 0; cnt < 100; cnt++ { + time.Sleep(10 * time.Millisecond) + sz, _ := t.curFile.Seek(0, os.SEEK_END) + if sz != pos { + if sz < pos { + // truncation case + pos = 0 + } + t.curFile.Seek(pos, os.SEEK_SET) + continue loop + } + } + // no change in file size, try reopening + t.reopen(&t.curStat) + } else { + fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) + close(t.lines) + return + } + } +} -- cgit v1.2.3