diff options
Diffstat (limited to 'client/go/script-utils/logfmt')
20 files changed, 1986 insertions, 0 deletions
diff --git a/client/go/script-utils/logfmt/cmd.go b/client/go/script-utils/logfmt/cmd.go new file mode 100644 index 00000000000..84322c7ff08 --- /dev/null +++ b/client/go/script-utils/logfmt/cmd.go @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa-logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/build" +) + +func NewLogfmtCmd() *cobra.Command { + var ( + curOptions Options = NewOptions() + ) + cmd := &cobra.Command{ + Use: "vespa-logfmt", + Short: "convert vespa.log to human-readable format", + Long: `vespa-logfmt takes input in the internal vespa format +and converts it to something human-readable`, + Version: build.Version, + Run: func(cmd *cobra.Command, args []string) { + RunLogfmt(&curOptions, args) + }, + } + cmd.Flags().VarP(&curOptions.ShowLevels, "level", "l", "turn levels on/off\n") + cmd.Flags().VarP(&curOptions.ShowFields, "show", "s", "turn fields shown on/off\n") + cmd.Flags().VarP(&curOptions.ComponentFilter, "component", "c", "select components by regexp") + cmd.Flags().VarP(&curOptions.MessageFilter, "message", "m", "select messages by regexp") + cmd.Flags().BoolVarP(&curOptions.OnlyInternal, "internal", "i", false, "select only internal components") + cmd.Flags().BoolVar(&curOptions.TruncateService, "truncateservice", false, "truncate service name") + cmd.Flags().BoolVar(&curOptions.TruncateService, "ts", false, "") + cmd.Flags().BoolVarP(&curOptions.FollowTail, "follow", "f", false, "follow logfile with tail -f") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "nldequote", "N", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "dequotenewlines", "n", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.TruncateComponent, "truncatecomponent", "t", false, "truncate component name") + cmd.Flags().BoolVar(&curOptions.TruncateComponent, "tc", false, "") + cmd.Flags().StringVarP(&curOptions.OnlyHostname, "host", "H", "", "select only one host") + cmd.Flags().StringVarP(&curOptions.OnlyPid, "pid", "p", "", "select only one process ID") + cmd.Flags().StringVarP(&curOptions.OnlyService, "service", "S", "", "select only one service") + cmd.Flags().VarP(&curOptions.Format, "format", "F", "select logfmt output format, vespa (default), json or raw are supported. The json output format is not stable, and will change in the future.") + cmd.Flags().MarkHidden("tc") + cmd.Flags().MarkHidden("ts") + cmd.Flags().MarkHidden("dequotenewlines") + return cmd +} diff --git a/client/go/script-utils/logfmt/formatflags.go b/client/go/script-utils/logfmt/formatflags.go new file mode 100644 index 00000000000..097746d696f --- /dev/null +++ b/client/go/script-utils/logfmt/formatflags.go @@ -0,0 +1,41 @@ +package logfmt + +import ( + "fmt" + "strings" +) + +type OutputFormat int + +const ( + FormatVespa OutputFormat = iota //default is vespa + FormatRaw + FormatJSON +) + +func (v *OutputFormat) Type() string { + return "output format" +} + +func (v *OutputFormat) String() string { + flagNames := []string{ + "vespa", + "raw", + "json", + } + return flagNames[*v] +} + +func (v *OutputFormat) Set(val string) error { + switch strings.ToLower(val) { + case "vespa": + *v = FormatVespa + case "raw": + *v = FormatRaw + case "json": + *v = FormatJSON + default: + return fmt.Errorf("'%s' is not a valid format argument", val) + } + return nil +} diff --git a/client/go/script-utils/logfmt/formatflags_test.go b/client/go/script-utils/logfmt/formatflags_test.go new file mode 100644 index 00000000000..53c47d24208 --- /dev/null +++ b/client/go/script-utils/logfmt/formatflags_test.go @@ -0,0 +1,30 @@ +package logfmt + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestOutputFormat(t *testing.T) { + type args struct { + val string + } + tests := []struct { + expected OutputFormat + arg string + wantErr assert.ErrorAssertionFunc + }{ + {FormatVespa, "vespa", assert.NoError}, + {FormatRaw, "raw", assert.NoError}, + {FormatJSON, "json", assert.NoError}, + {-1, "foo", assert.Error}, + } + for _, tt := range tests { + t.Run(tt.arg, func(t *testing.T) { + var v OutputFormat = -1 + tt.wantErr(t, v.Set(tt.arg), fmt.Sprintf("Set(%v)", tt.arg)) + assert.Equal(t, v, tt.expected) + }) + } +} diff --git a/client/go/script-utils/logfmt/handleline.go b/client/go/script-utils/logfmt/handleline.go new file mode 100644 index 00000000000..813ca82acb4 --- /dev/null +++ b/client/go/script-utils/logfmt/handleline.go @@ -0,0 +1,179 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +type logFields struct { + timestamp string // seconds, optional fractional seconds + host string + pid string // pid, optional tid + service string + component string + level string + messages []string +} + +// handle a line in "vespa.log" format; do filtering and formatting as specified in opts +func handleLine(opts *Options, line string) (string, error) { + fieldStrings := strings.SplitN(line, "\t", 7) + if len(fieldStrings) < 7 { + return "", fmt.Errorf("not enough fields: '%s'", line) + } + fields := logFields{ + timestamp: fieldStrings[0], + host: fieldStrings[1], + pid: fieldStrings[2], + service: fieldStrings[3], + component: fieldStrings[4], + level: fieldStrings[5], + messages: fieldStrings[6:], + } + + if !opts.showLevel(fields.level) { + return "", nil + } + if opts.OnlyHostname != "" && opts.OnlyHostname != fields.host { + return "", nil + } + if opts.OnlyPid != "" && opts.OnlyPid != fields.pid { + return "", nil + } + if opts.OnlyService != "" && opts.OnlyService != fields.service { + return "", nil + } + if opts.OnlyInternal && !isInternal(fields.component) { + return "", nil + } + if opts.ComponentFilter.unmatched(fields.component) { + return "", nil + } + if opts.MessageFilter.unmatched(strings.Join(fields.messages, "\t")) { + return "", nil + } + + switch opts.Format { + case FormatRaw: + return line + "\n", nil + case FormatJSON: + return handleLineJson(opts, &fields) + case FormatVespa: + fallthrough + default: + return handleLineVespa(opts, &fields) + } +} + +func parseTimestamp(timestamp string) (time.Time, error) { + secs, err := strconv.ParseFloat(timestamp, 64) + if err != nil { + return time.Time{}, err + } + nsecs := int64(secs * 1e9) + return time.Unix(0, nsecs), nil +} + +type logFieldsJson struct { + Timestamp string `json:"timestamp"` + Host string `json:"host"` + Pid string `json:"pid"` + Service string `json:"service"` + Component string `json:"component"` + Level string `json:"level"` + Messages []string `json:"messages"` +} + +func handleLineJson(_ *Options, fields *logFields) (string, error) { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + outputFields := logFieldsJson{ + Timestamp: timestamp.Format(time.RFC3339Nano), + Host: fields.host, + Pid: fields.pid, + Service: fields.service, + Component: fields.component, + Level: fields.level, + Messages: fields.messages, + } + buf := bytes.Buffer{} + if err := json.NewEncoder(&buf).Encode(&outputFields); err != nil { + return "", err + } + return buf.String(), nil +} + +func handleLineVespa(opts *Options, fields *logFields) (string, error) { + var buf strings.Builder + + if opts.showField("fmttime") { + timestamp, err := parseTimestamp(fields.timestamp) + if err != nil { + return "", err + } + if opts.showField("usecs") { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000000] ")) + } else if opts.showField("msecs") { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05.000] ")) + } else { + buf.WriteString(timestamp.Format("[2006-01-02 15:04:05] ")) + } + } else if opts.showField("time") { + buf.WriteString(fields.timestamp) + buf.WriteString(" ") + } + if opts.showField("host") { + buf.WriteString(fmt.Sprintf("%-8s ", fields.host)) + } + if opts.showField("level") { + buf.WriteString(fmt.Sprintf("%-7s ", strings.ToUpper(fields.level))) + } + if opts.showField("pid") { + // OnlyPid, _, _ := strings.Cut(pidfield, "/") + buf.WriteString(fmt.Sprintf("%6s ", fields.pid)) + } + if opts.showField("service") { + if opts.TruncateService { + buf.WriteString(fmt.Sprintf("%-9.9s ", fields.service)) + } else { + buf.WriteString(fmt.Sprintf("%-16s ", fields.service)) + } + } + if opts.showField("component") { + if opts.TruncateComponent { + buf.WriteString(fmt.Sprintf("%-15.15s ", fields.component)) + } else { + buf.WriteString(fmt.Sprintf("%s\t", fields.component)) + } + } + if opts.showField("message") { + var msgBuf strings.Builder + for idx, message := range fields.messages { + if idx > 0 { + msgBuf.WriteString("\n\t") + } + if opts.DequoteNewlines { + message = strings.ReplaceAll(message, "\\n\\t", "\n\t") + message = strings.ReplaceAll(message, "\\n", "\n\t") + } + msgBuf.WriteString(message) + } + message := msgBuf.String() + if strings.Contains(message, "\n") { + buf.WriteString("\n\t") + } + buf.WriteString(message) + } + buf.WriteString("\n") + return buf.String(), nil +} diff --git a/client/go/script-utils/logfmt/internal.go b/client/go/script-utils/logfmt/internal.go new file mode 100644 index 00000000000..992c537f939 --- /dev/null +++ b/client/go/script-utils/logfmt/internal.go @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// is componentName a vespa-internal name? + +func isInternal(componentName string) bool { + cs := strings.Split(componentName, ".") + if len(cs) == 0 || cs[0] != "Container" { + return true + } + if len(cs) < 3 { + return false + } + if cs[1] == "ai" && cs[2] == "vespa" { + return true + } + if cs[1] == "com" && cs[2] == "yahoo" && len(cs) > 3 { + return internalComYahooNames[cs[3]] + } + return false +} + +// a constant: +var internalComYahooNames = map[string]bool{ + "abicheck": true, + "api": true, + "application": true, + "binaryprefix": true, + "clientmetrics": true, + "cloud": true, + "collections": true, + "component": true, + "compress": true, + "concurrent": true, + "configtest": true, + "config": true, + "container": true, + "data": true, + "docprocs": true, + "docproc": true, + "documentapi": true, + "documentmodel": true, + "document": true, + "dummyreceiver": true, + "embedding": true, + "errorhandling": true, + "exception": true, + "feedapi": true, + "feedhandler": true, + "filedistribution": true, + "fs4": true, + "fsa": true, + "geo": true, + "io": true, + "javacc": true, + "jdisc": true, + "jrt": true, + "lang": true, + "language": true, + "logserver": true, + "log": true, + "messagebus": true, + "metrics": true, + "nativec": true, + "net": true, + "osgi": true, + "path": true, + "plugin": true, + "prelude": true, + "processing": true, + "protect": true, + "reflection": true, + "restapi": true, + "schema": true, + "searchdefinition": true, + "searchlib": true, + "search": true, + "security": true, + "slime": true, + "socket": true, + "statistics": true, + "stream": true, + "system": true, + "tensor": true, + "test": true, + "text": true, + "time": true, + "transaction": true, + "vdslib": true, + "vespaclient": true, + "vespafeeder": true, + "vespaget": true, + "vespastat": true, + "vespasummarybenchmark": true, + "vespa": true, + "vespavisit": true, + "vespaxmlparser": true, + "yolean": true, +} diff --git a/client/go/script-utils/logfmt/internal_names.txt b/client/go/script-utils/logfmt/internal_names.txt new file mode 100644 index 00000000000..cc554546fcc --- /dev/null +++ b/client/go/script-utils/logfmt/internal_names.txt @@ -0,0 +1,782 @@ +sentinel.sentinel.config +searchnode.eval.Test +Container.ai.vespa.client.dsl.Test +Container.ai.vespa.cloud.Test +Container.ai.vespa.embedding.Test +Container.ai.vespa.explicitversion.Test +Container.ai.vespa.explicitversion_dep.Test +Container.ai.vespa.feed.client.Test +Container.ai.vespa.feed.client.impl.Test +Container.ai.vespa.hosted.api.Test +Container.ai.vespa.hosted.cd.Test +Container.ai.vespa.hosted.cd.cloud.impl.Test +Container.ai.vespa.hosted.cd.commons.Test +Container.ai.vespa.hosted.cd.internal.Test +Container.ai.vespa.hosted.client.Test +Container.ai.vespa.hosted.plugin.Test +Container.ai.vespa.http.Test +Container.ai.vespa.intellij.schema.Test +Container.ai.vespa.intellij.schema.findUsages.Test +Container.ai.vespa.intellij.schema.hierarchy.Test +Container.ai.vespa.intellij.schema.lexer.Test +Container.ai.vespa.intellij.schema.model.Test +Container.ai.vespa.intellij.schema.parser.Test +Container.ai.vespa.intellij.schema.psi.Test +Container.ai.vespa.intellij.schema.psi.impl.Test +Container.ai.vespa.intellij.schema.structure.Test +Container.ai.vespa.intellij.schema.utils.Test +Container.ai.vespa.logserver.protocol.Test +Container.ai.vespa.metricsproxy.core.Test +Container.ai.vespa.metricsproxy.http.Test +Container.ai.vespa.metricsproxy.http.application.Test +Container.ai.vespa.metricsproxy.http.metrics.Test +Container.ai.vespa.metricsproxy.http.prometheus.Test +Container.ai.vespa.metricsproxy.http.yamas.Test +Container.ai.vespa.metricsproxy.metric.Test +Container.ai.vespa.metricsproxy.metric.dimensions.Test +Container.ai.vespa.metricsproxy.metric.model.Test +Container.ai.vespa.metricsproxy.metric.model.json.Test +Container.ai.vespa.metricsproxy.metric.model.processing.Test +Container.ai.vespa.metricsproxy.metric.model.prometheus.Test +Container.ai.vespa.metricsproxy.node.Test +Container.ai.vespa.metricsproxy.rpc.Test +Container.ai.vespa.metricsproxy.service.Test +Container.ai.vespa.metricsproxy.telegraf.Test +Container.ai.vespa.modelintegration.evaluator.Test +Container.ai.vespa.models.evaluation.Test +Container.ai.vespa.models.handler.Test +Container.ai.vespa.noversion.Test +Container.ai.vespa.noversion_dep.Test +Container.ai.vespa.rankingexpression.importer.Test +Container.ai.vespa.rankingexpression.importer.configmodelview.Test +Container.ai.vespa.rankingexpression.importer.lightgbm.Test +Container.ai.vespa.rankingexpression.importer.onnx.Test +Container.ai.vespa.rankingexpression.importer.operations.Test +Container.ai.vespa.rankingexpression.importer.tensorflow.Test +Container.ai.vespa.rankingexpression.importer.vespa.Test +Container.ai.vespa.rankingexpression.importer.vespa.parser.Test +Container.ai.vespa.rankingexpression.importer.xgboost.Test +Container.ai.vespa.reindexing.Test +Container.ai.vespa.reindexing.http.Test +Container.ai.vespa.searchlib.searchprotocol.protobuf.Test +Container.ai.vespa.util.http.hc4.Test +Container.ai.vespa.util.http.hc4.retry.Test +Container.ai.vespa.util.http.hc5.Test +Container.ai.vespa.validation.Test +Container.com.yahoo.abicheck.classtree.Test +Container.com.yahoo.abicheck.collector.Test +Container.com.yahoo.abicheck.mojo.Test +Container.com.yahoo.abicheck.setmatcher.Test +Container.com.yahoo.abicheck.signature.Test +Container.com.yahoo.api.annotations.Test +Container.com.yahoo.application.Test +Container.com.yahoo.application.container.Test +Container.com.yahoo.application.container.handler.Test +Container.com.yahoo.application.container.impl.Test +Container.com.yahoo.application.content.Test +Container.com.yahoo.application.preprocessor.Test +Container.com.yahoo.binaryprefix.Test +Container.com.yahoo.clientmetrics.Test +Container.com.yahoo.cloud.config.Test +Container.com.yahoo.collections.Test +Container.com.yahoo.component.Test +Container.com.yahoo.component.annotation.Test +Container.com.yahoo.component.chain.Test +Container.com.yahoo.component.chain.dependencies.Test +Container.com.yahoo.component.chain.dependencies.ordering.Test +Container.com.yahoo.component.chain.model.Test +Container.com.yahoo.component.provider.Test +Container.com.yahoo.compress.Test +Container.com.yahoo.concurrent.Test +Container.com.yahoo.concurrent.classlock.Test +Container.com.yahoo.concurrent.maintenance.Test +Container.com.yahoo.config.Test +Container.com.yahoo.config.application.Test +Container.com.yahoo.config.application.api.Test +Container.com.yahoo.config.application.api.xml.Test +Container.com.yahoo.config.codegen.Test +Container.com.yahoo.config.docproc.Test +Container.com.yahoo.config.ini.Test +Container.com.yahoo.config.model.Test +Container.com.yahoo.config.model.admin.Test +Container.com.yahoo.config.model.api.Test +Container.com.yahoo.config.model.api.container.Test +Container.com.yahoo.config.model.application.Test +Container.com.yahoo.config.model.application.provider.Test +Container.com.yahoo.config.model.builder.xml.Test +Container.com.yahoo.config.model.deploy.Test +Container.com.yahoo.config.model.graph.Test +Container.com.yahoo.config.model.producer.Test +Container.com.yahoo.config.model.provision.Test +Container.com.yahoo.config.model.test.Test +Container.com.yahoo.config.provision.Test +Container.com.yahoo.config.provision.exception.Test +Container.com.yahoo.config.provision.host.Test +Container.com.yahoo.config.provisioning.Test +Container.com.yahoo.config.provision.security.Test +Container.com.yahoo.config.provision.serialization.Test +Container.com.yahoo.config.provision.zone.Test +Container.com.yahoo.config.subscription.Test +Container.com.yahoo.config.subscription.impl.Test +Container.com.yahoo.configtest.Test +Container.com.yahoo.config.text.Test +Container.com.yahoo.container.Test +Container.com.yahoo.container.bundle.Test +Container.com.yahoo.container.core.Test +Container.com.yahoo.container.core.config.Test +Container.com.yahoo.container.core.config.testutil.Test +Container.com.yahoo.container.core.document.Test +Container.com.yahoo.container.core.documentapi.Test +Container.com.yahoo.container.core.http.Test +Container.com.yahoo.container.core.identity.Test +Container.com.yahoo.container.di.Test +Container.com.yahoo.container.di.componentgraph.Test +Container.com.yahoo.container.di.componentgraph.core.Test +Container.com.yahoo.container.di.componentgraph.cycle.Test +Container.com.yahoo.container.di.config.Test +Container.com.yahoo.container.handler.Test +Container.com.yahoo.container.handler.metrics.Test +Container.com.yahoo.container.handler.observability.Test +Container.com.yahoo.container.handler.test.Test +Container.com.yahoo.container.handler.threadpool.Test +Container.com.yahoo.container.http.Test +Container.com.yahoo.container.http.filter.Test +Container.com.yahoo.container.jdisc.Test +Container.com.yahoo.container.jdisc.athenz.Test +Container.com.yahoo.container.jdisc.component.Test +Container.com.yahoo.container.jdisc.config.Test +Container.com.yahoo.container.jdisc.messagebus.Test +Container.com.yahoo.container.jdisc.metric.Test +Container.com.yahoo.container.jdisc.metric.state.Test +Container.com.yahoo.container.jdisc.secretstore.Test +Container.com.yahoo.container.jdisc.state.Test +Container.com.yahoo.container.jdisc.utils.Test +Container.com.yahoo.container.logging.Test +Container.com.yahoo.container.plugin.bundle.Test +Container.com.yahoo.container.plugin.classanalysis.Test +Container.com.yahoo.container.plugin.mojo.Test +Container.com.yahoo.container.plugin.osgi.Test +Container.com.yahoo.container.plugin.util.Test +Container.com.yahoo.container.protect.Test +Container.com.yahoo.container.standalone.Test +Container.com.yahoo.container.usability.Test +Container.com.yahoo.data.Test +Container.com.yahoo.data.access.Test +Container.com.yahoo.data.access.simple.Test +Container.com.yahoo.data.access.slime.Test +Container.com.yahoo.docproc.Test +Container.com.yahoo.docproc.impl.Test +Container.com.yahoo.docproc.jdisc.Test +Container.com.yahoo.docproc.jdisc.messagebus.Test +Container.com.yahoo.docproc.jdisc.metric.Test +Container.com.yahoo.docproc.jdisc.observability.Test +Container.com.yahoo.docproc.proxy.Test +Container.com.yahoo.docprocs.indexing.Test +Container.com.yahoo.document.Test +Container.com.yahoo.document.annotation.Test +Container.com.yahoo.documentapi.Test +Container.com.yahoo.documentapi.local.Test +Container.com.yahoo.documentapi.messagebus.Test +Container.com.yahoo.documentapi.messagebus.protocol.Test +Container.com.yahoo.documentapi.messagebus.systemstate.rule.Test +Container.com.yahoo.documentapi.metrics.Test +Container.com.yahoo.document.config.Test +Container.com.yahoo.document.datatypes.Test +Container.com.yahoo.document.fieldpathupdate.Test +Container.com.yahoo.document.fieldset.Test +Container.com.yahoo.document.idstring.Test +Container.com.yahoo.document.internal.Test +Container.com.yahoo.document.json.Test +Container.com.yahoo.document.json.document.Test +Container.com.yahoo.document.json.readers.Test +Container.com.yahoo.documentmodel.Test +Container.com.yahoo.document.predicate.Test +Container.com.yahoo.document.restapi.resource.Test +Container.com.yahoo.document.select.Test +Container.com.yahoo.document.select.convert.Test +Container.com.yahoo.document.select.parser.Test +Container.com.yahoo.document.select.rule.Test +Container.com.yahoo.document.select.simple.Test +Container.com.yahoo.document.serialization.Test +Container.com.yahoo.document.update.Test +Container.com.yahoo.dummyreceiver.Test +Container.com.yahoo.embedding.Test +Container.com.yahoo.errorhandling.Test +Container.com.yahoo.exception.Test +Container.com.yahoo.feedapi.Test +Container.com.yahoo.feedhandler.Test +Container.com.yahoo.filedistribution.fileacquirer.Test +Container.com.yahoo.fs4.Test +Container.com.yahoo.fsa.Test +Container.com.yahoo.fsa.conceptnet.Test +Container.com.yahoo.fsa.segmenter.Test +Container.com.yahoo.fsa.topicpredictor.Test +Container.com.yahoo.geo.Test +Container.com.yahoo.io.Test +Container.com.yahoo.io.reader.Test +Container.com.yahoo.javacc.Test +Container.com.yahoo.jdisc.Test +Container.com.yahoo.jdisc.application.Test +Container.com.yahoo.jdisc.bundle.Test +Container.com.yahoo.jdisc.bundle.a.Test +Container.com.yahoo.jdisc.bundle.b.Test +Container.com.yahoo.jdisc.bundle.c.Test +Container.com.yahoo.jdisc.bundle.d.Test +Container.com.yahoo.jdisc.bundle.e.Test +Container.com.yahoo.jdisc.bundle.f.Test +Container.com.yahoo.jdisc.bundle.g.Test +Container.com.yahoo.jdisc.bundle.g_act.Test +Container.com.yahoo.jdisc.bundle.h.Test +Container.com.yahoo.jdisc.bundle.i.Test +Container.com.yahoo.jdisc.bundle.j.Test +Container.com.yahoo.jdisc.bundle.k.Test +Container.com.yahoo.jdisc.bundle.l.Test +Container.com.yahoo.jdisc.bundle.m.Test +Container.com.yahoo.jdisc.bundle.my_act.Test +Container.com.yahoo.jdisc.bundle.n.Test +Container.com.yahoo.jdisc.bundle.o.Test +Container.com.yahoo.jdisc.bundle.p.Test +Container.com.yahoo.jdisc.bundle.q.Test +Container.com.yahoo.jdisc.bundle.r.Test +Container.com.yahoo.jdisc.bundle.s.Test +Container.com.yahoo.jdisc.bundle.t.Test +Container.com.yahoo.jdisc.bundle.u.Test +Container.com.yahoo.jdisc.client.Test +Container.com.yahoo.jdisc.cloud.aws.Test +Container.com.yahoo.jdisc.core.Test +Container.com.yahoo.jdisc.handler.Test +Container.com.yahoo.jdisc.http.Test +Container.com.yahoo.jdisc.http.cloud.Test +Container.com.yahoo.jdisc.http.filter.Test +Container.com.yahoo.jdisc.http.filter.chain.Test +Container.com.yahoo.jdisc.http.filter.security.athenz.Test +Container.com.yahoo.jdisc.http.filter.security.base.Test +Container.com.yahoo.jdisc.http.filter.security.cors.Test +Container.com.yahoo.jdisc.http.filter.security.csp.Test +Container.com.yahoo.jdisc.http.filter.security.misc.Test +Container.com.yahoo.jdisc.http.filter.security.rule.Test +Container.com.yahoo.jdisc.http.filter.util.Test +Container.com.yahoo.jdisc.http.server.jetty.Test +Container.com.yahoo.jdisc.http.server.jetty.testutils.Test +Container.com.yahoo.jdisc.http.ssl.impl.Test +Container.com.yahoo.jdisc.metrics.yamasconsumer.cloud.Test +Container.com.yahoo.jdisc.refcount.Test +Container.com.yahoo.jdisc.service.Test +Container.com.yahoo.jdisc.statistics.Test +Container.com.yahoo.jdisc.test.Test +Container.com.yahoo.jrt.Test +Container.com.yahoo.jrt.slobrok.Test +Container.com.yahoo.jrt.slobrok.api.Test +Container.com.yahoo.jrt.slobrok.server.Test +Container.com.yahoo.jrt.tool.Test +Container.com.yahoo.lang.Test +Container.com.yahoo.language.Test +Container.com.yahoo.language.detect.Test +Container.com.yahoo.language.opennlp.Test +Container.com.yahoo.language.process.Test +Container.com.yahoo.language.provider.Test +Container.com.yahoo.language.sentencepiece.Test +Container.com.yahoo.language.simple.Test +Container.com.yahoo.language.simple.kstem.Test +Container.com.yahoo.language.tools.Test +Container.com.yahoo.language.wordpiece.Test +Container.com.yahoo.log.Test +Container.com.yahoo.log.event.Test +Container.com.yahoo.log.impl.Test +Container.com.yahoo.logserver.Test +Container.com.yahoo.logserver.filter.Test +Container.com.yahoo.logserver.handlers.Test +Container.com.yahoo.logserver.handlers.archive.Test +Container.com.yahoo.logserver.handlers.logmetrics.Test +Container.com.yahoo.logserver.testutils.Test +Container.com.yahoo.messagebus.Test +Container.com.yahoo.messagebus.jdisc.Test +Container.com.yahoo.messagebus.jdisc.test.Test +Container.com.yahoo.messagebus.network.Test +Container.com.yahoo.messagebus.network.local.Test +Container.com.yahoo.messagebus.network.rpc.Test +Container.com.yahoo.messagebus.network.rpc.test.Test +Container.com.yahoo.messagebus.routing.Test +Container.com.yahoo.messagebus.routing.test.Test +Container.com.yahoo.messagebus.shared.Test +Container.com.yahoo.messagebus.test.Test +Container.com.yahoo.metrics.Test +Container.com.yahoo.metrics.simple.Test +Container.com.yahoo.metrics.simple.jdisc.Test +Container.com.yahoo.metrics.simple.runtime.Test +Container.com.yahoo.nativec.Test +Container.com.yahoo.net.Test +Container.com.yahoo.osgi.Test +Container.com.yahoo.osgi.annotation.Test +Container.com.yahoo.osgi.provider.model.Test +Container.com.yahoo.path.Test +Container.com.yahoo.plugin.Test +Container.com.yahoo.prelude.Test +Container.com.yahoo.prelude.cluster.Test +Container.com.yahoo.prelude.fastsearch.Test +Container.com.yahoo.prelude.hitfield.Test +Container.com.yahoo.prelude.query.Test +Container.com.yahoo.prelude.query.parser.Test +Container.com.yahoo.prelude.query.textualrepresentation.Test +Container.com.yahoo.prelude.querytransform.Test +Container.com.yahoo.prelude.searcher.Test +Container.com.yahoo.prelude.semantics.Test +Container.com.yahoo.prelude.semantics.benchmark.Test +Container.com.yahoo.prelude.semantics.config.Test +Container.com.yahoo.prelude.semantics.engine.Test +Container.com.yahoo.prelude.semantics.parser.Test +Container.com.yahoo.prelude.semantics.rule.Test +Container.com.yahoo.prelude.statistics.Test +Container.com.yahoo.processing.Test +Container.com.yahoo.processing.execution.Test +Container.com.yahoo.processing.execution.chain.Test +Container.com.yahoo.processing.handler.Test +Container.com.yahoo.processing.impl.Test +Container.com.yahoo.processing.processors.Test +Container.com.yahoo.processing.rendering.Test +Container.com.yahoo.processing.request.Test +Container.com.yahoo.processing.request.properties.Test +Container.com.yahoo.processing.response.Test +Container.com.yahoo.processing.test.Test +Container.com.yahoo.protect.Test +Container.com.yahoo.reflection.Test +Container.com.yahoo.restapi.Test +Container.com.yahoo.schema.Test +Container.com.yahoo.schema.derived.Test +Container.com.yahoo.schema.derived.validation.Test +Container.com.yahoo.schema.document.Test +Container.com.yahoo.schema.document.annotation.Test +Container.com.yahoo.schema.expressiontransforms.Test +Container.com.yahoo.schema.fieldoperation.Test +Container.com.yahoo.schema.parser.Test +Container.com.yahoo.schema.processing.Test +Container.com.yahoo.schema.processing.multifieldresolver.Test +Container.com.yahoo.search.Test +Container.com.yahoo.search.cluster.Test +Container.com.yahoo.search.config.Test +Container.com.yahoo.search.dispatch.Test +Container.com.yahoo.search.dispatch.rpc.Test +Container.com.yahoo.search.dispatch.searchcluster.Test +Container.com.yahoo.search.federation.Test +Container.com.yahoo.search.federation.selection.Test +Container.com.yahoo.search.federation.sourceref.Test +Container.com.yahoo.search.grouping.Test +Container.com.yahoo.search.grouping.request.Test +Container.com.yahoo.search.grouping.request.parser.Test +Container.com.yahoo.search.grouping.result.Test +Container.com.yahoo.search.grouping.vespa.Test +Container.com.yahoo.search.handler.Test +Container.com.yahoo.search.handler.observability.Test +Container.com.yahoo.search.intent.model.Test +Container.com.yahoo.searchlib.Test +Container.com.yahoo.searchlib.aggregation.Test +Container.com.yahoo.searchlib.aggregation.hll.Test +Container.com.yahoo.searchlib.document.Test +Container.com.yahoo.searchlib.expression.Test +Container.com.yahoo.searchlib.gbdt.Test +Container.com.yahoo.searchlib.rankingexpression.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.gbdtoptimization.Test +Container.com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Test +Container.com.yahoo.searchlib.rankingexpression.parser.Test +Container.com.yahoo.searchlib.rankingexpression.rule.Test +Container.com.yahoo.searchlib.rankingexpression.transform.Test +Container.com.yahoo.searchlib.ranking.features.Test +Container.com.yahoo.searchlib.ranking.features.fieldmatch.Test +Container.com.yahoo.searchlib.tensor.Test +Container.com.yahoo.searchlib.treenet.Test +Container.com.yahoo.searchlib.treenet.parser.Test +Container.com.yahoo.searchlib.treenet.rule.Test +Container.com.yahoo.search.match.Test +Container.com.yahoo.search.pagetemplates.Test +Container.com.yahoo.search.pagetemplates.config.Test +Container.com.yahoo.search.pagetemplates.engine.Test +Container.com.yahoo.search.pagetemplates.engine.resolvers.Test +Container.com.yahoo.search.pagetemplates.model.Test +Container.com.yahoo.search.pagetemplates.result.Test +Container.com.yahoo.search.predicate.Test +Container.com.yahoo.search.predicate.annotator.Test +Container.com.yahoo.search.predicate.benchmarks.Test +Container.com.yahoo.search.predicate.index.Test +Container.com.yahoo.search.predicate.index.conjunction.Test +Container.com.yahoo.search.predicate.optimization.Test +Container.com.yahoo.search.predicate.serialization.Test +Container.com.yahoo.search.predicate.utils.Test +Container.com.yahoo.search.query.Test +Container.com.yahoo.search.query.context.Test +Container.com.yahoo.search.query.gui.Test +Container.com.yahoo.search.query.parser.Test +Container.com.yahoo.search.query.profile.Test +Container.com.yahoo.search.query.profile.compiled.Test +Container.com.yahoo.search.query.profile.config.Test +Container.com.yahoo.search.query.profile.types.Test +Container.com.yahoo.search.query.properties.Test +Container.com.yahoo.search.query.ranking.Test +Container.com.yahoo.search.query.restapi.Test +Container.com.yahoo.search.query.rewrite.Test +Container.com.yahoo.search.query.rewrite.rewriters.Test +Container.com.yahoo.search.query.textserialize.Test +Container.com.yahoo.search.query.textserialize.item.Test +Container.com.yahoo.search.query.textserialize.parser.Test +Container.com.yahoo.search.query.textserialize.serializer.Test +Container.com.yahoo.search.querytransform.Test +Container.com.yahoo.search.rendering.Test +Container.com.yahoo.search.result.Test +Container.com.yahoo.search.schema.Test +Container.com.yahoo.search.schema.internal.Test +Container.com.yahoo.search.searchchain.Test +Container.com.yahoo.search.searchchain.example.Test +Container.com.yahoo.search.searchchain.model.Test +Container.com.yahoo.search.searchchain.model.federation.Test +Container.com.yahoo.search.searchchain.testutil.Test +Container.com.yahoo.search.searchers.Test +Container.com.yahoo.search.statistics.Test +Container.com.yahoo.search.yql.Test +Container.com.yahoo.security.Test +Container.com.yahoo.security.tls.Test +Container.com.yahoo.slime.Test +Container.com.yahoo.socket.test.Test +Container.com.yahoo.stream.Test +Container.com.yahoo.system.Test +Container.com.yahoo.system.execution.Test +Container.com.yahoo.tensor.Test +Container.com.yahoo.tensor.evaluation.Test +Container.com.yahoo.tensor.functions.Test +Container.com.yahoo.tensor.serialization.Test +Container.com.yahoo.test.Test +Container.com.yahoo.test.json.Test +Container.com.yahoo.text.Test +Container.com.yahoo.text.internal.Test +Container.com.yahoo.text.interpretation.Test +Container.com.yahoo.time.Test +Container.com.yahoo.transaction.Test +Container.com.yahoo.vdslib.Test +Container.com.yahoo.vdslib.distribution.Test +Container.com.yahoo.vdslib.state.Test +Container.com.yahoo.vespa.Test +Container.com.yahoo.vespa.applicationmodel.Test +Container.com.yahoo.vespa.athenz.api.Test +Container.com.yahoo.vespa.athenz.aws.Test +Container.com.yahoo.vespa.athenz.client.Test +Container.com.yahoo.vespa.athenz.client.common.Test +Container.com.yahoo.vespa.athenz.client.common.bindings.Test +Container.com.yahoo.vespa.athenz.client.common.serializers.Test +Container.com.yahoo.vespa.athenz.client.zms.Test +Container.com.yahoo.vespa.athenz.client.zms.bindings.Test +Container.com.yahoo.vespa.athenz.client.zts.Test +Container.com.yahoo.vespa.athenz.client.zts.bindings.Test +Container.com.yahoo.vespa.athenz.client.zts.utils.Test +Container.com.yahoo.vespa.athenz.identity.Test +Container.com.yahoo.vespa.athenz.identityprovider.api.Test +Container.com.yahoo.vespa.athenz.identityprovider.api.bindings.Test +Container.com.yahoo.vespa.athenz.identityprovider.client.Test +Container.com.yahoo.vespa.athenz.tls.Test +Container.com.yahoo.vespa.athenz.utils.Test +Container.com.yahoo.vespa.athenz.zpe.Test +Container.com.yahoo.vespaclient.Test +Container.com.yahoo.vespa.clustercontroller.apps.clustercontroller.Test +Container.com.yahoo.vespa.clustercontroller.apputil.communication.http.Test +Container.com.yahoo.vespa.clustercontroller.core.Test +Container.com.yahoo.vespa.clustercontroller.core.database.Test +Container.com.yahoo.vespa.clustercontroller.core.hostinfo.Test +Container.com.yahoo.vespa.clustercontroller.core.listeners.Test +Container.com.yahoo.vespa.clustercontroller.core.restapiv2.Test +Container.com.yahoo.vespa.clustercontroller.core.restapiv2.requests.Test +Container.com.yahoo.vespa.clustercontroller.core.rpc.Test +Container.com.yahoo.vespa.clustercontroller.core.status.Test +Container.com.yahoo.vespa.clustercontroller.core.status.statuspage.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.async.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.http.Test +Container.com.yahoo.vespa.clustercontroller.utils.communication.http.writer.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.errors.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.response.Test +Container.com.yahoo.vespa.clustercontroller.utils.staterestapi.server.Test +Container.com.yahoo.vespa.clustercontroller.utils.util.Test +Container.com.yahoo.vespa.config.Test +Container.com.yahoo.vespa.config.benchmark.Test +Container.com.yahoo.vespa.config.buildergen.Test +Container.com.yahoo.vespa.config.content.Test +Container.com.yahoo.vespa.config.content.core.Test +Container.com.yahoo.vespa.config.content.reindexing.Test +Container.com.yahoo.vespa.config.core.Test +Container.com.yahoo.vespa.configdefinition.Test +Container.com.yahoo.vespa.configmodel.producers.Test +Container.com.yahoo.vespa.config.parser.Test +Container.com.yahoo.vespa.config.protocol.Test +Container.com.yahoo.vespa.config.proxy.Test +Container.com.yahoo.vespa.config.proxy.filedistribution.Test +Container.com.yahoo.vespa.config.search.Test +Container.com.yahoo.vespa.config.search.core.Test +Container.com.yahoo.vespa.config.server.Test +Container.com.yahoo.vespa.config.server.application.Test +Container.com.yahoo.vespa.config.server.configchange.Test +Container.com.yahoo.vespa.config.server.deploy.Test +Container.com.yahoo.vespa.config.server.filedistribution.Test +Container.com.yahoo.vespa.configserver.flags.Test +Container.com.yahoo.vespa.configserver.flags.db.Test +Container.com.yahoo.vespa.configserver.flags.http.Test +Container.com.yahoo.vespa.config.server.host.Test +Container.com.yahoo.vespa.config.server.http.Test +Container.com.yahoo.vespa.config.server.http.status.Test +Container.com.yahoo.vespa.config.server.http.v1.Test +Container.com.yahoo.vespa.config.server.http.v2.Test +Container.com.yahoo.vespa.config.server.http.v2.request.Test +Container.com.yahoo.vespa.config.server.http.v2.response.Test +Container.com.yahoo.vespa.config.server.maintenance.Test +Container.com.yahoo.vespa.config.server.metrics.Test +Container.com.yahoo.vespa.config.server.model.Test +Container.com.yahoo.vespa.config.server.modelfactory.Test +Container.com.yahoo.vespa.config.server.monitoring.Test +Container.com.yahoo.vespa.config.server.provision.Test +Container.com.yahoo.vespa.config.server.rpc.Test +Container.com.yahoo.vespa.config.server.rpc.security.Test +Container.com.yahoo.vespa.config.server.session.Test +Container.com.yahoo.vespa.config.server.tenant.Test +Container.com.yahoo.vespa.config.server.version.Test +Container.com.yahoo.vespa.config.server.zookeeper.Test +Container.com.yahoo.vespa.config.storage.Test +Container.com.yahoo.vespa.config.util.Test +Container.com.yahoo.vespa.curator.Test +Container.com.yahoo.vespa.curator.api.Test +Container.com.yahoo.vespa.curator.mock.Test +Container.com.yahoo.vespa.curator.recipes.Test +Container.com.yahoo.vespa.curator.stats.Test +Container.com.yahoo.vespa.curator.transaction.Test +Container.com.yahoo.vespa.defaults.Test +Container.com.yahoo.vespa.document.Test +Container.com.yahoo.vespa.document.dom.Test +Container.com.yahoo.vespa.documentmodel.Test +Container.com.yahoo.vespafeeder.Test +Container.com.yahoo.vespa.feed.perf.Test +Container.com.yahoo.vespa.filedistribution.Test +Container.com.yahoo.vespa.filedistribution.status.Test +Container.com.yahoo.vespa.flags.Test +Container.com.yahoo.vespa.flags.custom.Test +Container.com.yahoo.vespa.flags.file.Test +Container.com.yahoo.vespa.flags.json.Test +Container.com.yahoo.vespa.flags.json.wire.Test +Container.com.yahoo.vespaget.Test +Container.com.yahoo.vespa.hadoop.Test +Container.com.yahoo.vespa.hadoop.mapreduce.Test +Container.com.yahoo.vespa.hadoop.mapreduce.util.Test +Container.com.yahoo.vespa.hadoop.pig.Test +Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.Test +Container.com.yahoo.vespa.hosted.athenz.instanceproviderservice.config.Test +Container.com.yahoo.vespa.hosted.ca.Test +Container.com.yahoo.vespa.hosted.ca.instance.Test +Container.com.yahoo.vespa.hosted.ca.restapi.Test +Container.com.yahoo.vespa.hosted.controller.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.Test +Container.com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.Test +Container.com.yahoo.vespa.hosted.controller.api.configserver.Test +Container.com.yahoo.vespa.hosted.controller.api.identifiers.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.archive.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.artifact.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.athenz.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.aws.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.billing.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.certificates.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.configserver.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.deployment.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.dns.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.entity.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.horizon.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.jira.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.maven.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.noderepository.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.organization.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.repair.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.resource.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.routing.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.secrets.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.stubs.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.user.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.vcmr.Test +Container.com.yahoo.vespa.hosted.controller.api.integration.zone.Test +Container.com.yahoo.vespa.hosted.controller.api.role.Test +Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.Test +Container.com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.Test +Container.com.yahoo.vespa.hosted.controller.application.Test +Container.com.yahoo.vespa.hosted.controller.application.pkg.Test +Container.com.yahoo.vespa.hosted.controller.archive.Test +Container.com.yahoo.vespa.hosted.controller.athenz.Test +Container.com.yahoo.vespa.hosted.controller.athenz.config.Test +Container.com.yahoo.vespa.hosted.controller.athenz.impl.Test +Container.com.yahoo.vespa.hosted.controller.auditlog.Test +Container.com.yahoo.vespa.hosted.controller.certificate.Test +Container.com.yahoo.vespa.hosted.controller.concurrent.Test +Container.com.yahoo.vespa.hosted.controller.deployment.Test +Container.com.yahoo.vespa.hosted.controller.dns.Test +Container.com.yahoo.vespa.hosted.controller.maintenance.Test +Container.com.yahoo.vespa.hosted.controller.metric.Test +Container.com.yahoo.vespa.hosted.controller.notification.Test +Container.com.yahoo.vespa.hosted.controller.persistence.Test +Container.com.yahoo.vespa.hosted.controller.proxy.Test +Container.com.yahoo.vespa.hosted.controller.restapi.application.Test +Container.com.yahoo.vespa.hosted.controller.restapi.athenz.Test +Container.com.yahoo.vespa.hosted.controller.restapi.billing.Test +Container.com.yahoo.vespa.hosted.controller.restapi.changemanagement.Test +Container.com.yahoo.vespa.hosted.controller.restapi.configserver.Test +Container.com.yahoo.vespa.hosted.controller.restapi.controller.Test +Container.com.yahoo.vespa.hosted.controller.restapi.deployment.Test +Container.com.yahoo.vespa.hosted.controller.restapi.filter.Test +Container.com.yahoo.vespa.hosted.controller.restapi.flags.Test +Container.com.yahoo.vespa.hosted.controller.restapi.horizon.Test +Container.com.yahoo.vespa.hosted.controller.restapi.os.Test +Container.com.yahoo.vespa.hosted.controller.restapi.routing.Test +Container.com.yahoo.vespa.hosted.controller.restapi.systemflags.Test +Container.com.yahoo.vespa.hosted.controller.restapi.user.Test +Container.com.yahoo.vespa.hosted.controller.restapi.zone.v1.Test +Container.com.yahoo.vespa.hosted.controller.restapi.zone.v2.Test +Container.com.yahoo.vespa.hosted.controller.routing.Test +Container.com.yahoo.vespa.hosted.controller.routing.context.Test +Container.com.yahoo.vespa.hosted.controller.routing.rotation.Test +Container.com.yahoo.vespa.hosted.controller.security.Test +Container.com.yahoo.vespa.hosted.controller.support.access.Test +Container.com.yahoo.vespa.hosted.controller.tenant.Test +Container.com.yahoo.vespa.hosted.controller.tls.Test +Container.com.yahoo.vespa.hosted.controller.versions.Test +Container.com.yahoo.vespa.hosted.node.admin.Test +Container.com.yahoo.vespa.hosted.node.admin.component.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.flags.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.state.Test +Container.com.yahoo.vespa.hosted.node.admin.configserver.state.bindings.Test +Container.com.yahoo.vespa.hosted.node.admin.container.Test +Container.com.yahoo.vespa.hosted.node.admin.container.image.Test +Container.com.yahoo.vespa.hosted.node.admin.container.metrics.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.acl.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.coredump.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.disk.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.identity.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.Test +Container.com.yahoo.vespa.hosted.node.admin.maintenance.sync.Test +Container.com.yahoo.vespa.hosted.node.admin.nodeadmin.Test +Container.com.yahoo.vespa.hosted.node.admin.nodeagent.Test +Container.com.yahoo.vespa.hosted.node.admin.provider.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.editor.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.file.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.fs.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.network.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.process.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.systemd.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.template.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.text.Test +Container.com.yahoo.vespa.hosted.node.admin.task.util.yum.Test +Container.com.yahoo.vespa.hosted.provision.Test +Container.com.yahoo.vespa.hosted.provision.applications.Test +Container.com.yahoo.vespa.hosted.provision.autoscale.Test +Container.com.yahoo.vespa.hosted.provision.lb.Test +Container.com.yahoo.vespa.hosted.provision.maintenance.Test +Container.com.yahoo.vespa.hosted.provision.node.Test +Container.com.yahoo.vespa.hosted.provision.node.filter.Test +Container.com.yahoo.vespa.hosted.provision.os.Test +Container.com.yahoo.vespa.hosted.provision.persistence.Test +Container.com.yahoo.vespa.hosted.provision.provisioning.Test +Container.com.yahoo.vespa.hosted.provision.restapi.Test +Container.com.yahoo.vespa.hosted.provision.testutils.Test +Container.com.yahoo.vespa.hosted.routing.Test +Container.com.yahoo.vespa.hosted.routing.nginx.Test +Container.com.yahoo.vespa.hosted.routing.restapi.Test +Container.com.yahoo.vespa.hosted.routing.status.Test +Container.com.yahoo.vespa.hosted.testrunner.Test +Container.com.yahoo.vespa.http.server.Test +Container.com.yahoo.vespa.http.server.util.Test +Container.com.yahoo.vespa.indexinglanguage.Test +Container.com.yahoo.vespa.indexinglanguage.expressions.Test +Container.com.yahoo.vespa.indexinglanguage.linguistics.Test +Container.com.yahoo.vespa.indexinglanguage.parser.Test +Container.com.yahoo.vespa.indexinglanguage.predicate.Test +Container.com.yahoo.vespa.jaxrs.annotation.Test +Container.com.yahoo.vespa.maven.plugin.enforcer.Test +Container.com.yahoo.vespa.model.Test +Container.com.yahoo.vespa.model.admin.Test +Container.com.yahoo.vespa.model.admin.clustercontroller.Test +Container.com.yahoo.vespa.model.admin.metricsproxy.Test +Container.com.yahoo.vespa.model.admin.monitoring.Test +Container.com.yahoo.vespa.model.admin.monitoring.builder.Test +Container.com.yahoo.vespa.model.admin.monitoring.builder.xml.Test +Container.com.yahoo.vespa.model.application.validation.Test +Container.com.yahoo.vespa.model.application.validation.change.Test +Container.com.yahoo.vespa.model.application.validation.change.search.Test +Container.com.yahoo.vespa.model.application.validation.first.Test +Container.com.yahoo.vespa.model.builder.Test +Container.com.yahoo.vespa.model.builder.xml.dom.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.docproc.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.processing.Test +Container.com.yahoo.vespa.model.builder.xml.dom.chains.search.Test +Container.com.yahoo.vespa.model.clients.Test +Container.com.yahoo.vespa.model.container.Test +Container.com.yahoo.vespa.model.container.component.Test +Container.com.yahoo.vespa.model.container.component.chain.Test +Container.com.yahoo.vespa.model.container.configserver.Test +Container.com.yahoo.vespa.model.container.configserver.option.Test +Container.com.yahoo.vespa.model.container.docproc.Test +Container.com.yahoo.vespa.model.container.docproc.model.Test +Container.com.yahoo.vespa.model.container.http.Test +Container.com.yahoo.vespa.model.container.http.ssl.Test +Container.com.yahoo.vespa.model.container.http.xml.Test +Container.com.yahoo.vespa.model.container.ml.Test +Container.com.yahoo.vespa.model.container.processing.Test +Container.com.yahoo.vespa.model.container.search.Test +Container.com.yahoo.vespa.model.container.search.searchchain.Test +Container.com.yahoo.vespa.model.container.search.searchchain.defaultsearchchains.Test +Container.com.yahoo.vespa.model.container.xml.Test +Container.com.yahoo.vespa.model.container.xml.document.Test +Container.com.yahoo.vespa.model.container.xml.embedder.Test +Container.com.yahoo.vespa.model.content.Test +Container.com.yahoo.vespa.model.content.cluster.Test +Container.com.yahoo.vespa.model.content.engines.Test +Container.com.yahoo.vespa.model.content.storagecluster.Test +Container.com.yahoo.vespa.model.filedistribution.Test +Container.com.yahoo.vespa.model.ml.Test +Container.com.yahoo.vespa.model.routing.Test +Container.com.yahoo.vespa.model.search.Test +Container.com.yahoo.vespa.model.utils.Test +Container.com.yahoo.vespa.model.utils.internal.Test +Container.com.yahoo.vespa.objects.Test +Container.com.yahoo.vespa.orchestrator.Test +Container.com.yahoo.vespa.orchestrator.config.Test +Container.com.yahoo.vespa.orchestrator.controller.Test +Container.com.yahoo.vespa.orchestrator.model.Test +Container.com.yahoo.vespa.orchestrator.policy.Test +Container.com.yahoo.vespa.orchestrator.resources.Test +Container.com.yahoo.vespa.orchestrator.restapi.wire.Test +Container.com.yahoo.vespa.orchestrator.status.Test +Container.com.yahoo.vespa.orchestrator.status.json.Test +Container.com.yahoo.vespa.security.tool.securityenv.Test +Container.com.yahoo.vespa.service.duper.Test +Container.com.yahoo.vespa.service.executor.Test +Container.com.yahoo.vespa.service.health.Test +Container.com.yahoo.vespa.service.manager.Test +Container.com.yahoo.vespa.service.model.Test +Container.com.yahoo.vespa.service.monitor.Test +Container.com.yahoo.vespa.service.slobrok.Test +Container.com.yahoo.vespastat.Test +Container.com.yahoo.vespa.streamingvisitors.Test +Container.com.yahoo.vespa.streamingvisitors.tracing.Test +Container.com.yahoo.vespasummarybenchmark.Test +Container.com.yahoo.vespa.test.file.Test +Container.com.yahoo.vespa.testrunner.Test +Container.com.yahoo.vespavisit.Test +Container.com.yahoo.vespaxmlparser.Test +Container.com.yahoo.vespa.zookeeper.Test +Container.com.yahoo.vespa.zookeeper.cli.Test +Container.com.yahoo.vespa.zookeeper.client.Test +Container.com.yahoo.yolean.Test +Container.com.yahoo.yolean.chain.Test +Container.com.yahoo.yolean.concurrent.Test +Container.com.yahoo.yolean.function.Test +Container.com.yahoo.yolean.system.Test +Container.com.yahoo.yolean.trace.Test diff --git a/client/go/script-utils/logfmt/internal_notnames.txt b/client/go/script-utils/logfmt/internal_notnames.txt new file mode 100644 index 00000000000..49543758ac3 --- /dev/null +++ b/client/go/script-utils/logfmt/internal_notnames.txt @@ -0,0 +1,8 @@ +Container.ai.onnx.Test +Container.com.fasterxml.jackson.jaxrs.json.Test +Container.com.google.Test +Container.com.yahooapis.foo.Test +Container.com.yahoo.newssearch.Test +Container.com.yahoo.Test +Container.com.yahoo.testing.Test +Container.org.apache.http.Test diff --git a/client/go/script-utils/logfmt/internal_test.go b/client/go/script-utils/logfmt/internal_test.go new file mode 100644 index 00000000000..9b6b0f8404c --- /dev/null +++ b/client/go/script-utils/logfmt/internal_test.go @@ -0,0 +1,34 @@ +package logfmt + +import ( + "bufio" + "os" + "testing" +) + +// tests: func isInternal(componentName string) bool + +func TestIsInternal(t *testing.T) { + f, err := os.Open("internal_names.txt") + if err != nil { + t.Fatal("could not read test data") + } + defer f.Close() + for input := bufio.NewScanner(f); input.Scan(); { + if name := input.Text(); !isInternal(name) { + t.Logf("name '%s' should be internal but was not recognized", name) + t.Fail() + } + } + f, err = os.Open("internal_notnames.txt") + if err != nil { + t.Fatal("could not read test data") + } + defer f.Close() + for input := bufio.NewScanner(f); input.Scan(); { + if name := input.Text(); isInternal(name) { + t.Logf("name '%s' should not be internal but was recognized", name) + t.Fail() + } + } +} diff --git a/client/go/script-utils/logfmt/levelflags.go b/client/go/script-utils/logfmt/levelflags.go new file mode 100644 index 00000000000..4e6c1284753 --- /dev/null +++ b/client/go/script-utils/logfmt/levelflags.go @@ -0,0 +1,72 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// handle CLI flags for log level filtering + +type flagValueForLevel struct { + levels map[string]bool + changed bool +} + +func defaultLevelFlags() map[string]bool { + return map[string]bool{ + "fatal": true, + "error": true, + "warning": true, + "info": true, + "config": false, + "event": false, + "debug": false, + "spam": false, + } +} + +func (v *flagValueForLevel) Type() string { + return "level flags" +} + +func (v *flagValueForLevel) String() string { + var buf strings.Builder + flagNames := []string{ + "fatal", + "error", + "warning", + "info", + "config", + "event", + "debug", + "spam", + } + for _, flag := range flagNames { + if v.levels[flag] { + buf.WriteString(" +") + } else { + buf.WriteString(" -") + } + buf.WriteString(flag) + } + return buf.String() +} + +func (v *flagValueForLevel) flags() map[string]bool { + return v.levels +} + +func (v *flagValueForLevel) name() string { + return "level" +} + +func (v *flagValueForLevel) unchanged() bool { + return !v.changed +} + +func (v *flagValueForLevel) Set(val string) error { + return applyPlusMinus(val, v) +} diff --git a/client/go/script-utils/logfmt/levelflags_test.go b/client/go/script-utils/logfmt/levelflags_test.go new file mode 100644 index 00000000000..186ea2d96b0 --- /dev/null +++ b/client/go/script-utils/logfmt/levelflags_test.go @@ -0,0 +1,74 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" + "testing" +) + +func TestLevelFlags(t *testing.T) { + none := " -fatal -error -warning -info -config -event -debug -spam" + + var flag flagValueForLevel + if flag.String() != none { + t.Logf("unset flag displays as '%s', expected '%s'", flag.String(), none) + t.Fail() + } + if flag.Type() != "level flags" { + t.Logf("flag type was '%s'", flag.Type()) + t.Fail() + } + check := func(expected string, texts ...string) { + var target flagValueForLevel + // target.levels = defaultLevelFlags() + target.levels = defaultLevelFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + t.Fatalf("unexpected error with level flags Set('%s'): %v", text, err) + } + } + got := target.String() + if got != expected { + t.Logf("expected flags [%s] but got: [%s]", expected, got) + t.Fail() + } + } + check(" +fatal +error +warning +info -config -event -debug -spam") + check(" -fatal -error -warning -info -config -event -debug -spam", "-all") + check(" +fatal +error +warning +info +config +event +debug +spam", "all") + check(" +fatal +error +warning +info +config +event +debug +spam", "+all") + check(" -fatal -error -warning -info -config -event +debug -spam", "debug") + check(" +fatal +error +warning +info +config +event +debug -spam", "all-spam") + check(" +fatal +error +warning +info +config +event +debug -spam", "all", "-spam") + check(" +fatal +error +warning -info -config +event -debug -spam", "+event", "-info") + check(" +fatal +error -warning -info -config +event -debug -spam", "+event,-info,-warning,config") + check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info,-warning,+config") + check(" +fatal +error -warning -info +config +event -debug -spam", "+event,-info", "-warning,+config") + check(" -fatal -error -warning -info +config -event -debug -spam", "+event", "-info", "-warning", "config") + check = func(expectErr string, texts ...string) { + var target flagValueForLevel + target.levels = defaultLevelFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + if err.Error() == expectErr { + return + } + t.Fatalf("expected error [%s] with level flags Set('%s'), but got [%v]", expectErr, text, err) + } + } + t.Logf("Did not get expected error '%s' from %s", expectErr, strings.Join(texts, ",")) + t.Fail() + } + check("not a valid level flag: 'foo'", "foo") + check("not a valid level flag: 'foo'", "event,foo,config") + check("not a valid level flag: 'foo'", "-event,-foo,-config") + check("not a valid level flag: 'foo'", "+event,+foo,+config") + check("not a valid level flag: 'foo'", "event", "foo", "config") + check("not a valid level flag: 'foo'", "-event", "-foo", "-config") + check("not a valid level flag: 'foo'", "+event", "+foo", "+config") +} diff --git a/client/go/script-utils/logfmt/options.go b/client/go/script-utils/logfmt/options.go new file mode 100644 index 00000000000..864868d4ce5 --- /dev/null +++ b/client/go/script-utils/logfmt/options.go @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "fmt" + "os" +) + +// options designed for compatibility with perl version of vespa-logfmt + +type Options struct { + ShowFields flagValueForShow + ShowLevels flagValueForLevel + OnlyHostname string + OnlyPid string + OnlyService string + OnlyInternal bool + FollowTail bool + DequoteNewlines bool + TruncateService bool + TruncateComponent bool + ComponentFilter regexFlag + MessageFilter regexFlag + Format OutputFormat +} + +func NewOptions() (ret Options) { + ret.ShowLevels.levels = defaultLevelFlags() + ret.ShowFields.shown = defaultShowFlags() + return +} + +func (o *Options) showField(field string) bool { + return o.ShowFields.shown[field] +} + +func (o *Options) showLevel(level string) bool { + rv, ok := o.ShowLevels.levels[level] + if !ok { + o.ShowLevels.levels[level] = true + fmt.Fprintf(os.Stderr, "Warnings: unknown level '%s' in input\n", level) + return true + } + return rv +} diff --git a/client/go/script-utils/logfmt/plusminusflag.go b/client/go/script-utils/logfmt/plusminusflag.go new file mode 100644 index 00000000000..1768cf0e7be --- /dev/null +++ b/client/go/script-utils/logfmt/plusminusflag.go @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "fmt" + "strings" +) + +// common code for showFlags and levelFlags + +type plusMinusFlag interface { + flags() map[string]bool + name() string + unchanged() bool +} + +func trimPrefix(value, prefix string) (newValue string, hadPrefix bool) { + hadPrefix = strings.HasPrefix(value, prefix) + if hadPrefix { + newValue = strings.TrimPrefix(value, prefix) + } else { + newValue = value + } + return +} + +func applyPlusMinus(val string, target plusMinusFlag) error { + minus := strings.HasPrefix(val, "-") + plus := strings.HasPrefix(val, "+") + val = strings.ReplaceAll(val, "-", ",-") + val = strings.ReplaceAll(val, "+", ",+") + if target.unchanged() { + // user wants to reset flags? + if minus == false && plus == false { + for k, _ := range target.flags() { + target.flags()[k] = false + } + } + } + changeTo := !minus + for _, k := range strings.Split(val, ",") { + if suppress, minus := trimPrefix(k, "-"); minus { + k = suppress + changeTo = false + } + if surface, plus := trimPrefix(k, "+"); plus { + k = surface + changeTo = true + } + if k == "" { + continue + } + if k == "all" { + for k, _ := range target.flags() { + target.flags()[k] = changeTo + } + } else if _, ok := target.flags()[k]; !ok { + return fmt.Errorf("not a valid %s flag: '%s'", target.name(), k) + } else { + target.flags()[k] = changeTo + } + } + return nil +} diff --git a/client/go/script-utils/logfmt/regexflag.go b/client/go/script-utils/logfmt/regexflag.go new file mode 100644 index 00000000000..8f7d2a91373 --- /dev/null +++ b/client/go/script-utils/logfmt/regexflag.go @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "regexp" +) + +// optional regular expression filter, as a CLI flag + +type regexFlag struct { + regex *regexp.Regexp +} + +func (re regexFlag) unmatched(s string) bool { + if re.regex == nil { + return false + } + return re.regex.FindStringIndex(s) == nil +} + +func (v *regexFlag) Type() string { + return "regular expression" +} + +func (v *regexFlag) String() string { + if v.regex == nil { + return "<none>" + } + return v.regex.String() +} + +func (v *regexFlag) Set(val string) (r error) { + v.regex, r = regexp.Compile(val) + return +} diff --git a/client/go/script-utils/logfmt/regexflag_test.go b/client/go/script-utils/logfmt/regexflag_test.go new file mode 100644 index 00000000000..489439863a2 --- /dev/null +++ b/client/go/script-utils/logfmt/regexflag_test.go @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" +) + +func assertMatch(t *testing.T, re string, flag *regexFlag, texts ...string) { + t.Helper() + err := flag.Set(re) + require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) + assert.Equal(t, re, flag.String(), "set flag displays as '%s', expected '%s'", flag.String(), re) + for _, text := range texts { + assert.False(t, flag.unmatched(text), "flag '%s' claims a non-match for '%s'", flag.String(), text) + } +} + +func assertUnmatch(t *testing.T, re string, flag *regexFlag, texts ...string) { + t.Helper() + err := flag.Set(re) + require.Nil(t, err, "unexpected error with flag.Set('%s'): %v", re, err) + assert.Equal(t, re, flag.String()) + for _, text := range texts { + assert.True(t, flag.unmatched(text), "flag '%s' should claim a non-match for '%s'", flag.String(), text) + } +} + +func TestRegexFlag(t *testing.T) { + var flag regexFlag + assert.Equal(t, "<none>", flag.String()) + assert.Equal(t, "regular expression", flag.Type()) + assert.False(t, flag.unmatched("foobar"), "unset flag claims a non-match") + assert.EqualError(t, flag.Set("*"), "error parsing regexp: missing argument to repetition operator: `*`") + assertMatch(t, "foo.*bar", new(regexFlag), "foobar", "foo bar", "x foobar y", "xfoobary", "xfooybarz") + assertUnmatch(t, "foo.*bar", new(regexFlag), "Foobar", "foo Bar", "fxoobar", "whatever") +} diff --git a/client/go/script-utils/logfmt/runlogfmt.go b/client/go/script-utils/logfmt/runlogfmt.go new file mode 100644 index 00000000000..5b9a3ac0870 --- /dev/null +++ b/client/go/script-utils/logfmt/runlogfmt.go @@ -0,0 +1,85 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "bufio" + "fmt" + "os" + + "github.com/vespa-engine/vespa/client/go/vespa" +) + +func inputIsPipe() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } else { + return true + } +} + +// main entry point for vespa-logfmt + +func RunLogfmt(opts *Options, args []string) { + if len(args) == 0 { + if !inputIsPipe() { + args = append(args, vespa.FindHome()+"/logs/vespa/vespa.log") + } else { + formatFile(opts, os.Stdin) + } + } + if opts.FollowTail { + if len(args) != 1 { + fmt.Fprintf(os.Stderr, "Must have exact 1 file for 'follow' option, got %d\n", len(args)) + return + } + if err := tailFile(opts, args[0]); err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + return + } + for _, arg := range args { + file, err := os.Open(arg) + if err != nil { + fmt.Fprintf(os.Stderr, "Cannot open '%s': %v\n", arg, err) + } else { + formatFile(opts, file) + file.Close() + } + } +} + +func formatLine(opts *Options, line string) { + output, err := handleLine(opts, line) + if err != nil { + fmt.Fprintln(os.Stderr, "bad log line:", err) + } else { + os.Stdout.WriteString(output) + } +} + +func tailFile(opts *Options, fn string) error { + tailed, err := FollowFile(fn) + if err != nil { + return err + } + for line := range tailed.Lines() { + formatLine(opts, line.Text) + } + return nil +} + +func formatFile(opts *Options, arg *os.File) { + input := bufio.NewScanner(arg) + input.Buffer(make([]byte, 64*1024), 4*1024*1024) + for input.Scan() { + formatLine(opts, input.Text()) + } +} diff --git a/client/go/script-utils/logfmt/showflags.go b/client/go/script-utils/logfmt/showflags.go new file mode 100644 index 00000000000..b69860e0312 --- /dev/null +++ b/client/go/script-utils/logfmt/showflags.go @@ -0,0 +1,76 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "strings" +) + +// handle CLI flags for which fields to show when formatting a line + +type flagValueForShow struct { + shown map[string]bool + changed bool +} + +func defaultShowFlags() map[string]bool { + return map[string]bool{ + "time": true, + "fmttime": true, + "msecs": true, + "usecs": false, + "host": false, + "level": true, + "pid": false, + "service": true, + "component": true, + "message": true, + } +} + +func (v *flagValueForShow) Type() string { + return "show flags" +} + +func (v *flagValueForShow) String() string { + var buf strings.Builder + flagNames := []string{ + "time", + "fmttime", + "msecs", + "usecs", + "host", + "level", + "pid", + "service", + "component", + "message", + } + for _, flag := range flagNames { + if v.shown[flag] { + buf.WriteString(" +") + } else { + buf.WriteString(" -") + } + buf.WriteString(flag) + } + return buf.String() +} + +func (v *flagValueForShow) flags() map[string]bool { + return v.shown +} + +func (v *flagValueForShow) name() string { + return "show" +} + +func (v *flagValueForShow) unchanged() bool { + return !v.changed +} + +func (v *flagValueForShow) Set(val string) error { + return applyPlusMinus(val, v) +} diff --git a/client/go/script-utils/logfmt/showflags_test.go b/client/go/script-utils/logfmt/showflags_test.go new file mode 100644 index 00000000000..d1b66118afd --- /dev/null +++ b/client/go/script-utils/logfmt/showflags_test.go @@ -0,0 +1,61 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +package logfmt + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func TestShowFlags(t *testing.T) { + none := " -time -fmttime -msecs -usecs -host -level -pid -service -component -message" + var flag flagValueForShow + assert.Equal(t, none, flag.String(), "unset flag displays as '%s', expected '%s'", flag.String(), none) + assert.Equal(t, "show flags", flag.Type()) + check := func(expected string, texts ...string) { + var target flagValueForShow + // target.levels = defaultLevelFlags() + target.shown = defaultShowFlags() + for _, text := range texts { + err := target.Set(text) + require.Nil(t, err, "unexpected error with show flags Set('%s'): %v", text, err) + } + assert.Equal(t, expected, target.String()) + } + check(" +time +fmttime +msecs -usecs -host +level -pid +service +component +message") + check(" -time -fmttime -msecs -usecs -host -level -pid -service -component -message", "-all") + check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "all") + check(" +time +fmttime +msecs +usecs +host +level +pid +service +component +message", "+all") + check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "pid") + check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all-host") + check(" +time +fmttime +msecs +usecs -host +level +pid +service +component +message", "all", "-host") + check(" +time +fmttime -msecs -usecs -host +level +pid +service +component +message", "+pid", "-msecs") + check(" +time -fmttime +msecs -usecs +host +level -pid -service +component +message", "+host,-fmttime,-service,pid") + check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime,-service,+pid") + check(" +time -fmttime +msecs -usecs +host +level +pid -service +component +message", "+host,-fmttime", "-service,+pid") + check(" -time -fmttime -msecs -usecs -host -level +pid -service -component -message", "+host", "-fmttime", "-service", "pid") + check = func(expectErr string, texts ...string) { + var target flagValueForShow + target.shown = defaultShowFlags() + for _, text := range texts { + err := target.Set(text) + if err != nil { + require.Equal(t, expectErr, err.Error()) + return + } + } + t.Logf("Did not get expected error [%s] from %s", expectErr, strings.Join(texts, " ")) + t.Fail() + } + check("not a valid show flag: 'foo'", "foo") + check("not a valid show flag: 'foo'", "level,foo,message") + check("not a valid show flag: 'foo'", "-level,-foo,-message") + check("not a valid show flag: 'foo'", "+level,+foo,+message") + check("not a valid show flag: 'foo'", "level", "foo", "message") + check("not a valid show flag: 'foo'", "-level", "-foo", "-message") + check("not a valid show flag: 'foo'", "+level", "+foo", "+message") +} diff --git a/client/go/script-utils/logfmt/tail.go b/client/go/script-utils/logfmt/tail.go new file mode 100644 index 00000000000..75e7cbb0693 --- /dev/null +++ b/client/go/script-utils/logfmt/tail.go @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +package logfmt + +type Line struct { + Text string +} + +type Tail interface { + Lines() chan Line +} diff --git a/client/go/script-utils/logfmt/tail_not_unix.go b/client/go/script-utils/logfmt/tail_not_unix.go new file mode 100644 index 00000000000..7030572575d --- /dev/null +++ b/client/go/script-utils/logfmt/tail_not_unix.go @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +//go:build windows + +package logfmt + +import ( + "fmt" +) + +func FollowFile(fn string) (Tail, error) { + return nil, fmt.Errorf("tail is not supported on this platform") +} diff --git a/client/go/script-utils/logfmt/tail_unix.go b/client/go/script-utils/logfmt/tail_unix.go new file mode 100644 index 00000000000..7703844da48 --- /dev/null +++ b/client/go/script-utils/logfmt/tail_unix.go @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +//go:build !windows + +package logfmt + +import ( + "bufio" + "fmt" + "io" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const lastLinesSize = 4 * 1024 + +// an active "tail -f" like object + +type unixTail struct { + lines chan Line + lineBuf []byte + curFile *os.File + fn string + reader *bufio.Reader + curStat unix.Stat_t +} + +func (t *unixTail) Lines() chan Line { return t.lines } + +// API for starting to follow a log file + +func FollowFile(fn string) (Tail, error) { + res := unixTail{} + res.fn = fn + res.lineBuf = make([]byte, lastLinesSize) + res.openTail() + res.lines = make(chan Line, 20) + res.lineBuf = res.lineBuf[:0] + go runTailWith(&res) + return &res, nil +} + +func (t *unixTail) setFile(f *os.File) { + if t.curFile != nil { + t.curFile.Close() + } + t.curFile = f + if f != nil { + err := unix.Fstat(int(f.Fd()), &t.curStat) + if err != nil { + f.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + return + } + t.reader = bufio.NewReaderSize(f, 1024*1024) + } else { + t.reader = nil + } +} + +// open log file and seek to the start of a line near the end, if possible. +func (t *unixTail) openTail() { + file, err := os.Open(t.fn) + if err != nil { + return + } + sz, err := file.Seek(0, os.SEEK_END) + if err != nil { + return + } + if sz < lastLinesSize { + sz, err = file.Seek(0, os.SEEK_SET) + if err == nil { + // just read from start of file, all OK + t.setFile(file) + } + return + } + sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) + n, err := file.Read(t.lineBuf) + if err != nil { + return + } + for i := 0; i < n; i++ { + if t.lineBuf[i] == '\n' { + sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) + if err == nil { + t.setFile(file) + } + return + } + } +} + +func (t *unixTail) reopen(cur *unix.Stat_t) { + for cnt := 0; cnt < 100; cnt++ { + file, err := os.Open(t.fn) + if err != nil { + t.setFile(nil) + if cnt == 0 { + fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) + } + time.Sleep(1000 * time.Millisecond) + continue + } + var stat unix.Stat_t + err = unix.Fstat(int(file.Fd()), &stat) + if err != nil { + file.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + time.Sleep(5000 * time.Millisecond) + continue + } + if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { + // same file, continue following it + file.Close() + return + } + // new file, start following it + t.setFile(file) + return + } +} + +// runs as a goroutine +func runTailWith(t *unixTail) { + defer t.setFile(nil) +loop: + for { + for t.curFile == nil { + t.reopen(nil) + } + bytes, err := t.reader.ReadSlice('\n') + t.lineBuf = append(t.lineBuf, bytes...) + if err == bufio.ErrBufferFull { + continue + } + if err == nil { + ll := len(t.lineBuf) - 1 + t.lines <- Line{Text: string(t.lineBuf[:ll])} + t.lineBuf = t.lineBuf[:0] + continue + } + if err == io.EOF { + pos, _ := t.curFile.Seek(0, os.SEEK_CUR) + for cnt := 0; cnt < 100; cnt++ { + time.Sleep(10 * time.Millisecond) + sz, _ := t.curFile.Seek(0, os.SEEK_END) + if sz != pos { + if sz < pos { + // truncation case + pos = 0 + } + t.curFile.Seek(pos, os.SEEK_SET) + continue loop + } + } + // no change in file size, try reopening + t.reopen(&t.curStat) + } else { + fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) + close(t.lines) + return + } + } +} |