diff options
52 files changed, 611 insertions, 150 deletions
diff --git a/client/go/build/build.go b/client/go/build/build.go index a51518dbb8f..a8342a9fb1e 100644 --- a/client/go/build/build.go +++ b/client/go/build/build.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package build var Version string = "0.0.0-devel" // Overriden by linker flag as part of build diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index b3284daa993..ba2df8c40dc 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa api-key command // Author: mpolden package cmd diff --git a/client/go/cmd/api_key_test.go b/client/go/cmd/api_key_test.go index 1deb628c21e..b08758ae21d 100644 --- a/client/go/cmd/api_key_test.go +++ b/client/go/cmd/api_key_test.go @@ -17,7 +17,7 @@ func TestAPIKey(t *testing.T) { out, _ := execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) assert.Contains(t, out, "Success: API private key written to "+keyFile+"\n") - out, _ = execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) - assert.Contains(t, out, "Error: File "+keyFile+" already exists\nHint: Use -f to overwrite it\n") + out, outErr := execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) + assert.Contains(t, outErr, "Error: File "+keyFile+" already exists\nHint: Use -f to overwrite it\n") assert.Contains(t, out, "This is your public key") } diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index 54a2c09256f..eaf3fc564dd 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa cert command // Author: mpolden package cmd diff --git a/client/go/cmd/cert_test.go b/client/go/cmd/cert_test.go index cd5f88764b9..96b626b5c98 100644 --- a/client/go/cmd/cert_test.go +++ b/client/go/cmd/cert_test.go @@ -28,8 +28,8 @@ func TestCert(t *testing.T) { assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), out) - out, _ = execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) - assert.Contains(t, out, fmt.Sprintf("Error: Application package %s already contains a certificate", appDir)) + _, outErr := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + assert.Contains(t, outErr, fmt.Sprintf("Error: Application package %s already contains a certificate", appDir)) } func TestCertCompressedPackage(t *testing.T) { @@ -41,13 +41,13 @@ func TestCertCompressedPackage(t *testing.T) { _, err = os.Create(zipFile) assert.Nil(t, err) - out, _ := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) - assert.Contains(t, out, "Error: Cannot add certificate to compressed application package") + _, outErr := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + assert.Contains(t, outErr, "Error: Cannot add certificate to compressed application package") err = os.Remove(zipFile) assert.Nil(t, err) - out, _ = execute(command{args: []string{"cert", "-f", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + out, _ := execute(command{args: []string{"cert", "-f", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) assert.Contains(t, out, "Success: Certificate written to") assert.Contains(t, out, "Success: Private key written to") } diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index 508ad49438f..9bae300c399 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa clone command // author: bratseth diff --git a/client/go/cmd/clone_list.go b/client/go/cmd/clone_list.go index f0ded5385cb..cb8e1acf4e9 100644 --- a/client/go/cmd/clone_list.go +++ b/client/go/cmd/clone_list.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( diff --git a/client/go/cmd/clone_list_test.go b/client/go/cmd/clone_list_test.go index 9ef4be47c9c..1138e5de064 100644 --- a/client/go/cmd/clone_list_test.go +++ b/client/go/cmd/clone_list_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( diff --git a/client/go/cmd/clone_test.go b/client/go/cmd/clone_test.go index 6cf11dd4d40..054dc7b21fb 100644 --- a/client/go/cmd/clone_test.go +++ b/client/go/cmd/clone_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // init command tests // Author: bratseth diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index 6929b59decb..8eaf6be2c22 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -27,6 +27,18 @@ type command struct { moreArgs []string } +func resetFlag(f *pflag.Flag) { + switch v := f.Value.(type) { + case pflag.SliceValue: + _ = v.Replace([]string{}) + default: + switch v.Type() { + case "bool", "string", "int": + _ = v.Set(f.DefValue) + } + } +} + func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) { if client != nil { util.ActiveHttpClient = client @@ -44,17 +56,8 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) os.Setenv("VESPA_CLI_CACHE_DIR", cmd.cacheDir) // Reset flags to their default value - persistent flags in Cobra persists over tests - rootCmd.Flags().VisitAll(func(f *pflag.Flag) { - switch v := f.Value.(type) { - case pflag.SliceValue: - _ = v.Replace([]string{}) - default: - switch v.Type() { - case "bool", "string", "int": - _ = v.Set(f.DefValue) - } - } - }) + rootCmd.Flags().VisitAll(resetFlag) + documentCmd.Flags().VisitAll(resetFlag) // Do not exit in tests exitFunc = func(code int) {} diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 863f247bd7c..a5d6a05a048 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa config command // author: bratseth diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index 25ba7cc0655..0e74e53c5e5 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( @@ -9,7 +10,7 @@ import ( func TestConfig(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") - assertConfigCommand(t, "invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") + assertConfigCommandErr(t, "invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") assertConfigCommand(t, "foo = <unset>\n", homeDir, "config", "get", "foo") assertConfigCommand(t, "target = local\n", homeDir, "config", "get", "target") assertConfigCommand(t, "", homeDir, "config", "set", "target", "cloud") @@ -18,15 +19,15 @@ func TestConfig(t *testing.T) { assertConfigCommand(t, "", homeDir, "config", "set", "target", "https://127.0.0.1") assertConfigCommand(t, "target = https://127.0.0.1\n", homeDir, "config", "get", "target") - assertConfigCommand(t, "invalid application: \"foo\"\n", homeDir, "config", "set", "application", "foo") + assertConfigCommandErr(t, "invalid application: \"foo\"\n", homeDir, "config", "set", "application", "foo") assertConfigCommand(t, "application = <unset>\n", homeDir, "config", "get", "application") assertConfigCommand(t, "", homeDir, "config", "set", "application", "t1.a1.i1") assertConfigCommand(t, "application = t1.a1.i1\n", homeDir, "config", "get", "application") - assertConfigCommand(t, "application = t1.a1.i1\ncolor = auto\ntarget = https://127.0.0.1\nwait = 0\n", homeDir, "config", "get") + assertConfigCommand(t, "application = t1.a1.i1\ncolor = auto\nquiet = false\ntarget = https://127.0.0.1\nwait = 0\n", homeDir, "config", "get") assertConfigCommand(t, "", homeDir, "config", "set", "wait", "60") - assertConfigCommand(t, "wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") + assertConfigCommandErr(t, "wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") assertConfigCommand(t, "wait = 60\n", homeDir, "config", "get", "wait") } @@ -34,3 +35,8 @@ func assertConfigCommand(t *testing.T, expected, homeDir string, args ...string) out, _ := execute(command{homeDir: homeDir, args: args}, t, nil) assert.Equal(t, expected, out) } + +func assertConfigCommandErr(t *testing.T, expected, homeDir string, args ...string) { + _, outErr := execute(command{homeDir: homeDir, args: args}, t, nil) + assert.Equal(t, expected, outErr) +} diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index b3171d184e0..1380eca5bbb 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa deploy command // Author: bratseth diff --git a/client/go/cmd/deploy_test.go b/client/go/cmd/deploy_test.go index 9614806b968..5bb45e70fad 100644 --- a/client/go/cmd/deploy_test.go +++ b/client/go/cmd/deploy_test.go @@ -61,9 +61,10 @@ func TestDeployApplicationDirectoryWithPomAndTarget(t *testing.T) { func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) { client := &mockHttpClient{} + _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withEmptyTarget"}}, t, client) assert.Equal(t, "Error: pom.xml exists but no target/application.zip. Run mvn package first\n", - executeCommand(t, client, []string{"deploy", "testdata/applications/withEmptyTarget"}, []string{})) + outErr) } func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) { @@ -85,7 +86,7 @@ func TestDeployApplicationPackageErrorWithExpectedFormat(t *testing.T) { "Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here", `{ "error-code": "INVALID_APPLICATION_PACKAGE", - "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here\n" + "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" }`) } @@ -94,7 +95,7 @@ func TestPrepareApplicationPackageErrorWithExpectedFormat(t *testing.T) { "Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here", `{ "error-code": "INVALID_APPLICATION_PACKAGE", - "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here\n" + "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here" }`) } @@ -158,18 +159,20 @@ func assertDeployRequestMade(target string, client *mockHttpClient, t *testing.T assertPackageUpload(-1, target+"/application/v2/tenant/default/prepareandactivate", client, t) } -func assertApplicationPackageError(t *testing.T, command string, status int, expectedMessage string, returnBody string) { +func assertApplicationPackageError(t *testing.T, cmd string, status int, expectedMessage string, returnBody string) { client := &mockHttpClient{} client.NextResponse(status, returnBody) + _, outErr := execute(command{args: []string{cmd, "testdata/applications/withTarget/target/application.zip"}}, t, client) assert.Equal(t, "Error: Invalid application package (Status "+strconv.Itoa(status)+")\n\n"+expectedMessage+"\n", - executeCommand(t, client, []string{command, "testdata/applications/withTarget/target/application.zip"}, []string{})) + outErr) } func assertDeployServerError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} client.NextResponse(status, errorMessage) + _, outErr := execute(command{args: []string{"deploy", "testdata/applications/withTarget/target/application.zip"}}, t, client) assert.Equal(t, "Error: Error from deploy service at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n", - executeCommand(t, client, []string{"deploy", "testdata/applications/withTarget/target/application.zip"}, []string{})) + outErr) } diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index cc5fb948e3b..1d27a475172 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -5,9 +5,9 @@ package cmd import ( + "fmt" "io" "io/ioutil" - "log" "strings" "github.com/spf13/cobra" @@ -123,20 +123,29 @@ func curlOutput() io.Writer { } func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) { + out := stdout if !result.Success { - log.Print(color.Red("Error: "), result.Message) + out = stderr + } + + if !result.Success { + fmt.Fprintln(out, color.Red("Error:"), result.Message) } else if !(payloadOnlyOnSuccess && result.Payload != "") { - log.Print(color.Green("Success: "), result.Message) + fmt.Fprintln(out, color.Green("Success:"), result.Message) } if result.Detail != "" { - log.Print(color.Yellow(result.Detail)) + fmt.Fprintln(out, color.Yellow(result.Detail)) } if result.Payload != "" { if !payloadOnlyOnSuccess { - log.Println("") + fmt.Fprintln(out) } - log.Print(result.Payload) + fmt.Fprintln(out, result.Payload) + } + + if !result.Success { + exitFunc(1) } } diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index 1f82b85f915..649aca8703a 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -67,17 +67,19 @@ func TestDocumentRemoveWithoutIdArg(t *testing.T) { func TestDocumentSendMissingId(t *testing.T) { arguments := []string{"document", "put", "testdata/A-Head-Full-of-Dreams-Without-Operation.json"} client := &mockHttpClient{} + _, outErr := execute(command{args: arguments}, t, client) assert.Equal(t, "Error: No document id given neither as argument or as a 'put' key in the json file\n", - executeCommand(t, client, arguments, []string{})) + outErr) } func TestDocumentSendWithDisagreeingOperations(t *testing.T) { arguments := []string{"document", "update", "testdata/A-Head-Full-of-Dreams-Put.json"} client := &mockHttpClient{} + _, outErr := execute(command{args: arguments}, t, client) assert.Equal(t, "Error: Wanted document operation is update but the JSON file specifies put\n", - executeCommand(t, client, arguments, []string{})) + outErr) } func TestDocumentPutDocumentError(t *testing.T) { @@ -139,21 +141,23 @@ func assertDocumentGet(arguments []string, documentId string, t *testing.T) { func assertDocumentError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} client.NextResponse(status, errorMessage) + _, outErr := execute(command{args: []string{"document", "put", + "id:mynamespace:music::a-head-full-of-dreams", + "testdata/A-Head-Full-of-Dreams-Put.json"}}, t, client) assert.Equal(t, "Error: Invalid document operation: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", - executeCommand(t, client, []string{"document", "put", - "id:mynamespace:music::a-head-full-of-dreams", - "testdata/A-Head-Full-of-Dreams-Put.json"}, []string{})) + outErr) } func assertDocumentServerError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} client.NextResponse(status, errorMessage) + _, outErr := execute(command{args: []string{"document", "put", + "id:mynamespace:music::a-head-full-of-dreams", + "testdata/A-Head-Full-of-Dreams-Put.json"}}, t, client) assert.Equal(t, "Error: Container (document API) at 127.0.0.1:8080: Status "+strconv.Itoa(status)+"\n\n"+errorMessage+"\n", - executeCommand(t, client, []string{"document", "put", - "id:mynamespace:music::a-head-full-of-dreams", - "testdata/A-Head-Full-of-Dreams-Put.json"}, []string{})) + outErr) } func documentServiceURL(client *mockHttpClient) string { diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 98d6814d16f..b5525cf11fe 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -32,16 +32,16 @@ func fatalErr(err error, msg ...interface{}) { func printErrHint(err error, hints ...string) { printErr(nil, err.Error()) for _, hint := range hints { - log.Print(color.Cyan("Hint: "), hint) + fmt.Fprintln(stderr, color.Cyan("Hint:"), hint) } } func printErr(err error, msg ...interface{}) { if len(msg) > 0 { - log.Print(color.Red("Error: "), fmt.Sprint(msg...)) + fmt.Fprintln(stderr, color.Red("Error:"), fmt.Sprint(msg...)) } if err != nil { - log.Print(color.Yellow(err)) + fmt.Fprintln(stderr, color.Yellow(err)) } } @@ -215,11 +215,9 @@ func waitForService(service string, sessionOrRunID int64) { if status/100 == 2 { log.Print(s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Green("ready")) } else { - log.Print(s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready")) if err == nil { - log.Print(color.Yellow(fmt.Sprintf("Status %d", status))) - } else { - log.Print(color.Yellow(err)) + err = fmt.Errorf("Status %d", status) } + fatalErr(err, s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Red("not ready")) } } diff --git a/client/go/cmd/log.go b/client/go/cmd/log.go new file mode 100644 index 00000000000..4577e890959 --- /dev/null +++ b/client/go/cmd/log.go @@ -0,0 +1,98 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/vespa" +) + +var ( + fromArg string + toArg string + levelArg string + followArg bool + dequoteArg bool +) + +func init() { + rootCmd.AddCommand(logCmd) + logCmd.Flags().StringVarP(&fromArg, "from", "F", "", "Include logs since this timestamp (RFC3339 format)") + logCmd.Flags().StringVarP(&toArg, "to", "T", "", "Include logs until this timestamp (RFC3339 format)") + logCmd.Flags().StringVarP(&levelArg, "level", "l", "debug", `The maximum log level to show. Must be "error", "warning", "info" or "debug"`) + logCmd.Flags().BoolVarP(&followArg, "follow", "f", false, "Follow logs") + logCmd.Flags().BoolVarP(&dequoteArg, "nldequote", "n", true, "Dequote LF and TAB characters in log messages") +} + +var logCmd = &cobra.Command{ + Use: "log [relative-period]", + Short: "Show the Vespa log", + Long: `Show the Vespa log. + +The logs shown can be limited to a relative or fixed period. All timestamps are shown in UTC. +`, + Example: `$ vespa log 1h +$ vespa log --nldequote=false 10m +$ vespa log --from 2021-08-25T15:00:00Z --to 2021-08-26T02:00:00Z +$ vespa log --follow`, + DisableAutoGenTag: true, + Args: cobra.MaximumNArgs(1), + Run: func(cmd *cobra.Command, args []string) { + target := getTarget() + options := vespa.LogOptions{ + Level: vespa.LogLevel(levelArg), + Follow: followArg, + Writer: stdout, + Dequote: dequoteArg, + } + if options.Follow { + if fromArg != "" || toArg != "" || len(args) > 0 { + fatalErr(fmt.Errorf("cannot combine --from/--to or relative time with --follow"), "Could not follow logs") + } + options.From = time.Now().Add(-5 * time.Minute) + } else { + from, to, err := parsePeriod(args) + if err != nil { + fatalErr(err, "Invalid period") + return + } + options.From = from + options.To = to + } + if err := target.PrintLog(options); err != nil { + fatalErr(err, "Could not retrieve logs") + } + }, +} + +func parsePeriod(args []string) (time.Time, time.Time, error) { + if len(args) == 1 { + if fromArg != "" || toArg != "" { + return time.Time{}, time.Time{}, fmt.Errorf("cannot combine --from/--to with relative value: %s", args[0]) + } + d, err := time.ParseDuration(args[0]) + if err != nil { + return time.Time{}, time.Time{}, err + } + if d > 0 { + d = -d + } + to := time.Now() + from := to.Add(d) + return from, to, nil + } + from, err := time.Parse(time.RFC3339, fromArg) + if err != nil { + return time.Time{}, time.Time{}, err + } + to, err := time.Parse(time.RFC3339, toArg) + if err != nil { + return time.Time{}, time.Time{}, err + } + if !to.After(from) { + return time.Time{}, time.Time{}, fmt.Errorf("--to must specify a time after --from") + } + return from, to, nil +} diff --git a/client/go/cmd/log_test.go b/client/go/cmd/log_test.go new file mode 100644 index 00000000000..f239bebc488 --- /dev/null +++ b/client/go/cmd/log_test.go @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLog(t *testing.T) { + homeDir := filepath.Join(t.TempDir(), ".vespa") + pkgDir := mockApplicationPackage(t, false) + httpClient := &mockHttpClient{} + httpClient.NextResponse(200, `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532`) + execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + + out, _ := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T10:00:00Z", "--to", "2021-09-27T11:00:00Z"}}, t, httpClient) + + expected := "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication Switching to the latest deployed set of configurations and components. Application config generation: 52532\n" + assert.Equal(t, expected, out) + + _, errOut := execute(command{homeDir: homeDir, args: []string{"log", "--from", "2021-09-27T13:12:49Z", "--to", "2021-09-27T13:15:00", "1h"}}, t, httpClient) + assert.Equal(t, "Error: Invalid period\ncannot combine --from/--to with relative value: 1h\n", errOut) +} diff --git a/client/go/cmd/man.go b/client/go/cmd/man.go index ff7f6fb1b6a..d90898117de 100644 --- a/client/go/cmd/man.go +++ b/client/go/cmd/man.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( diff --git a/client/go/cmd/man_test.go b/client/go/cmd/man_test.go index f7c33c8b3a1..dfbe04f4c8e 100644 --- a/client/go/cmd/man_test.go +++ b/client/go/cmd/man_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index f05914eb9a7..5e2b268865d 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa query command // author: bratseth @@ -47,7 +47,7 @@ func query(arguments []string) { response, err := service.Do(&http.Request{URL: url}, time.Second*10) if err != nil { - log.Print(color.Red("Error: "), "Request failed: ", err) + printErr(nil, "Request failed: ", err) return } defer response.Body.Close() @@ -55,11 +55,9 @@ func query(arguments []string) { if response.StatusCode == 200 { log.Print(util.ReaderToJSON(response.Body)) } else if response.StatusCode/100 == 4 { - log.Print(color.Red("Error: "), "Invalid query: ", response.Status, "\n") - log.Print(util.ReaderToJSON(response.Body)) + printErr(nil, "Invalid query: ", response.Status, "\n", util.ReaderToJSON(response.Body)) } else { - log.Print(color.Red("Error: "), response.Status, " from container at ", color.Cyan(url.Host), "\n") - log.Print(util.ReaderToJSON(response.Body)) + printErr(nil, response.Status, " from container at ", color.Cyan(url.Host), "\n", util.ReaderToJSON(response.Body)) } } diff --git a/client/go/cmd/query_test.go b/client/go/cmd/query_test.go index 137ffa01cd5..81dc03766be 100644 --- a/client/go/cmd/query_test.go +++ b/client/go/cmd/query_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // query command tests // Author: bratseth @@ -57,18 +57,20 @@ func assertQuery(t *testing.T, expectedQuery string, query ...string) { func assertQueryError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} client.NextResponse(status, errorMessage) + _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) assert.Equal(t, "Error: Invalid query: Status "+strconv.Itoa(status)+"\n"+errorMessage+"\n", - executeCommand(t, client, []string{"query"}, []string{"yql=select from sources * where title contains 'foo'"}), + outErr, "error output") } func assertQueryServiceError(t *testing.T, status int, errorMessage string) { client := &mockHttpClient{} client.NextResponse(status, errorMessage) + _, outErr := execute(command{args: []string{"query", "yql=select from sources * where title contains 'foo'"}}, t, client) assert.Equal(t, "Error: Status "+strconv.Itoa(status)+" from container at 127.0.0.1:8080\n"+errorMessage+"\n", - executeCommand(t, client, []string{"query"}, []string{"yql=select from sources * where title contains 'foo'"}), + outErr, "error output") } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index cd8427c3ac6..7fe704e4918 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" + "io/ioutil" "log" "os" @@ -36,6 +37,7 @@ Vespa documentation: https://docs.vespa.ai`, applicationArg string waitSecsArg int colorArg string + quietArg bool color = aurora.NewAurora(false) stdout = colorable.NewColorableStdout() @@ -47,17 +49,23 @@ const ( targetFlag = "target" waitFlag = "wait" colorFlag = "color" + quietFlag = "quiet" ) func isTerminal() bool { - file, ok := stdout.(*os.File) - if ok { - return isatty.IsTerminal(file.Fd()) + if f, ok := stdout.(*os.File); ok { + return isatty.IsTerminal(f.Fd()) + } + if f, ok := stderr.(*os.File); ok { + return isatty.IsTerminal(f.Fd()) } return false } func configureOutput() { + if quietArg { + stdout = ioutil.Discard + } log.SetFlags(0) // No timestamps log.SetOutput(stdout) @@ -88,10 +96,12 @@ func init() { rootCmd.PersistentFlags().StringVarP(&applicationArg, applicationFlag, "a", "", "The application to manage") rootCmd.PersistentFlags().IntVarP(&waitSecsArg, waitFlag, "w", 0, "Number of seconds to wait for a service to become ready") rootCmd.PersistentFlags().StringVarP(&colorArg, colorFlag, "c", "auto", "Whether to use colors in output. Can be \"auto\", \"never\" or \"always\"") + rootCmd.PersistentFlags().BoolVarP(&quietArg, quietFlag, "q", false, "Quiet mode. Only errors are printed.") bindFlagToConfig(targetFlag, rootCmd) bindFlagToConfig(applicationFlag, rootCmd) bindFlagToConfig(waitFlag, rootCmd) bindFlagToConfig(colorFlag, rootCmd) + bindFlagToConfig(quietFlag, rootCmd) } // Execute executes the root command. diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index 5fdcaa07d8a..c72df481547 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa status command // author: bratseth diff --git a/client/go/cmd/status_test.go b/client/go/cmd/status_test.go index 0c1c8e4e3a7..757ef5f3b06 100644 --- a/client/go/cmd/status_test.go +++ b/client/go/cmd/status_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // status command tests // Author: bratseth @@ -78,8 +78,11 @@ func assertDocumentStatus(target string, args []string, t *testing.T) { func assertQueryStatusError(target string, args []string, t *testing.T) { client := &mockHttpClient{} client.NextStatus(500) + cmd := []string{"status", "container"} + cmd = append(cmd, args...) + _, outErr := execute(command{args: cmd}, t, client) assert.Equal(t, - "Container (query API) at "+target+" is not ready\nStatus 500\n", - executeCommand(t, client, []string{"status", "container"}, args), + "Error: Container (query API) at "+target+" is not ready\nStatus 500\n", + outErr, "vespa status container") } diff --git a/client/go/cmd/version.go b/client/go/cmd/version.go index 749d17a41d9..d2760402851 100644 --- a/client/go/cmd/version.go +++ b/client/go/cmd/version.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( diff --git a/client/go/cmd/version_test.go b/client/go/cmd/version_test.go index 9eeaaaa4692..039f75a6ecd 100644 --- a/client/go/cmd/version_test.go +++ b/client/go/cmd/version_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( diff --git a/client/go/cmd/vespa/main.go b/client/go/cmd/vespa/main.go index 73183da2a16..5fdf64f5ab4 100644 --- a/client/go/cmd/vespa/main.go +++ b/client/go/cmd/vespa/main.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Cobra commands main file // Author: bratseth diff --git a/client/go/util/http.go b/client/go/util/http.go index a8e2dbc2195..acd9bb4f7ec 100644 --- a/client/go/util/http.go +++ b/client/go/util/http.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // A HTTP wrapper which handles some errors and provides a way to replace the HTTP client by a mock. // Author: bratseth diff --git a/client/go/util/http_test.go b/client/go/util/http_test.go index 47c710bb068..0a0de1fdd4c 100644 --- a/client/go/util/http_test.go +++ b/client/go/util/http_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Basic testing of our HTTP client wrapper // Author: bratseth diff --git a/client/go/util/io.go b/client/go/util/io.go index 51361e344f0..f51c6060cb7 100644 --- a/client/go/util/io.go +++ b/client/go/util/io.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // File utilities. // Author: bratseth diff --git a/client/go/util/operation_result.go b/client/go/util/operation_result.go index fba73a68dc9..5e79f727d4e 100644 --- a/client/go/util/operation_result.go +++ b/client/go/util/operation_result.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // A struct containing the result of an operation // Author: bratseth diff --git a/client/go/version/version.go b/client/go/version/version.go index 27b7da1d0f5..00e26d25135 100644 --- a/client/go/version/version.go +++ b/client/go/version/version.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package version import ( diff --git a/client/go/version/version_test.go b/client/go/version/version_test.go index 3602715cca8..759b1c1d0c1 100644 --- a/client/go/version/version_test.go +++ b/client/go/version/version_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package version import ( diff --git a/client/go/vespa/crypto.go b/client/go/vespa/crypto.go index fb336b88210..b4a5a5b7da8 100644 --- a/client/go/vespa/crypto.go +++ b/client/go/vespa/crypto.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa import ( diff --git a/client/go/vespa/crypto_test.go b/client/go/vespa/crypto_test.go index 87d6587c850..89d50d15d70 100644 --- a/client/go/vespa/crypto_test.go +++ b/client/go/vespa/crypto_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa import ( diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index 19319724d18..eec0182b0ce 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa deploy API // Author: bratseth diff --git a/client/go/vespa/deploy_test.go b/client/go/vespa/deploy_test.go index 32b31eebf7c..d353dafca19 100644 --- a/client/go/vespa/deploy_test.go +++ b/client/go/vespa/deploy_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa import ( diff --git a/client/go/vespa/id.go b/client/go/vespa/id.go index ad6e289586e..b0dc770ad52 100644 --- a/client/go/vespa/id.go +++ b/client/go/vespa/id.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa document ids // Author: bratseth diff --git a/client/go/vespa/id_test.go b/client/go/vespa/id_test.go index 61183465186..343affc1602 100644 --- a/client/go/vespa/id_test.go +++ b/client/go/vespa/id_test.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa import ( diff --git a/client/go/vespa/log.go b/client/go/vespa/log.go new file mode 100644 index 00000000000..0e2cb5d0bfd --- /dev/null +++ b/client/go/vespa/log.go @@ -0,0 +1,100 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package vespa + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +var dequoter = strings.NewReplacer("\\n", "\n", "\\t", "\t") + +// LogEntry represents a Vespa log entry. +type LogEntry struct { + Time time.Time + Host string + Service string + Component string + Level string + Message string +} + +func (le *LogEntry) Format(dequote bool) string { + t := le.Time.Format("2006-01-02 15:04:05.000000") + msg := le.Message + if dequote { + msg = dequoter.Replace(msg) + } + return fmt.Sprintf("[%s] %-8s %-7s %-16s %s\t%s", t, le.Host, le.Level, le.Service, le.Component, msg) +} + +// ParseLogEntry parses a Vespa log entry from string s. +func ParseLogEntry(s string) (LogEntry, error) { + parts := strings.SplitN(s, "\t", 7) + if len(parts) != 7 { + return LogEntry{}, fmt.Errorf("invalid number of log parts: %d: %q", len(parts), s) + } + time, err := parseLogTimestamp(parts[0]) + if err != nil { + return LogEntry{}, err + } + return LogEntry{ + Time: time, + Host: parts[1], + Service: parts[3], + Component: parts[4], + Level: parts[5], + Message: parts[6], + }, nil +} + +// ReadLogEntries reads and parses all log entries from reader r. +func ReadLogEntries(r io.Reader) ([]LogEntry, error) { + var entries []LogEntry + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + logEntry, err := ParseLogEntry(line) + if err != nil { + return nil, err + } + entries = append(entries, logEntry) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return entries, nil +} + +// LogLevel returns an int representing a named log level. +func LogLevel(name string) int { + switch name { + case "error": + return 0 + case "warning": + return 1 + case "info": + return 2 + default: // everything else, e.g. debug + return 3 + } +} + +func parseLogTimestamp(s string) (time.Time, error) { + parts := strings.Split(s, ".") + if len(parts) != 2 { + return time.Time{}, fmt.Errorf("invalid number of log timestamp parts: %d", len(parts)) + } + unixSecs, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("invalid timestamp seconds: %s", parts[0]) + } + unixMicros, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("invalid timestamp microseconds: %s", parts[1]) + } + return time.Unix(unixSecs, unixMicros*1000).UTC(), nil +} diff --git a/client/go/vespa/log_test.go b/client/go/vespa/log_test.go new file mode 100644 index 00000000000..2d0c75d0a0a --- /dev/null +++ b/client/go/vespa/log_test.go @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package vespa + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseLogEntry(t *testing.T) { + expected := LogEntry{ + Time: time.Date(2021, 9, 27, 10, 31, 30, 905535000, time.UTC), + Host: "host1a.dev.aws-us-east-1c", + Service: "logserver-container", + Component: "Container.com.yahoo.container.jdisc.ConfiguredApplication", + Level: "info", + Message: "Switching to the latest deployed set of configurations and components. Application config generation: 52532", + } + in := "1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532" + logEntry, err := ParseLogEntry(in) + assert.Nil(t, err) + assert.Equal(t, expected, logEntry) + + formatted := "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication\tSwitching to the latest deployed set of configurations and components. Application config generation: 52532" + assert.Equal(t, formatted, logEntry.Format(false)) + + in = "1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info message containing newline\\nand\\ttab" + logEntry, err = ParseLogEntry(in) + assert.Nil(t, err) + assert.Equal(t, "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication\tmessage containing newline\nand\ttab", logEntry.Format(true)) +} diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index aa0ddb8babb..8a09440f5cc 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -1,11 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa import ( + "bytes" "crypto/tls" "encoding/json" "fmt" "io" "io/ioutil" + "math" "net/http" "net/url" "sort" @@ -41,6 +44,9 @@ type Target interface { // Service returns the service for given name. If timeout is non-zero, wait for the service to converge. Service(name string, timeout time.Duration, sessionOrRunID int64) (*Service, error) + + // PrintLog writes the logs of this deployment using given options to control output. + PrintLog(options LogOptions) error } // TLSOptions configures the certificate to use for service requests. @@ -50,10 +56,14 @@ type TLSOptions struct { PrivateKeyFile string } -// LogOptions configures the log output to produce when waiting for services. +// LogOptions configures the log output to produce when writing log messages. type LogOptions struct { - Writer io.Writer - Level int + From time.Time + To time.Time + Follow bool + Dequote bool + Writer io.Writer + Level int } type customTarget struct { @@ -119,6 +129,10 @@ func (t *customTarget) Service(name string, timeout time.Duration, sessionID int return nil, fmt.Errorf("unknown service: %s", name) } +func (t *customTarget) PrintLog(options LogOptions) error { + return fmt.Errorf("reading logs from non-cloud deployment is currently unsupported") +} + func (t *customTarget) urlWithPort(serviceName string) (string, error) { u, err := url.Parse(t.baseURL) if err != nil { @@ -207,6 +221,64 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64) ( return nil, fmt.Errorf("unknown service: %s", name) } +func (t *cloudTarget) logsURL() string { + return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs", + t.apiURL, + t.deployment.Application.Tenant, t.deployment.Application.Application, t.deployment.Application.Instance, + t.deployment.Zone.Environment, t.deployment.Zone.Region) +} + +func (t *cloudTarget) PrintLog(options LogOptions) error { + req, err := http.NewRequest("GET", t.logsURL(), nil) + if err != nil { + return err + } + signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) + lastFrom := options.From + requestFunc := func() *http.Request { + fromMillis := lastFrom.Unix() * 1000 + q := req.URL.Query() + q.Set("from", strconv.FormatInt(fromMillis, 10)) + if !options.To.IsZero() { + toMillis := options.To.Unix() * 1000 + q.Set("to", strconv.FormatInt(toMillis, 10)) + } + req.URL.RawQuery = q.Encode() + if err := signer.SignRequest(req); err != nil { + panic(err) + } + return req + } + logFunc := func(status int, response []byte) (bool, error) { + if ok, err := isOK(status); !ok { + return ok, err + } + logEntries, err := ReadLogEntries(bytes.NewReader(response)) + if err != nil { + return true, err + } + for _, le := range logEntries { + if !le.Time.After(lastFrom) { + continue + } + if LogLevel(le.Level) > options.Level { + continue + } + fmt.Fprintln(options.Writer, le.Format(options.Dequote)) + } + if len(logEntries) > 0 { + lastFrom = logEntries[len(logEntries)-1].Time + } + return false, nil + } + var timeout time.Duration + if options.Follow { + timeout = math.MaxInt64 // No timeout + } + _, err = wait(logFunc, requestFunc, &t.tlsOptions.KeyPair, timeout) + return err +} + func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error { signer := NewRequestSigner(t.deployment.Application.SerializedForm(), t.apiKey) if runID > 0 { @@ -348,20 +420,6 @@ func CloudTarget(apiURL string, deployment Deployment, apiKey []byte, tlsOptions } } -// LogLevel returns an int representing a named log level. -func LogLevel(name string) int { - switch name { - case "error": - return 0 - case "warning": - return 1 - case "info": - return 2 - default: // everything else, e.g. debug - return 3 - } -} - type deploymentEndpoint struct { URL string `json:"url"` } diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index 2c90baefbbc..ed924059297 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -1,9 +1,12 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa import ( "bytes" "crypto/tls" "fmt" + "io" + "io/ioutil" "net/http" "net/http/httptest" "testing" @@ -40,6 +43,11 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request case "/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge": response := fmt.Sprintf(`{"converged": %t}`, v.deploymentConverged) w.Write([]byte(response)) + case "/application/v4/tenant/t1/application/a1/instance/i1/environment/dev/region/us-north-1/logs": + log := `1632738690.905535 host1a.dev.aws-us-east-1c 806/53 logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication info Switching to the latest deployed set of configurations and components. Application config generation: 52532 +1632738698.600189 host1a.dev.aws-us-east-1c 1723/33590 config-sentinel sentinel.sentinel.config-owner config Sentinel got 3 service elements [tenant(vespa-team), application(music), instance(mpolden)] for config generation 52532 +` + w.Write([]byte(log)) case "/status.html": w.Write([]byte("OK")) case "/ApplicationStatus": @@ -92,6 +100,44 @@ func TestCloudTargetWait(t *testing.T) { defer srv.Close() vc.serverURL = srv.URL + var logWriter bytes.Buffer + target := createCloudTarget(t, srv.URL, &logWriter) + assertServiceWait(t, 200, target, "deploy") + + _, err := target.Service("query", time.Millisecond, 42) + assert.NotNil(t, err) + + vc.deploymentConverged = true + _, err = target.Service("query", time.Millisecond, 42) + assert.Nil(t, err) + + assertServiceWait(t, 500, target, "query") + assertServiceWait(t, 500, target, "document") + + // Log timestamp is converted to local time, do the same here in case the local time where tests are run varies + tm := time.Unix(1631707708, 431000) + expectedTime := tm.Format("[15:04:05]") + assert.Equal(t, expectedTime+" info Deploying platform version 7.465.17 and application version 1.0.2 ...\n", logWriter.String()) +} + +func TestLog(t *testing.T) { + vc := mockVespaApi{} + srv := httptest.NewServer(http.HandlerFunc(vc.mockVespaHandler)) + defer srv.Close() + vc.serverURL = srv.URL + vc.deploymentConverged = true + + var buf bytes.Buffer + target := createCloudTarget(t, srv.URL, ioutil.Discard) + if err := target.PrintLog(LogOptions{Writer: &buf, Level: 3}); err != nil { + t.Fatal(err) + } + expected := "[2021-09-27 10:31:30.905535] host1a.dev.aws-us-east-1c info logserver-container Container.com.yahoo.container.jdisc.ConfiguredApplication\tSwitching to the latest deployed set of configurations and components. Application config generation: 52532\n" + + "[2021-09-27 10:31:38.600189] host1a.dev.aws-us-east-1c config config-sentinel sentinel.sentinel.config-owner\tSentinel got 3 service elements [tenant(vespa-team), application(music), instance(mpolden)] for config generation 52532\n" + assert.Equal(t, expected, buf.String()) +} + +func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { kp, err := CreateKeyPair() assert.Nil(t, err) @@ -100,7 +146,6 @@ func TestCloudTargetWait(t *testing.T) { apiKey, err := CreateAPIKey() assert.Nil(t, err) - var logWriter bytes.Buffer target := CloudTarget( "https://example.com", Deployment{ @@ -109,28 +154,13 @@ func TestCloudTargetWait(t *testing.T) { }, apiKey, TLSOptions{KeyPair: x509KeyPair}, - LogOptions{Writer: &logWriter}) + LogOptions{Writer: logWriter}) if ct, ok := target.(*cloudTarget); ok { - ct.apiURL = srv.URL + ct.apiURL = url } else { t.Fatalf("Wrong target type %T", ct) } - assertServiceWait(t, 200, target, "deploy") - - _, err = target.Service("query", time.Millisecond, 42) - assert.NotNil(t, err) - - vc.deploymentConverged = true - _, err = target.Service("query", time.Millisecond, 42) - assert.Nil(t, err) - - assertServiceWait(t, 500, target, "query") - assertServiceWait(t, 500, target, "document") - - // Log timestamp is converted to local time, do the same here in case the local time where tests are run varies - tm := time.Unix(1631707708, 431000) - expectedTime := tm.Format("[15:04:05]") - assert.Equal(t, expectedTime+" info Deploying platform version 7.465.17 and application version 1.0.2 ...\n", logWriter.String()) + return target } func assertServiceURL(t *testing.T, url string, target Target, service string) { diff --git a/client/go/vespa/version.go b/client/go/vespa/version.go index e5a89f4cb58..b20c6d360d7 100644 --- a/client/go/vespa/version.go +++ b/client/go/vespa/version.go @@ -1,3 +1,4 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package vespa // Version is the Vespa CLI version number diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 40071f90c34..b7ce40f19a2 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -7074,6 +7074,7 @@ ], "methods": [ "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean)", + "public void <init>(com.fasterxml.jackson.core.JsonGenerator, boolean, boolean)", "public void accept(java.lang.String, java.lang.Object)", "public void accept(java.lang.String, byte[], int, int)", "protected boolean shouldRender(java.lang.String, java.lang.Object)", @@ -7439,6 +7440,7 @@ "public static com.yahoo.search.result.FeatureData empty()", "public com.yahoo.data.access.Inspector inspect()", "public java.lang.String toJson()", + "public java.lang.String toJson(boolean)", "public java.lang.StringBuilder writeJson(java.lang.StringBuilder)", "public java.lang.Double getDouble(java.lang.String)", "public com.yahoo.tensor.Tensor getTensor(java.lang.String)", diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 0a87ad7ec2b..ee0e7f4fe0e 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -45,6 +45,7 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.NanNumber; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.serialization.JsonFormat; import java.io.IOException; import java.io.OutputStream; @@ -58,7 +59,6 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; import java.util.Deque; -import java.util.Iterator; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -78,6 +78,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); + private static final CompoundName TENSOR_FORMAT = new CompoundName("format.tensors"); // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; @@ -127,6 +128,8 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private LongSupplier timeSource; private OutputStream stream; + private boolean tensorShortFormRendering = false; + public JsonRenderer() { this(null); } @@ -166,6 +169,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { public void beginResponse(OutputStream stream) throws IOException { beginJsonCallback(stream); debugRendering = getDebugRendering(getResult().getQuery()); + tensorShortFormRendering = getTensorShortFormRendering(getResult().getQuery()); setGenerator(generatorFactory.createGenerator(stream, JsonEncoding.UTF8), debugRendering); renderedChildren = new ArrayDeque<>(); generator.writeStartObject(); @@ -200,6 +204,12 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return q != null && q.properties().getBoolean(DEBUG_RENDERING_KEY, false); } + private boolean getTensorShortFormRendering(Query q) { + if (q == null || q.properties().get(TENSOR_FORMAT) == null) + return false; + return q.properties().getString(TENSOR_FORMAT).equalsIgnoreCase("short"); + } + protected void renderTrace(Trace trace) throws IOException { if (!trace.traceNode().children().iterator().hasNext()) return; if (getResult().getQuery().getTraceLevel() == 0) return; @@ -285,8 +295,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { generator.writeEndObject(); } generator.writeEndArray(); - - } protected void renderCoverage() throws IOException { @@ -510,7 +518,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected FieldConsumer createFieldConsumer(JsonGenerator generator, boolean debugRendering) { - return new FieldConsumer(generator, debugRendering); + return new FieldConsumer(generator, debugRendering, tensorShortFormRendering); } /** @@ -529,12 +537,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private final JsonGenerator generator; private final boolean debugRendering; + private final boolean tensorShortForm; private MutableBoolean hasFieldsField; public FieldConsumer(JsonGenerator generator, boolean debugRendering) { + this(generator, debugRendering, false); + } + + public FieldConsumer(JsonGenerator generator, boolean debugRendering, boolean tensorShortForm) { this.generator = generator; this.debugRendering = debugRendering; + this.tensorShortForm = tensorShortForm; } /** @@ -659,7 +673,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } else if (field instanceof Tensor) { renderTensor(Optional.of((Tensor)field)); } else if (field instanceof FeatureData) { - generator.writeRawValue(((FeatureData)field).toJson()); + generator.writeRawValue(((FeatureData)field).toJson(tensorShortForm)); } else if (field instanceof Inspectable) { renderInspectorDirect(((Inspectable)field).inspect()); } else if (field instanceof JsonProducer) { @@ -697,26 +711,18 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } private void renderTensor(Optional<Tensor> tensor) throws IOException { - generator.writeStartObject(); - generator.writeArrayFieldStart("cells"); - if (tensor.isPresent()) { - for (Iterator<Tensor.Cell> i = tensor.get().cellIterator(); i.hasNext(); ) { - Tensor.Cell cell = i.next(); - - generator.writeStartObject(); - - generator.writeObjectFieldStart("address"); - for (int d = 0; d < cell.getKey().size(); d++) - generator.writeObjectField(tensor.get().type().dimensions().get(d).name(), cell.getKey().label(d)); - generator.writeEndObject(); - - generator.writeObjectField("value", cell.getValue()); - - generator.writeEndObject(); - } + if (tensor.isEmpty()) { + generator.writeStartObject(); + generator.writeArrayFieldStart("cells"); + generator.writeEndArray(); + generator.writeEndObject(); + return; + } + if (tensorShortForm) { + generator.writeRawValue(new String(JsonFormat.encodeShortForm(tensor.get()), StandardCharsets.UTF_8)); + } else { + generator.writeRawValue(new String(JsonFormat.encode(tensor.get()), StandardCharsets.UTF_8)); } - generator.writeEndArray(); - generator.writeEndObject(); } } diff --git a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java index fd41d4ee10c..7673352576d 100644 --- a/container-search/src/main/java/com/yahoo/search/result/FeatureData.java +++ b/container-search/src/main/java/com/yahoo/search/result/FeatureData.java @@ -62,9 +62,17 @@ public class FeatureData implements Inspectable, JsonProducer { return jsonForm; } + public String toJson(boolean tensorShortForm) { + if (this == empty) return "{}"; + if (jsonForm != null) return jsonForm; + + jsonForm = JsonRender.render(value, new Encoder(new StringBuilder(), true, tensorShortForm)).toString(); + return jsonForm; + } + @Override public StringBuilder writeJson(StringBuilder target) { - return JsonRender.render(value, new Encoder(target, true)); + return JsonRender.render(value, new Encoder(target, true, false)); } /** @@ -162,15 +170,19 @@ public class FeatureData implements Inspectable, JsonProducer { /** A JSON encoder which encodes DATA as a tensor */ private static class Encoder extends JsonRender.StringEncoder { - Encoder(StringBuilder out, boolean compact) { + private final boolean tensorShortForm; + + Encoder(StringBuilder out, boolean compact, boolean tensorShortForm) { super(out, compact); + this.tensorShortForm = tensorShortForm; } @Override public void encodeDATA(byte[] value) { // This could be done more efficiently ... - target().append(new String(JsonFormat.encodeWithType(TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value))), - StandardCharsets.UTF_8)); + Tensor tensor = TypedBinaryFormat.decode(Optional.empty(), GrowableByteBuffer.wrap(value)); + byte[] encodedTensor = tensorShortForm ? JsonFormat.encodeShortForm(tensor) : JsonFormat.encodeWithType(tensor); + target().append(new String(encodedTensor, StandardCharsets.UTF_8)); } } diff --git a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java index d0df131ae31..290b7266a3a 100644 --- a/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/rendering/JsonRendererTestCase.java @@ -124,6 +124,55 @@ public class JsonRendererTestCase { } @Test + public void testTensorShortForm() throws ExecutionException, InterruptedException, IOException { + String expected = "{" + + "\"root\":{" + + "\"id\":\"toplevel\"," + + "\"relevance\":1.0," + + "\"fields\":{" + + "\"totalCount\":1" + + "}," + + "\"children\":[{" + + "\"id\":\"tensors\"," + + "\"relevance\":1.0," + + "\"fields\":{" + + "\"tensor_standard\":{\"type\":\"tensor(x{},y{})\",\"cells\":[{\"address\":{\"x\":\"a\",\"y\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"b\",\"y\":\"1\"},\"value\":2.0}]}," + + "\"tensor_indexed\":{\"type\":\"tensor(x[2],y[3])\",\"values\":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}," + + "\"tensor_single_mapped\":{\"type\":\"tensor(x{})\",\"cells\":{\"a\":1.0,\"b\":2.0}}," + + "\"tensor_mixed\":{\"type\":\"tensor(x{},y[2])\",\"blocks\":{\"a\":[1.0,2.0],\"b\":[3.0,4.0]}}," + + "\"summaryfeatures\":{" + + "\"tensor_standard\":{\"type\":\"tensor(x{},y{})\",\"cells\":[{\"address\":{\"x\":\"a\",\"y\":\"0\"},\"value\":1.0},{\"address\":{\"x\":\"b\",\"y\":\"1\"},\"value\":2.0}]}," + + "\"tensor_indexed\":{\"type\":\"tensor(x[2],y[3])\",\"values\":[[1.0,2.0,3.0],[4.0,5.0,6.0]]}," + + "\"tensor_single_mapped\":{\"type\":\"tensor(x{})\",\"cells\":{\"a\":1.0,\"b\":2.0}}," + + "\"tensor_mixed\":{\"type\":\"tensor(x{},y[2])\",\"blocks\":{\"a\":[1.0,2.0],\"b\":[3.0,4.0]}}" + + "}" + + "}" + + "}]" + + "}}\n"; + + Slime slime = new Slime(); + Cursor features = slime.setObject(); + features.setData("tensor_standard", TypedBinaryFormat.encode(Tensor.from("tensor(x{},y{}):{ {x:a,y:0}:1.0, {x:b,y:1}:2.0 }"))); + features.setData("tensor_indexed", TypedBinaryFormat.encode(Tensor.from("tensor(x[2],y[3]):[[1,2,3],[4,5,6]]"))); + features.setData("tensor_single_mapped", TypedBinaryFormat.encode(Tensor.from("tensor(x{}):{ a:1, b:2 }"))); + features.setData("tensor_mixed", TypedBinaryFormat.encode(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"))); + FeatureData summaryFeatures = new FeatureData(new SlimeAdapter(slime.get())); + + Hit h = new Hit("tensors"); + h.setField("tensor_standard", new TensorFieldValue(Tensor.from("tensor(x{},y{}):{ {x:a,y:0}:1.0, {x:b,y:1}:2.0 }"))); + h.setField("tensor_indexed", new TensorFieldValue(Tensor.from("tensor(x[2],y[3]):[[1,2,3],[4,5,6]]"))); + h.setField("tensor_single_mapped", new TensorFieldValue(Tensor.from("tensor(x{}):{ a:1, b:2 }"))); + h.setField("tensor_mixed", new TensorFieldValue(Tensor.from("tensor(x{},y[2]):{a:[1,2], b:[3,4]}"))); + h.setField("summaryfeatures", summaryFeatures); + + Result r = new Result(new Query("/?format.tensors=short")); + r.hits().add(h); + r.setTotalHitCount(1L); + String summary = render(r); + assertEqualJson(expected, summary); + } + + @Test public void testDataTypes() throws IOException, InterruptedException, ExecutionException { String expected = "{" + " \"root\": {" @@ -463,7 +512,6 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - @Test public void testTracingOfNodesWithBothChildrenAndDataAndEmptySubnode() throws IOException, InterruptedException, ExecutionException { String expected = "{" @@ -553,7 +601,6 @@ public class JsonRendererTestCase { assertEqualJson(expected, summary); } - @Test public void test() throws IOException, InterruptedException, ExecutionException { String expected = "{" diff --git a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java index 9e365056355..8283d79fa2e 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java +++ b/model-evaluation/src/main/java/ai/vespa/models/handler/ModelsEvaluationHandler.java @@ -10,7 +10,6 @@ import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.JsonFormat; @@ -90,7 +89,7 @@ public class ModelsEvaluationHandler extends ThreadedHttpRequestHandler { } Tensor result = evaluator.evaluate(); - Optional<String> format = property(request, "format"); + Optional<String> format = property(request, "format.tensors"); if (format.isPresent() && format.get().equalsIgnoreCase("short")) { return new Response(200, JsonFormat.encodeShortForm(result)); } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 7029be24a60..992747b7866 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -1,7 +1,6 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; -import ai.vespa.models.evaluation.ModelTester; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporterWithMockedConstants; import com.yahoo.config.subscription.ConfigGetter; @@ -186,7 +185,7 @@ public class ModelsEvaluationHandlerTest { public void testMnistSoftmaxEvaluateSpecificFunctionWithShortOutput() { Map<String, String> properties = new HashMap<>(); properties.put("Placeholder", inputTensorShortForm()); - properties.put("format", "short"); + properties.put("format.tensors", "short"); String url = "http://localhost/model-evaluation/v1/mnist_softmax/default.add/eval"; String expected = "{\"type\":\"tensor(d0[],d1[10])\",\"values\":[[-0.3546536862850189,0.3759574592113495,0.06054411828517914,-0.251544713973999,0.017951013520359993,1.2899067401885986,-0.10389615595340729,0.6367976665496826,-1.4136744737625122,-0.2573896050453186]]}"; handler.assertResponse(url, properties, 200, expected); @@ -216,7 +215,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testVespaModelShortOutput() { Map<String, String> properties = new HashMap<>(); - properties.put("format", "short"); + properties.put("format.tensors", "short"); String url = "http://localhost/model-evaluation/v1/vespa_model/"; handler.assertResponse(url + "test_mapped/eval", properties, 200, "{\"type\":\"tensor(d0{})\",\"cells\":{\"a\":1.0,\"b\":2.0}}"); @@ -231,7 +230,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testVespaModelLiteralOutput() { Map<String, String> properties = new HashMap<>(); - properties.put("format", "string"); + properties.put("format.tensors", "string"); String url = "http://localhost/model-evaluation/v1/vespa_model/"; handler.assertResponse(url + "test_mapped/eval", properties, 200, "tensor(d0{}):{a:1.0,b:2.0}"); |