diff options
65 files changed, 564 insertions, 306 deletions
diff --git a/.copr/Makefile b/.copr/Makefile index 5b097ba0ad9..b0322bf29b3 100644 --- a/.copr/Makefile +++ b/.copr/Makefile @@ -1,17 +1,14 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. TOP = $(realpath $(dir $(lastword $(MAKEFILE_LIST)))) -# Version -VESPA_VERSION := $(shell git tag --points-at HEAD | grep -oP "\d+\.\d+\.\d+" | sort -V | tail -1) - RPMTOPDIR := $(HOME)/rpmbuild SOURCEDIR := $(RPMTOPDIR)/SOURCES SPECDIR := $(RPMTOPDIR)/SPECS SPECFILE := $(SPECDIR)/vespa-$(VESPA_VERSION).spec srpm: - dnf install -y rpmdevtools - $(TOP)/../dist.sh $(VESPA_VERSION) + dnf install -y git rpmdevtools + $(TOP)/../dist.sh $$(git tag --points-at HEAD | grep -oP "\d+\.\d+\.\d+" | sort -V | tail -1) spectool -g -C $(SOURCEDIR) $(SPECFILE) rpmbuild -bs --define "_topdir $(RPMTOPDIR)" $(SPECFILE) cp -a $(RPMTOPDIR)/SRPMS/* $(outdir) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3598d259144..7d9968b6329 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -58,7 +58,6 @@ add_subdirectory(config) add_subdirectory(config-model-fat) add_subdirectory(configd) add_subdirectory(configdefinitions) -add_subdirectory(configgen) add_subdirectory(configserver) add_subdirectory(configserver-flags) add_subdirectory(configutil) diff --git a/client/go/cmd/api_key_test.go b/client/go/cmd/api_key_test.go index c00f520aa25..2497568604f 100644 --- a/client/go/cmd/api_key_test.go +++ b/client/go/cmd/api_key_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: mpolden package cmd @@ -14,10 +14,10 @@ func TestAPIKey(t *testing.T) { homeDir := t.TempDir() keyFile := homeDir + "/.vespa/t1.api-key.pem" - out := execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) + out, _ := execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) assert.True(t, strings.HasPrefix(out, "Success: API private key written to "+keyFile+"\n")) - out = execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) + out, _ = execute(command{args: []string{"api-key", "-a", "t1.a1.i1"}, homeDir: homeDir}, t, nil) assert.True(t, strings.HasPrefix(out, "Error: File "+keyFile+" already exists\nHint: Use -f to overwrite it\n")) assert.True(t, strings.Contains(out, "This is your public key")) } diff --git a/client/go/cmd/cert_test.go b/client/go/cmd/cert_test.go index 36abdae1787..d93def2fa70 100644 --- a/client/go/cmd/cert_test.go +++ b/client/go/cmd/cert_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Author: mpolden package cmd @@ -16,7 +16,7 @@ import ( func TestCert(t *testing.T) { homeDir := t.TempDir() pkgDir := mockApplicationPackage(t, false) - out := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + out, _ := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) app, err := vespa.ApplicationFromString("t1.a1.i1") assert.Nil(t, err) @@ -28,7 +28,7 @@ func TestCert(t *testing.T) { assert.Equal(t, fmt.Sprintf("Success: Certificate written to %s\nSuccess: Certificate written to %s\nSuccess: Private key written to %s\n", pkgCertificate, certificate, privateKey), out) - out = execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + out, _ = execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) assert.Contains(t, out, fmt.Sprintf("Error: Application package %s already contains a certificate", appDir)) } @@ -41,13 +41,13 @@ func TestCertCompressedPackage(t *testing.T) { _, err = os.Create(zipFile) assert.Nil(t, err) - out := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + out, _ := execute(command{args: []string{"cert", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) assert.Contains(t, out, "Error: Cannot add certificate to compressed application package") err = os.Remove(zipFile) assert.Nil(t, err) - out = execute(command{args: []string{"cert", "-f", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) + out, _ = execute(command{args: []string{"cert", "-f", "-a", "t1.a1.i1", pkgDir}, homeDir: homeDir}, t, nil) assert.Contains(t, out, "Success: Certificate written to") assert.Contains(t, out, "Success: Private key written to") } diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index 3d19e772875..f455ffa9957 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // A helper for testing commands // Author: bratseth @@ -17,7 +17,6 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "github.com/stretchr/testify/assert" "github.com/vespa-engine/vespa/client/go/util" ) @@ -27,7 +26,7 @@ type command struct { moreArgs []string } -func execute(cmd command, t *testing.T, client *mockHttpClient) string { +func execute(cmd command, t *testing.T, client *mockHttpClient) (string, string) { if client != nil { util.ActiveHttpClient = client } @@ -56,19 +55,20 @@ func execute(cmd command, t *testing.T, client *mockHttpClient) string { exitFunc = func(code int) {} // Capture stdout and execute command - var b bytes.Buffer - stdout = &b + var capturedOut bytes.Buffer + var capturedErr bytes.Buffer + stdout = &capturedOut + stderr = &capturedErr // Execute command and return output rootCmd.SetArgs(append(cmd.args, cmd.moreArgs...)) rootCmd.Execute() - out, err := ioutil.ReadAll(&b) - assert.Nil(t, err, "No error") - return string(out) + return capturedOut.String(), capturedErr.String() } func executeCommand(t *testing.T, client *mockHttpClient, args []string, moreArgs []string) string { - return execute(command{args: args, moreArgs: moreArgs}, t, client) + out, _ := execute(command{args: args, moreArgs: moreArgs}, t, client) + return out } type mockHttpClient struct { diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index 70ad5d558e1..cf50f561f0f 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -8,23 +8,28 @@ import ( func TestConfig(t *testing.T) { homeDir := t.TempDir() - assert.Equal(t, "invalid option or value: \"foo\": \"bar\"\n", execute(command{homeDir: homeDir, args: []string{"config", "set", "foo", "bar"}}, t, nil)) - assert.Equal(t, "foo = <unset>\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "foo"}}, t, nil)) - assert.Equal(t, "target = local\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "target"}}, t, nil)) - assert.Equal(t, "", execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, nil)) - assert.Equal(t, "target = cloud\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "target"}}, t, nil)) - assert.Equal(t, "", execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "http://127.0.0.1:8080"}}, t, nil)) - assert.Equal(t, "", execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "https://127.0.0.1"}}, t, nil)) - assert.Equal(t, "target = https://127.0.0.1\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "target"}}, t, nil)) + assertConfigCommand(t, "invalid option or value: \"foo\": \"bar\"\n", homeDir, "config", "set", "foo", "bar") + assertConfigCommand(t, "foo = <unset>\n", homeDir, "config", "get", "foo") + assertConfigCommand(t, "target = local\n", homeDir, "config", "get", "target") + assertConfigCommand(t, "", homeDir, "config", "set", "target", "cloud") + assertConfigCommand(t, "target = cloud\n", homeDir, "config", "get", "target") + assertConfigCommand(t, "", homeDir, "config", "set", "target", "http://127.0.0.1:8080") + assertConfigCommand(t, "", homeDir, "config", "set", "target", "https://127.0.0.1") + assertConfigCommand(t, "target = https://127.0.0.1\n", homeDir, "config", "get", "target") - assert.Equal(t, "invalid application: \"foo\"\n", execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "foo"}}, t, nil)) - assert.Equal(t, "application = <unset>\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "application"}}, t, nil)) - assert.Equal(t, "", execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, nil)) - assert.Equal(t, "application = t1.a1.i1\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "application"}}, t, nil)) + assertConfigCommand(t, "invalid application: \"foo\"\n", homeDir, "config", "set", "application", "foo") + assertConfigCommand(t, "application = <unset>\n", homeDir, "config", "get", "application") + assertConfigCommand(t, "", homeDir, "config", "set", "application", "t1.a1.i1") + assertConfigCommand(t, "application = t1.a1.i1\n", homeDir, "config", "get", "application") - assert.Equal(t, "application = t1.a1.i1\ncolor = auto\ntarget = https://127.0.0.1\nwait = 0\n", execute(command{homeDir: homeDir, args: []string{"config", "get"}}, t, nil)) + assertConfigCommand(t, "application = t1.a1.i1\ncolor = auto\ntarget = https://127.0.0.1\nwait = 0\n", homeDir, "config", "get") - assert.Equal(t, "", execute(command{homeDir: homeDir, args: []string{"config", "set", "wait", "60"}}, t, nil)) - assert.Equal(t, "wait option must be an integer >= 0, got \"foo\"\n", execute(command{homeDir: homeDir, args: []string{"config", "set", "wait", "foo"}}, t, nil)) - assert.Equal(t, "wait = 60\n", execute(command{homeDir: homeDir, args: []string{"config", "get", "wait"}}, t, nil)) + assertConfigCommand(t, "", homeDir, "config", "set", "wait", "60") + assertConfigCommand(t, "wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") + assertConfigCommand(t, "wait = 60\n", homeDir, "config", "get", "wait") +} + +func assertConfigCommand(t *testing.T, expected, homeDir string, args ...string) { + out, _ := execute(command{homeDir: homeDir, args: args}, t, nil) + assert.Equal(t, expected, out) } diff --git a/client/go/cmd/curl.go b/client/go/cmd/curl.go index f6b40e10f35..bd9fad1b47e 100644 --- a/client/go/cmd/curl.go +++ b/client/go/cmd/curl.go @@ -1,22 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( - "fmt" "log" "os" - "os/exec" "strings" - "github.com/kballard/go-shellquote" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/curl" ) var curlDryRun bool -var curlPath string func init() { rootCmd.AddCommand(curlCmd) - curlCmd.Flags().StringVarP(&curlPath, "path", "p", "", "The path to curl. If this is unset, curl from PATH is used") curlCmd.Flags().BoolVarP(&curlDryRun, "dry-run", "n", false, "Print the curl command that would be executed") } @@ -50,16 +47,20 @@ $ vespa curl -t local -- -v /search/?yql=query return } service := getService("query", 0) - c := &curl{privateKeyPath: privateKeyFile, certificatePath: certificateFile} + url := joinURL(service.BaseURL, args[len(args)-1]) + rawArgs := args[:len(args)-1] + c, err := curl.RawArgs(url, rawArgs...) + if err != nil { + fatalErr(err) + return + } + c.PrivateKey = privateKeyFile + c.Certificate = certificateFile + if curlDryRun { - cmd, err := c.command(service.BaseURL, args...) - if err != nil { - fatalErr(err, "Failed to create curl command") - return - } - log.Print(shellquote.Join(cmd.Args...)) + log.Print(c.String()) } else { - if err := c.run(service.BaseURL, args...); err != nil { + if err := c.Run(os.Stdout, os.Stderr); err != nil { fatalErr(err, "Failed to run curl") return } @@ -67,72 +68,8 @@ $ vespa curl -t local -- -v /search/?yql=query }, } -type curl struct { - path string - certificatePath string - privateKeyPath string -} - -func (c *curl) run(baseURL string, args ...string) error { - cmd, err := c.command(baseURL, args...) - if err != nil { - return err - } - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Start(); err != nil { - return err - } - return cmd.Wait() -} - -func (c *curl) command(baseURL string, args ...string) (*exec.Cmd, error) { - if len(args) == 0 { - return nil, fmt.Errorf("need at least one argument") - } - - if c.path == "" { - resolvedPath, err := resolveCurlPath() - if err != nil { - return nil, err - } - c.path = resolvedPath - } - - path := args[len(args)-1] - args = args[:len(args)-1] - if !hasOption("--key", args) && c.privateKeyPath != "" { - args = append(args, "--key", c.privateKeyPath) - } - if !hasOption("--cert", args) && c.certificatePath != "" { - args = append(args, "--cert", c.certificatePath) - } - +func joinURL(baseURL, path string) string { baseURL = strings.TrimSuffix(baseURL, "/") path = strings.TrimPrefix(path, "/") - args = append(args, baseURL+"/"+path) - - return exec.Command(c.path, args...), nil -} - -func hasOption(option string, args []string) bool { - for _, arg := range args { - if arg == option { - return true - } - } - return false -} - -func resolveCurlPath() (string, error) { - var curlPath string - var err error - curlPath, err = exec.LookPath("curl") - if err != nil { - curlPath, err = exec.LookPath("curl.exe") - if err != nil { - return "", err - } - } - return curlPath, nil + return baseURL + "/" + path } diff --git a/client/go/cmd/curl_test.go b/client/go/cmd/curl_test.go index c3163e731ce..340eacd0bd3 100644 --- a/client/go/cmd/curl_test.go +++ b/client/go/cmd/curl_test.go @@ -1,9 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package cmd import ( "fmt" "path/filepath" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -13,41 +13,10 @@ func TestCurl(t *testing.T) { homeDir := t.TempDir() httpClient := &mockHttpClient{} convergeServices(httpClient) - out := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "-p", "/usr/bin/curl", "-a", "t1.a1.i1", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) + out, _ := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "-a", "t1.a1.i1", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) - expected := fmt.Sprintf("/usr/bin/curl -v --data-urlencode 'arg=with space' --key %s --cert %s https://127.0.0.1:8080/search\n", + expected := fmt.Sprintf("curl --key %s --cert %s -v --data-urlencode 'arg=with space' https://127.0.0.1:8080/search\n", filepath.Join(homeDir, ".vespa", "t1.a1.i1", "data-plane-private-key.pem"), filepath.Join(homeDir, ".vespa", "t1.a1.i1", "data-plane-public-cert.pem")) assert.Equal(t, expected, out) } - -func TestCurlCommand(t *testing.T) { - c := &curl{path: "/usr/bin/curl", privateKeyPath: "/tmp/priv-key", certificatePath: "/tmp/cert-key"} - assertCurl(t, c, "/usr/bin/curl -v --key /tmp/priv-key --cert /tmp/cert-key https://example.com/", "-v", "/") - - c = &curl{path: "/usr/bin/curl", privateKeyPath: "/tmp/priv-key", certificatePath: "/tmp/cert-key"} - assertCurl(t, c, "/usr/bin/curl -v --cert my-cert --key my-key https://example.com/", "-v", "--cert", "my-cert", "--key", "my-key", "/") - - c = &curl{path: "/usr/bin/curl2"} - assertCurl(t, c, "/usr/bin/curl2 -v https://example.com/foo", "-v", "/foo") - - c = &curl{path: "/usr/bin/curl"} - assertCurl(t, c, "/usr/bin/curl -v https://example.com/foo/bar", "-v", "/foo/bar") - - c = &curl{path: "/usr/bin/curl"} - assertCurl(t, c, "/usr/bin/curl -v https://example.com/foo/bar", "-v", "foo/bar") - - c = &curl{path: "/usr/bin/curl"} - assertCurlURL(t, c, "/usr/bin/curl -v https://example.com/foo/bar", "https://example.com/", "-v", "foo/bar") -} - -func assertCurl(t *testing.T, c *curl, expectedOutput string, args ...string) { - assertCurlURL(t, c, expectedOutput, "https://example.com", args...) -} - -func assertCurlURL(t *testing.T, c *curl, expectedOutput string, url string, args ...string) { - cmd, err := c.command("https://example.com", args...) - assert.Nil(t, err) - - assert.Equal(t, expectedOutput, strings.Join(cmd.Args, " ")) -} diff --git a/client/go/cmd/deploy_test.go b/client/go/cmd/deploy_test.go index f24ba0829f9..443f7e8846f 100644 --- a/client/go/cmd/deploy_test.go +++ b/client/go/cmd/deploy_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // deploy command tests // Author: bratseth @@ -130,9 +130,10 @@ func assertActivate(applicationPackage string, arguments []string, t *testing.T) if err := cfg.WriteSessionID(vespa.DefaultApplication, 42); err != nil { t.Fatal(err) } + out, _ := execute(command{args: arguments, homeDir: homeDir}, t, client) assert.Equal(t, "Success: Activated "+applicationPackage+" with session 42\n", - execute(command{args: arguments, homeDir: homeDir}, t, client)) + out) url := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/active" assert.Equal(t, url, client.lastRequest.URL.String()) assert.Equal(t, "PUT", client.lastRequest.Method) diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index 78c6596f511..cc5fb948e3b 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -1,10 +1,12 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa document command // author: bratseth package cmd import ( + "io" + "io/ioutil" "log" "strings" @@ -13,12 +15,15 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) +var printCurl bool + func init() { rootCmd.AddCommand(documentCmd) documentCmd.AddCommand(documentPutCmd) documentCmd.AddCommand(documentUpdateCmd) documentCmd.AddCommand(documentRemoveCmd) documentCmd.AddCommand(documentGetCmd) + documentCmd.PersistentFlags().BoolVarP(&printCurl, "verbose", "v", false, "Print the equivalent curl command for the document operation") } var documentCmd = &cobra.Command{ @@ -38,7 +43,7 @@ should be used instead of this.`, DisableAutoGenTag: true, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - printResult(vespa.Send(args[0], documentService()), false) + printResult(vespa.Send(args[0], documentService(), curlOutput()), false) }, } @@ -54,9 +59,9 @@ $ vespa document put id:mynamespace:music::a-head-full-of-dreams src/test/resour DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { if len(args) == 1 { - printResult(vespa.Put("", args[0], documentService()), false) + printResult(vespa.Put("", args[0], documentService(), curlOutput()), false) } else { - printResult(vespa.Put(args[0], args[1], documentService()), false) + printResult(vespa.Put(args[0], args[1], documentService(), curlOutput()), false) } }, } @@ -72,9 +77,9 @@ $ vespa document update id:mynamespace:music::a-head-full-of-dreams src/test/res DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { if len(args) == 1 { - printResult(vespa.Update("", args[0], documentService()), false) + printResult(vespa.Update("", args[0], documentService(), curlOutput()), false) } else { - printResult(vespa.Update(args[0], args[1], documentService()), false) + printResult(vespa.Update(args[0], args[1], documentService(), curlOutput()), false) } }, } @@ -90,9 +95,9 @@ $ vespa document remove id:mynamespace:music::a-head-full-of-dreams`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { if strings.HasPrefix(args[0], "id:") { - printResult(vespa.RemoveId(args[0], documentService()), false) + printResult(vespa.RemoveId(args[0], documentService(), curlOutput()), false) } else { - printResult(vespa.RemoveOperation(args[0], documentService()), false) + printResult(vespa.RemoveOperation(args[0], documentService(), curlOutput()), false) } }, } @@ -104,12 +109,19 @@ var documentGetCmd = &cobra.Command{ DisableAutoGenTag: true, Example: `$ vespa document get id:mynamespace:music::a-head-full-of-dreams`, Run: func(cmd *cobra.Command, args []string) { - printResult(vespa.Get(args[0], documentService()), true) + printResult(vespa.Get(args[0], documentService(), curlOutput()), true) }, } func documentService() *vespa.Service { return getService("document", 0) } +func curlOutput() io.Writer { + if printCurl { + return stderr + } + return ioutil.Discard +} + func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) { if !result.Success { log.Print(color.Red("Error: "), result.Message) diff --git a/client/go/cmd/document_test.go b/client/go/cmd/document_test.go index c298d5ef285..8aecb538f89 100644 --- a/client/go/cmd/document_test.go +++ b/client/go/cmd/document_test.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // document command tests // Author: bratseth @@ -19,6 +19,11 @@ func TestDocumentSendPut(t *testing.T) { "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) } +func TestDocumentSendPutVerbose(t *testing.T) { + assertDocumentSend([]string{"document", "-v", "testdata/A-Head-Full-of-Dreams-Put.json"}, + "put", "POST", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Put.json", t) +} + func TestDocumentSendUpdate(t *testing.T) { assertDocumentSend([]string{"document", "testdata/A-Head-Full-of-Dreams-Update.json"}, "update", "PUT", "id:mynamespace:music::a-head-full-of-dreams", "testdata/A-Head-Full-of-Dreams-Update.json", t) @@ -93,11 +98,22 @@ func TestDocumentGet(t *testing.T) { func assertDocumentSend(arguments []string, expectedOperation string, expectedMethod string, expectedDocumentId string, expectedPayloadFile string, t *testing.T) { client := &mockHttpClient{} documentURL := documentServiceURL(client) - assert.Equal(t, - "Success: "+expectedOperation+" "+expectedDocumentId+"\n", - executeCommand(t, client, arguments, []string{})) expectedPath, _ := vespa.IdToURLPath(expectedDocumentId) - assert.Equal(t, documentURL+"/document/v1/"+expectedPath, client.lastRequest.URL.String()) + expectedURL := documentURL + "/document/v1/" + expectedPath + out, errOut := execute(command{args: arguments}, t, client) + + verbose := false + for _, a := range arguments { + if a == "-v" { + verbose = true + } + } + if verbose { + expectedCurl := "curl -X " + expectedMethod + " -H 'Content-Type: application/json' --data-binary @" + expectedPayloadFile + " " + expectedURL + "\n" + assert.Equal(t, expectedCurl, errOut) + } + assert.Equal(t, "Success: "+expectedOperation+" "+expectedDocumentId+"\n", out) + assert.Equal(t, expectedURL, client.lastRequest.URL.String()) assert.Equal(t, "application/json", client.lastRequest.Header.Get("Content-Type")) assert.Equal(t, expectedMethod, client.lastRequest.Method) diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 3493a4b32a8..f29a842aed2 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Helpers used by multiple sub-commands. // Author: mpolden @@ -147,7 +147,16 @@ func getTarget() vespa.Target { if err != nil { fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'") } - return vespa.CloudTarget(deployment, kp, apiKey, vespa.LogOptions{Writer: stdout, Level: vespa.LogLevel(logLevelArg)}) + return vespa.CloudTarget(deployment, apiKey, + vespa.TLSOptions{ + KeyPair: kp, + CertificateFile: certificateFile, + PrivateKeyFile: privateKeyFile, + }, + vespa.LogOptions{ + Writer: stdout, + Level: vespa.LogLevel(logLevelArg), + }) } fatalErrHint(fmt.Errorf("Invalid target: %s", targetType), "Valid targets are 'local', 'cloud' or an URL") return nil diff --git a/client/go/cmd/man_test.go b/client/go/cmd/man_test.go index 59efc64b8de..f7c33c8b3a1 100644 --- a/client/go/cmd/man_test.go +++ b/client/go/cmd/man_test.go @@ -11,7 +11,7 @@ import ( func TestMan(t *testing.T) { tmpDir := t.TempDir() - out := execute(command{args: []string{"man", tmpDir}}, t, nil) + out, _ := execute(command{args: []string{"man", tmpDir}}, t, nil) assert.Equal(t, fmt.Sprintf("Success: Man pages written to %s\n", tmpDir), out) assert.True(t, util.PathExists(filepath.Join(tmpDir, "vespa.1"))) } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 6bfca7fd613..cd8427c3ac6 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // Root Cobra command: vespa // author: bratseth @@ -17,7 +17,6 @@ import ( var ( // TODO: add timeout flag - // TODO: add flag to show http request made rootCmd = &cobra.Command{ Use: "vespa command-name", Short: "The command-line tool for Vespa.ai", @@ -40,6 +39,7 @@ Vespa documentation: https://docs.vespa.ai`, color = aurora.NewAurora(false) stdout = colorable.NewColorableStdout() + stderr = colorable.NewColorableStderr() ) const ( diff --git a/client/go/cmd/version_test.go b/client/go/cmd/version_test.go index 3b0f73de408..9eeaaaa4692 100644 --- a/client/go/cmd/version_test.go +++ b/client/go/cmd/version_test.go @@ -14,7 +14,7 @@ func TestVersion(t *testing.T) { util.ActiveHttpClient = c sp = &mockSubprocess{} - out := execute(command{args: []string{"version"}}, t, nil) + out, _ := execute(command{args: []string{"version"}}, t, nil) assert.Contains(t, out, "vespa version 0.0.0-devel compiled with") assert.Contains(t, out, "New release available: 1.2.3\nhttps://github.com/vespa-engine/vespa/releases/tag/v1.2.3") } @@ -25,7 +25,7 @@ func TestVersionCheckHomebrew(t *testing.T) { util.ActiveHttpClient = c sp = &mockSubprocess{programPath: "/usr/local/bin/vespa", output: "/usr/local"} - out := execute(command{args: []string{"version"}}, t, nil) + out, _ := execute(command{args: []string{"version"}}, t, nil) assert.Contains(t, out, "vespa version 0.0.0-devel compiled with") assert.Contains(t, out, "New release available: 1.2.3\n"+ "https://github.com/vespa-engine/vespa/releases/tag/v1.2.3\n"+ diff --git a/client/go/curl/curl.go b/client/go/curl/curl.go new file mode 100644 index 00000000000..44c3a0ad2a9 --- /dev/null +++ b/client/go/curl/curl.go @@ -0,0 +1,104 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package curl + +import ( + "io" + "net/url" + "os/exec" + "runtime" + + "github.com/kballard/go-shellquote" +) + +type header struct { + key string + value string +} + +type Command struct { + Path string + Method string + PrivateKey string + Certificate string + BodyFile string + url *url.URL + headers []header + rawArgs []string +} + +func (c *Command) Args() []string { + var args []string + if c.PrivateKey != "" { + args = append(args, "--key", c.PrivateKey) + } + if c.Certificate != "" { + args = append(args, "--cert", c.Certificate) + } + if c.Method != "" { + args = append(args, "-X", c.Method) + } + for _, header := range c.headers { + args = append(args, "-H", header.key+": "+header.value) + } + if c.BodyFile != "" { + args = append(args, "--data-binary", "@"+c.BodyFile) + } + args = append(args, c.rawArgs...) + args = append(args, c.url.String()) + return args +} + +func (c *Command) String() string { + args := []string{c.Path} + args = append(args, c.Args()...) + return shellquote.Join(args...) +} + +func (c *Command) Header(key, value string) { + c.headers = append(c.headers, header{key: key, value: value}) +} + +func (c *Command) Param(key, value string) { + query := c.url.Query() + query.Set(key, value) + c.url.RawQuery = query.Encode() +} + +func (c *Command) Run(stdout, stderr io.Writer) error { + cmd := exec.Command(c.Path, c.Args()...) + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() +} + +func Post(url string) (*Command, error) { return curl("POST", url) } + +func Get(url string) (*Command, error) { return curl("", url) } + +func RawArgs(url string, args ...string) (*Command, error) { + c, err := curl("", url) + if err != nil { + return nil, err + } + c.rawArgs = args + return c, nil +} + +func curl(method, rawurl string) (*Command, error) { + path := "curl" + if runtime.GOOS == "windows" { + path = "curl.exe" + } + realURL, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + return &Command{ + Path: path, + Method: method, + url: realURL, + }, nil +} diff --git a/client/go/curl/curl_test.go b/client/go/curl/curl_test.go new file mode 100644 index 00000000000..90bf274f7a2 --- /dev/null +++ b/client/go/curl/curl_test.go @@ -0,0 +1,45 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package curl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPost(t *testing.T) { + c, err := Post("https://example.com") + if err != nil { + t.Fatal(err) + } + c.PrivateKey = "key.pem" + c.Certificate = "cert.pem" + c.BodyFile = "file.json" + c.Header("Content-Type", "application/json") + + assert.Equal(t, "curl --key key.pem --cert cert.pem -X POST -H 'Content-Type: application/json' --data-binary @file.json https://example.com", c.String()) +} + +func TestGet(t *testing.T) { + c, err := Get("https://example.com") + if err != nil { + t.Fatal(err) + } + c.PrivateKey = "key.pem" + c.Certificate = "cert.pem" + c.Param("yql", "select * from sources * where title contains 'foo';") + c.Param("hits", "5") + + assert.Equal(t, `curl --key key.pem --cert cert.pem https://example.com\?hits=5\&yql=select+%2A+from+sources+%2A+where+title+contains+%27foo%27%3B`, c.String()) +} + +func TestRawArgs(t *testing.T) { + c, err := RawArgs("https://example.com/search", "-v", "-m", "10", "-H", "foo: bar") + if err != nil { + t.Fatal(err) + } + c.PrivateKey = "key.pem" + c.Certificate = "cert.pem" + + assert.Equal(t, `curl --key key.pem --cert cert.pem -v -m 10 -H 'foo: bar' https://example.com/search`, c.String()) +} diff --git a/client/go/vespa/document.go b/client/go/vespa/document.go index 7b750b86728..cfac1930199 100644 --- a/client/go/vespa/document.go +++ b/client/go/vespa/document.go @@ -1,4 +1,4 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa document API client // Author: bratseth @@ -7,34 +7,36 @@ package vespa import ( "bytes" "encoding/json" + "io" "io/ioutil" "net/http" "net/url" "os" "time" + "github.com/vespa-engine/vespa/client/go/curl" "github.com/vespa-engine/vespa/client/go/util" ) // Sends the operation given in the file -func Send(jsonFile string, service *Service) util.OperationResult { - return sendOperation("", jsonFile, service, anyOperation) +func Send(jsonFile string, service *Service, curlOutput io.Writer) util.OperationResult { + return sendOperation("", jsonFile, service, anyOperation, curlOutput) } -func Put(documentId string, jsonFile string, service *Service) util.OperationResult { - return sendOperation(documentId, jsonFile, service, putOperation) +func Put(documentId string, jsonFile string, service *Service, curlOutput io.Writer) util.OperationResult { + return sendOperation(documentId, jsonFile, service, putOperation, curlOutput) } -func Update(documentId string, jsonFile string, service *Service) util.OperationResult { - return sendOperation(documentId, jsonFile, service, updateOperation) +func Update(documentId string, jsonFile string, service *Service, curlOutput io.Writer) util.OperationResult { + return sendOperation(documentId, jsonFile, service, updateOperation, curlOutput) } -func RemoveId(documentId string, service *Service) util.OperationResult { - return sendOperation(documentId, "", service, removeOperation) +func RemoveId(documentId string, service *Service, curlOutput io.Writer) util.OperationResult { + return sendOperation(documentId, "", service, removeOperation, curlOutput) } -func RemoveOperation(jsonFile string, service *Service) util.OperationResult { - return sendOperation("", jsonFile, service, removeOperation) +func RemoveOperation(jsonFile string, service *Service, curlOutput io.Writer) util.OperationResult { + return sendOperation("", jsonFile, service, removeOperation, curlOutput) } const ( @@ -44,7 +46,7 @@ const ( removeOperation string = "remove" ) -func sendOperation(documentId string, jsonFile string, service *Service, operation string) util.OperationResult { +func sendOperation(documentId string, jsonFile string, service *Service, operation string, curlOutput io.Writer) util.OperationResult { header := http.Header{} header.Add("Content-Type", "application/json") @@ -93,7 +95,7 @@ func sendOperation(documentId string, jsonFile string, service *Service, operati Header: header, Body: ioutil.NopCloser(bytes.NewReader(documentData)), } - response, err := service.Do(request, time.Second*60) + response, err := serviceDo(service, request, jsonFile, curlOutput) if response == nil { return util.Failure("Request failed: " + err.Error()) } @@ -132,7 +134,28 @@ func operationToHTTPMethod(operation string) string { panic("Unexpected document operation ''" + operation + "'") } -func Get(documentId string, service *Service) util.OperationResult { +func serviceDo(service *Service, request *http.Request, filename string, curlOutput io.Writer) (*http.Response, error) { + cmd, err := curl.RawArgs(request.URL.String()) + if err != nil { + return nil, err + } + cmd.Method = request.Method + for k, vs := range request.Header { + for _, v := range vs { + cmd.Header(k, v) + } + } + cmd.BodyFile = filename + cmd.Certificate = service.TLSOptions.CertificateFile + cmd.PrivateKey = service.TLSOptions.PrivateKeyFile + out := cmd.String() + "\n" + if _, err := io.WriteString(curlOutput, out); err != nil { + return nil, err + } + return service.Do(request, time.Second*60) +} + +func Get(documentId string, service *Service, curlOutput io.Writer) util.OperationResult { documentPath, documentPathError := IdToURLPath(documentId) if documentPathError != nil { return util.Failure("Invalid document id '" + documentId + "': " + documentPathError.Error()) @@ -147,7 +170,7 @@ func Get(documentId string, service *Service) util.OperationResult { URL: url, Method: "GET", } - response, err := service.Do(request, time.Second*60) + response, err := serviceDo(service, request, "", curlOutput) if response == nil { return util.Failure("Request failed: " + err.Error()) } diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 065471d2a1e..69dc876c1c8 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -31,9 +31,9 @@ const ( // Service represents a Vespa service. type Service struct { - BaseURL string - Name string - certificate tls.Certificate + BaseURL string + Name string + TLSOptions TLSOptions } // Target represents a Vespa platform, running named Vespa services. @@ -48,6 +48,13 @@ type Target interface { DiscoverServices(timeout time.Duration, runID int64) error } +// TLSOptions configures the certificate to use for service requests. +type TLSOptions struct { + KeyPair tls.Certificate + CertificateFile string + PrivateKeyFile string +} + // LogOptions configures the log output to produce when waiting for services. type LogOptions struct { Writer io.Writer @@ -61,8 +68,8 @@ type customTarget struct { // Do sends request to this service. Any required authentication happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { - if s.certificate.Certificate != nil { - util.ActiveHttpClient.UseCertificate(s.certificate) + if s.TLSOptions.KeyPair.Certificate != nil { + util.ActiveHttpClient.UseCertificate(s.TLSOptions.KeyPair) } return util.HttpDo(request, timeout, s.Description()) } @@ -83,7 +90,7 @@ func (s *Service) Wait(timeout time.Duration) (int, error) { return 0, err } okFunc := func(status int, response []byte) (bool, error) { return status/100 == 2, nil } - return wait(okFunc, func() *http.Request { return req }, &s.certificate, timeout) + return wait(okFunc, func() *http.Request { return req }, &s.TLSOptions.KeyPair, timeout) } func (s *Service) Description() string { @@ -167,8 +174,8 @@ type cloudTarget struct { cloudAPI string targetType string deployment Deployment - keyPair tls.Certificate apiKey []byte + tlsOptions TLSOptions logOptions LogOptions queryURL string @@ -185,12 +192,12 @@ func (t *cloudTarget) Service(name string) (*Service, error) { if t.queryURL == "" { return nil, fmt.Errorf("service %s not discovered", name) } - return &Service{Name: name, BaseURL: t.queryURL, certificate: t.keyPair}, nil + return &Service{Name: name, BaseURL: t.queryURL, TLSOptions: t.tlsOptions}, nil case documentService: if t.documentURL == "" { return nil, fmt.Errorf("service %s not discovered", name) } - return &Service{Name: name, BaseURL: t.documentURL, certificate: t.keyPair}, nil + return &Service{Name: name, BaseURL: t.documentURL, TLSOptions: t.tlsOptions}, nil } return nil, fmt.Errorf("unknown service: %s", name) } @@ -245,7 +252,7 @@ func (t *cloudTarget) waitForRun(signer *RequestSigner, runID int64, timeout tim } return true, nil } - _, err = wait(jobSuccessFunc, requestFunc, &t.keyPair, timeout) + _, err = wait(jobSuccessFunc, requestFunc, &t.tlsOptions.KeyPair, timeout) return err } @@ -298,7 +305,7 @@ func (t *cloudTarget) discoverEndpoints(signer *RequestSigner, timeout time.Dura endpointURL = resp.Endpoints[0].URL return true, nil } - if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.keyPair, timeout); err != nil { + if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.tlsOptions.KeyPair, timeout); err != nil { return err } if endpointURL == "" { @@ -320,13 +327,13 @@ func CustomTarget(baseURL string) Target { } // CloudTarget creates a Target for the Vespa Cloud platform. -func CloudTarget(deployment Deployment, keyPair tls.Certificate, apiKey []byte, logOptions LogOptions) Target { +func CloudTarget(deployment Deployment, apiKey []byte, tlsOptions TLSOptions, logOptions LogOptions) Target { return &cloudTarget{ cloudAPI: defaultCloudAPI, targetType: cloudTargetType, deployment: deployment, - keyPair: keyPair, apiKey: apiKey, + tlsOptions: tlsOptions, logOptions: logOptions, } } diff --git a/client/go/vespa/target_test.go b/client/go/vespa/target_test.go index b524f73c5d3..31f145f0db3 100644 --- a/client/go/vespa/target_test.go +++ b/client/go/vespa/target_test.go @@ -106,8 +106,8 @@ func TestCloudTargetWait(t *testing.T) { Application: ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"}, Zone: ZoneID{Environment: "dev", Region: "us-north-1"}, }, - x509KeyPair, apiKey, + TLSOptions{KeyPair: x509KeyPair}, LogOptions{Writer: &logWriter}) if ct, ok := target.(*cloudTarget); ok { ct.cloudAPI = srv.URL diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 49e3bff50b7..9ee36831d6a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; import java.io.File; @@ -70,7 +69,6 @@ public interface ModelContext { * - Remove all flag data files from hosted-feature-flag repository */ interface FeatureFlags { - @ModelFeatureFlag(owners = {"jonmv"}, removeAfter = "7.457") default Optional<NodeResources> dedicatedClusterControllerFlavor() { return Optional.of(new NodeResources(0.25, 1, 10, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any)); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -93,7 +91,6 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @ModelFeatureFlag(owners = {"vekterli"}) default int maxActivationInhibitedOutOfSyncGroups() { return 0; } @ModelFeatureFlag(owners = {"hmusum"}) default String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return ""; } - @ModelFeatureFlag(owners = {"tokle", "bjorncs"}, removeAfter = "7.450") default boolean enableCustomAclMapping() { return true; } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int numDistributorStripes() { return 0; } @ModelFeatureFlag(owners = {"arnej"}) default boolean requireConnectivityCheck() { return true; } @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.470") default boolean throwIfResourceLimitsSpecified() { return true; } diff --git a/configgen/CMakeLists.txt b/configgen/CMakeLists.txt deleted file mode 100644 index 107037f8008..00000000000 --- a/configgen/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact(configgen) diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 5debf6c9c02..9b6ccd93a41 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -167,7 +167,6 @@ <buildLegacyVespaPlatformBundle>true</buildLegacyVespaPlatformBundle> <discPreInstallBundle> <!-- Vespa bundles --> - configgen.jar, config-bundle-jar-with-dependencies.jar, configdefinitions-jar-with-dependencies.jar, container-search-and-docproc-jar-with-dependencies.jar, @@ -178,7 +177,6 @@ vespaclient-container-plugin-jar-with-dependencies.jar, vespa-athenz-jar-with-dependencies.jar, security-utils-jar-with-dependencies.jar, - defaults-jar-with-dependencies.jar, zkfacade-jar-with-dependencies.jar, zookeeper-server-jar-with-dependencies.jar, <!-- Apache http client repackaged as bundle --> diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 9fe3728dc2c..853224a5b91 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -40,8 +40,10 @@ import com.yahoo.log.LogSetup; import com.yahoo.messagebus.network.rpc.SlobrokConfigSubscriber; import com.yahoo.net.HostName; import com.yahoo.vespa.config.ConfigKey; +import com.yahoo.vespa.defaults.Defaults; import com.yahoo.yolean.Exceptions; +import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.IdentityHashMap; @@ -400,9 +402,17 @@ public final class ConfiguredApplication implements Application { shutdownDeadlineExecutor = new ScheduledThreadPoolExecutor(1, new DaemonThreadFactory("Shutdown deadline timer")); shutdownDeadlineExecutor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); long delayMillis = 50 * 1000; - shutdownDeadlineExecutor.schedule(() -> com.yahoo.protect.Process.logAndDie( - "Timed out waiting for application shutdown. Please check that all your request handlers " + - "drain their request content channels.", true), delayMillis, TimeUnit.MILLISECONDS); + shutdownDeadlineExecutor.schedule(() -> { + String heapDumpName = Defaults.getDefaults().underVespaHome("var/crash/java_pid.") + ProcessHandle.current().pid() + ".hprof"; + try { + com.yahoo.protect.Process.dumpHeap(heapDumpName, true); + } catch (IOException e) { + log.log(Level.WARNING, "Failed writing heap dump:", e); + } + com.yahoo.protect.Process.logAndDie( + "Timed out waiting for application shutdown. Please check that all your request handlers " + + "drain their request content channels.", true); + }, delayMillis, TimeUnit.MILLISECONDS); } private static void addHandlerBindings(ContainerBuilder builder, diff --git a/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java b/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java index 46dfb780e2d..32ab9d682e3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ProgramCompileException.java @@ -1,7 +1,9 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; -class ProgramCompileException extends RuntimeException { +import com.yahoo.processing.IllegalInputException; + +class ProgramCompileException extends IllegalInputException { private Location sourceLocation; @@ -9,27 +11,6 @@ class ProgramCompileException extends RuntimeException { super(message); } - public ProgramCompileException(String message, Object... args) { - super(formatMessage(message, args)); - } - - private static String formatMessage(String message, Object... args) { - return args == null ? message : String.format(message, args); - } - - public ProgramCompileException(String message, Throwable cause) { - super(message, cause); - } - - public ProgramCompileException(Throwable cause) { - super(cause); - } - - public ProgramCompileException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } - - public ProgramCompileException(Location sourceLocation, String message, Object... args) { super(String.format("%s %s", sourceLocation != null ? sourceLocation : "", args == null ? message : String.format(message, args))); this.sourceLocation = sourceLocation; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 2dbf910b49e..6dd5e7f53e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -180,8 +180,11 @@ public class DeploymentTrigger { public List<JobId> forceTrigger(ApplicationId applicationId, JobType jobType, String user, boolean requireTests) { Application application = applications().requireApplication(TenantAndApplicationId.from(applicationId)); Instance instance = application.require(applicationId.instance()); - DeploymentStatus status = jobs.deploymentStatus(application); JobId job = new JobId(instance.id(), jobType); + if (job.type().environment().isManuallyDeployed()) + return forceTriggerManualJob(job); + + DeploymentStatus status = jobs.deploymentStatus(application); Versions versions = Versions.from(instance.change(), application, status.deploymentFor(job), controller.readSystemVersion()); Map<JobId, List<Versions>> jobs = status.testJobs(Map.of(job, versions)); if (jobs.isEmpty() || ! requireTests) @@ -192,6 +195,16 @@ public class DeploymentTrigger { return List.copyOf(jobs.keySet()); } + private List<JobId> forceTriggerManualJob(JobId job) { + Run last = jobs.last(job).orElseThrow(() -> new IllegalArgumentException(job + " has never been run")); + Versions target = new Versions(controller.readSystemVersion(), + last.versions().targetApplication(), + Optional.of(last.versions().targetPlatform()), + Optional.of(last.versions().targetApplication())); + jobs.start(job.application(), job.type(), target, true); + return List.of(job); + } + /** Retrigger job. If the job is already running, it will be canceled, and retrigger enqueued. */ public Optional<JobId> reTriggerOrAddToQueue(DeploymentId deployment) { JobType jobType = JobType.from(controller.system(), deployment.zoneId()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index cd7ce8c3fa6..98fd0342ecd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -24,8 +24,10 @@ import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Optional; import java.util.OptionalInt; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -60,6 +62,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deployRefreshedCertificates(); updateRefreshedCertificates(); deleteUnusedCertificates(); + reportUnmanagedCertificates(); } catch (Exception e) { log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); return 0.0; @@ -134,6 +137,16 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { }); } + private void reportUnmanagedCertificates() { + Set<String> managedRequestIds = curator.readAllEndpointCertificateMetadata().values().stream().map(EndpointCertificateMetadata::requestId).collect(Collectors.toSet()); + + for (EndpointCertificateMetadata cameoCertificateMetadata : endpointCertificateProvider.listCertificates()) { + if (!managedRequestIds.contains(cameoCertificateMetadata.requestId())) { + log.info("Certificate metadata exists with provider but is not managed by controller: " + cameoCertificateMetadata.requestId() + ", " + cameoCertificateMetadata.issuer() + ", " + cameoCertificateMetadata.requestedDnsSans()); + } + } + } + private Lock lock(ApplicationId applicationId) { return curator.lock(TenantAndApplicationId.from(applicationId)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 66c3f7bba16..fb860063696 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -271,6 +271,12 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}"); app1.runJob(JobType.devUsEast1); + // POST (deploy) a job to restart a manual deployment to dev + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1", POST) + .userIdentity(USER_ID), + "{\"message\":\"Triggered dev-us-east-1 for tenant1.application1.instance1\"}"); + app1.runJob(JobType.devUsEast1); + // GET dev application package tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/dev-us-east-1/package", GET) .userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 7ebc2d24fe9..736e1fe082c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -4,6 +4,37 @@ "jobName": "dev-us-east-1", "runs": [ { + "id": 2, + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/2", + "start": "(ignore)", + "end": "(ignore)", + "status": "success", + "versions": { + "targetPlatform": "6.1.0", + "targetApplication": { + "build": 1 + }, + "sourcePlatform":"6.1.0", + "sourceApplication": { + "build": 1 + } + }, + "steps": [ + { + "name": "deployReal", + "status": "succeeded" + }, + { + "name": "installReal", + "status": "succeeded" + }, + { + "name": "copyVespaLogs", + "status": "succeeded" + } + ] + }, + { "id": 1, "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/run/1", "start": "(ignore)", diff --git a/defaults/CMakeLists.txt b/defaults/CMakeLists.txt index c42e5402688..ed0ab1e6fb0 100644 --- a/defaults/CMakeLists.txt +++ b/defaults/CMakeLists.txt @@ -7,4 +7,4 @@ vespa_define_module( src/apps/printdefault ) -install_fat_java_artifact(defaults) +# No separate java artifact is installed (part of config-bundle) diff --git a/dist/release-vespa-rpm.sh b/dist/release-vespa-rpm.sh index b217affe8fd..c975e10dd1a 100755 --- a/dist/release-vespa-rpm.sh +++ b/dist/release-vespa-rpm.sh @@ -1,62 +1,38 @@ #!/bin/bash -# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + set -e -if [ $# -ne 2 ]; then +if [[ $# -ne 2 ]]; then echo "Usage: $0 <version> <git ref>" exit 1 fi +if [[ -z $COPR_WEBHOOK ]]; then + echo "This script requires the COPR_WEBHOOK environment variable to be set." + exit 1 +fi + readonly VERSION=$1 readonly GITREF=$2 -readonly DIST_DIR="dist" -readonly SPECFILE="${DIST_DIR}/vespa.spec" -readonly TITO_DIR="${DIST_DIR}/.tito" -readonly RPM_BRANCH="rpmbuild" +readonly RELEASE_TAG="v$VERSION" readonly CURRENT_BRANCH=$(git branch | grep "^\*" | cut -d' ' -f2) # Make sure we are up to date git checkout master git pull --rebase -# Update the VERSION file on master to be the next releasable version -echo "$VERSION" | awk -F. '{print $1"."($2+1)".0"}' > VERSION -git commit -am "Updating VERSION file to next releasable minor version." -for i in 1 2 3; do - if git push; then - break; - fi - git pull --rebase -done - # Create a proper release tag -git tag -a "v$VERSION" -m "Release $VERSION" $GITREF -git push origin "v$VERSION" - -# Delete existing branch if exists and create new one -git push --delete origin $RPM_BRANCH &> /dev/null || true -git branch -D $RPM_BRANCH &> /dev/null || true -git checkout -b $RPM_BRANCH $GITREF - -# Tito expects spec file and .tito directory to be on root -git mv $TITO_DIR . -git mv $SPECFILE . - -# Hide pom.xml to avoid tito doing anything to our pom.xml files -mv pom.xml pom.xml.hide - -# Run tito to update spec file and tag -tito tag --use-version=$VERSION --no-auto-changelog -# Push changes and tag to branc -git push -u origin --follow-tags $RPM_BRANCH +git tag -a "$RELEASE_TAG" -m "Release version $VERSION" $GITREF +git push origin "$RELEASE_TAG" # Trig the build on Copr curl -X POST \ -H "Content-type: application/json" \ -H "X-GitHub-Event: create" \ - -d '{ "ref": "rpmbuild", "ref_type": "branch", "repository": { "clone_url": "https://github.com/vespa-engine/vespa.git" } }' \ - https://copr.fedorainfracloud.org/webhooks/github/8037/d1dd5867-b493-4647-a888-0c887e6087b3/ + -d '{ "ref": "$RELEASE_TAG", "ref_type": "tag", "repository": { "clone_url": "https://github.com/vespa-engine/vespa.git" } }' \ + "$COPR_WEBHOOK" git reset --hard HEAD git checkout $CURRENT_BRANCH diff --git a/dist/vespa.spec b/dist/vespa.spec index aa3d64401d3..e976d710fb5 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -791,7 +791,6 @@ fi %{_prefix}/lib/jars/bcprov-jdk15on-*.jar %{_prefix}/lib/jars/config-bundle-jar-with-dependencies.jar %{_prefix}/lib/jars/configdefinitions-jar-with-dependencies.jar -%{_prefix}/lib/jars/configgen.jar %{_prefix}/lib/jars/config-model-api-jar-with-dependencies.jar %{_prefix}/lib/jars/config-model-jar-with-dependencies.jar %{_prefix}/lib/jars/config-provisioning-jar-with-dependencies.jar @@ -799,7 +798,6 @@ fi %{_prefix}/lib/jars/container-disc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-and-docproc-jar-with-dependencies.jar %{_prefix}/lib/jars/container-search-gui-jar-with-dependencies.jar -%{_prefix}/lib/jars/defaults-jar-with-dependencies.jar %{_prefix}/lib/jars/docprocs-jar-with-dependencies.jar %{_prefix}/lib/jars/flags-jar-with-dependencies.jar %{_prefix}/lib/jars/hk2-*.jar diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 8a596ad38d4..4d757f6ebbc 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -14,6 +14,19 @@ using vespalib::make_string_short::fmt; namespace { +struct IgnoreJava : TestBuilder { + TestBuilder &dst; + IgnoreJava(TestBuilder &dst_in) : TestBuilder(dst_in.full), dst(dst_in) {} + void add(const vespalib::string &expression, + const std::map<vespalib::string,TensorSpec> &inputs, + const std::set<vespalib::string> &ignore) override + { + auto my_ignore = ignore; + my_ignore.insert("vespajlib"); + dst.add(expression, inputs, my_ignore); + } +}; + //----------------------------------------------------------------------------- const std::vector<vespalib::string> basic_layouts = { @@ -273,6 +286,8 @@ void generate_join(TestBuilder &dst) { generate_op2_join("min(a,b)", Div16(N()), dst); generate_op2_join("max(a,b)", Div16(N()), dst); generate_op2_join("bit(a,b)", Seq({-128, -43, -1, 0, 85, 127}), Seq({0, 1, 2, 3, 4, 5, 6, 7}), dst); + IgnoreJava ignore_java(dst); + generate_op2_join("hamming(a,b)", Seq({-128, -43, -1, 0, 85, 127}), ignore_java); // TODO: require java // inverted lambda generate_join_expr("join(a,b,f(a,b)(b-a))", Div16(N()), dst); // custom lambda @@ -331,6 +346,8 @@ void generate_merge(TestBuilder &dst) { generate_op2_merge("min(a,b)", Div16(N()), dst); generate_op2_merge("max(a,b)", Div16(N()), dst); generate_op2_merge("bit(a,b)", Seq({-128, -43, -1, 0, 85, 127}), Seq({0, 1, 2, 3, 4, 5, 6, 7}), dst); + IgnoreJava ignore_java(dst); + generate_op2_merge("hamming(a,b)", Seq({-128, -43, -1, 0, 85, 127}), ignore_java); // TODO: require java // inverted lambda generate_merge_expr("merge(a,b,f(a,b)(b-a))", Div16(N()), dst); // custom lambda diff --git a/eval/src/apps/tensor_conformance/generate.h b/eval/src/apps/tensor_conformance/generate.h index e9482b9015c..9aa90ae9a7a 100644 --- a/eval/src/apps/tensor_conformance/generate.h +++ b/eval/src/apps/tensor_conformance/generate.h @@ -18,11 +18,6 @@ struct TestBuilder { { add(expression, inputs, {}); } - void add_ignore_java(const vespalib::string &expression, - const std::map<vespalib::string,TensorSpec> &inputs) - { - add(expression, inputs, {"vespajlib"}); - } virtual ~TestBuilder() {} }; diff --git a/eval/src/apps/tensor_conformance/tensor_conformance.cpp b/eval/src/apps/tensor_conformance/tensor_conformance.cpp index e6bbb1f8a41..6c28b1e652e 100644 --- a/eval/src/apps/tensor_conformance/tensor_conformance.cpp +++ b/eval/src/apps/tensor_conformance/tensor_conformance.cpp @@ -167,6 +167,15 @@ void print_test(const Inspector &test, OutputWriter &dst) { } auto result = eval_expr(test, prod_factory); dst.printf("result: %s\n", result.to_string().c_str()); + auto ignore = extract_fields(test["ignore"]); + if (!ignore.empty()) { + dst.printf("ignore:"); + for (const auto &impl: ignore) { + REQUIRE(test["ignore"][impl].asBool()); + dst.printf(" %s", impl.c_str()); + } + dst.printf("\n"); + } } //----------------------------------------------------------------------------- diff --git a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp index ae5f503b680..8e765708574 100644 --- a/eval/src/tests/eval/inline_operation/inline_operation_test.cpp +++ b/eval/src/tests/eval/inline_operation/inline_operation_test.cpp @@ -116,6 +116,7 @@ TEST(InlineOperationTest, op2_lambdas_are_recognized) { EXPECT_EQ(as_op2("min(a,b)"), &Min::f); EXPECT_EQ(as_op2("max(a,b)"), &Max::f); EXPECT_EQ(as_op2("bit(a,b)"), &Bit::f); + EXPECT_EQ(as_op2("hamming(a,b)"), &Hamming::f); } TEST(InlineOperationTest, op2_lambdas_are_recognized_with_different_parameter_names) { diff --git a/eval/src/tests/eval/node_tools/node_tools_test.cpp b/eval/src/tests/eval/node_tools/node_tools_test.cpp index e8296c01d73..b95ea2d4b14 100644 --- a/eval/src/tests/eval/node_tools/node_tools_test.cpp +++ b/eval/src/tests/eval/node_tools/node_tools_test.cpp @@ -101,6 +101,7 @@ TEST("require that call node types can be copied") { TEST_DO(verify_copy("elu(a)")); TEST_DO(verify_copy("erf(a)")); TEST_DO(verify_copy("bit(a,b)")); + TEST_DO(verify_copy("hamming(a,b)")); } TEST("require that tensor node types can NOT be copied (yet)") { diff --git a/eval/src/tests/eval/node_types/node_types_test.cpp b/eval/src/tests/eval/node_types/node_types_test.cpp index b2373f0d8f5..5b860f0e1b3 100644 --- a/eval/src/tests/eval/node_types/node_types_test.cpp +++ b/eval/src/tests/eval/node_types/node_types_test.cpp @@ -219,6 +219,7 @@ TEST("require that various operations resolve appropriate type") { TEST_DO(verify_op1("elu(%s)")); // Elu TEST_DO(verify_op1("erf(%s)")); // Erf TEST_DO(verify_op2("bit(%s,%s)")); // Bit + TEST_DO(verify_op2("hamming(%s,%s)")); // Hamming } TEST("require that map resolves correct type") { diff --git a/eval/src/vespa/eval/eval/call_nodes.cpp b/eval/src/vespa/eval/eval/call_nodes.cpp index 798583cf89a..95dbecdd153 100644 --- a/eval/src/vespa/eval/eval/call_nodes.cpp +++ b/eval/src/vespa/eval/eval/call_nodes.cpp @@ -44,6 +44,7 @@ CallRepo::CallRepo() : _map() { add(nodes::Elu()); add(nodes::Erf()); add(nodes::Bit()); + add(nodes::Hamming()); } } // namespace vespalib::eval::nodes diff --git a/eval/src/vespa/eval/eval/call_nodes.h b/eval/src/vespa/eval/eval/call_nodes.h index 945aba69596..47fc5d6eccd 100644 --- a/eval/src/vespa/eval/eval/call_nodes.h +++ b/eval/src/vespa/eval/eval/call_nodes.h @@ -140,6 +140,7 @@ struct Sigmoid : CallHelper<Sigmoid> { Sigmoid() : Helper("sigmoid", 1) {} }; struct Elu : CallHelper<Elu> { Elu() : Helper("elu", 1) {} }; struct Erf : CallHelper<Erf> { Erf() : Helper("erf", 1) {} }; struct Bit : CallHelper<Bit> { Bit() : Helper("bit", 2) {} }; +struct Hamming : CallHelper<Hamming> { Hamming() : Helper("hamming", 2) {} }; //----------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/hamming_distance.h b/eval/src/vespa/eval/eval/hamming_distance.h new file mode 100644 index 00000000000..3419de3569f --- /dev/null +++ b/eval/src/vespa/eval/eval/hamming_distance.h @@ -0,0 +1,13 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +namespace vespalib::eval { + +inline double hamming_distance(double a, double b) { + uint8_t x = (uint8_t) a; + uint8_t y = (uint8_t) b; + return __builtin_popcount(x ^ y); +} + +} diff --git a/eval/src/vespa/eval/eval/key_gen.cpp b/eval/src/vespa/eval/eval/key_gen.cpp index a40a8887119..cbbce61402c 100644 --- a/eval/src/vespa/eval/eval/key_gen.cpp +++ b/eval/src/vespa/eval/eval/key_gen.cpp @@ -88,6 +88,7 @@ struct KeyGen : public NodeVisitor, public NodeTraverser { void visit(const Elu &) override { add_byte(61); } void visit(const Erf &) override { add_byte(62); } void visit(const Bit &) override { add_byte(63); } + void visit(const Hamming &) override { add_byte(64); } // traverse bool open(const Node &node) override { node.accept(*this); return true; } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 3e4f4fe8257..a101745dca0 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -5,6 +5,7 @@ #include <vespa/eval/eval/node_visitor.h> #include <vespa/eval/eval/node_traverser.h> #include <vespa/eval/eval/extract_bit.h> +#include <vespa/eval/eval/hamming_distance.h> #include <llvm/IR/Verifier.h> #include <llvm/Support/TargetSelect.h> #include <llvm/IR/IRBuilder.h> @@ -31,6 +32,7 @@ double vespalib_eval_relu(double a) { return std::max(a, 0.0); } double vespalib_eval_sigmoid(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double vespalib_eval_elu(double a) { return (a < 0) ? std::exp(a) - 1.0 : a; } double vespalib_eval_bit(double a, double b) { return vespalib::eval::extract_bit(a, b); } +double vespalib_eval_hamming(double a, double b) { return vespalib::eval::hamming_distance(a, b); } using vespalib::eval::gbdt::Forest; using resolve_function = double (*)(void *ctx, size_t idx); @@ -651,6 +653,9 @@ struct FunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Bit &) override { make_call_2("vespalib_eval_bit"); } + void visit(const Hamming &) override { + make_call_2("vespalib_eval_hamming"); + } }; FunctionBuilder::~FunctionBuilder() { } diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h index e04b477750d..727954d59e9 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.h @@ -20,6 +20,7 @@ extern "C" { double vespalib_eval_sigmoid(double a); double vespalib_eval_elu(double a); double vespalib_eval_bit(double a, double b); + double vespalib_eval_hamming(double a, double b); }; namespace vespalib::eval { diff --git a/eval/src/vespa/eval/eval/make_tensor_function.cpp b/eval/src/vespa/eval/eval/make_tensor_function.cpp index 498be2a738b..7746676f86b 100644 --- a/eval/src/vespa/eval/eval/make_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/make_tensor_function.cpp @@ -360,6 +360,9 @@ struct TensorFunctionBuilder : public NodeVisitor, public NodeTraverser { void visit(const Bit &node) override { make_join(node, operation::Bit::f); } + void visit(const Hamming &node) override { + make_join(node, operation::Hamming::f); + } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_tools.cpp b/eval/src/vespa/eval/eval/node_tools.cpp index fa2d16a2271..48ee1a90b67 100644 --- a/eval/src/vespa/eval/eval/node_tools.cpp +++ b/eval/src/vespa/eval/eval/node_tools.cpp @@ -183,6 +183,7 @@ struct CopyNode : NodeTraverser, NodeVisitor { void visit(const Elu &node) override { copy_call(node); } void visit(const Erf &node) override { copy_call(node); } void visit(const Bit &node) override { copy_call(node); } + void visit(const Hamming &node) override { copy_call(node); } // traverse nodes bool open(const Node &) override { return !error; } diff --git a/eval/src/vespa/eval/eval/node_types.cpp b/eval/src/vespa/eval/eval/node_types.cpp index 8622fd734f1..2cb6e637201 100644 --- a/eval/src/vespa/eval/eval/node_types.cpp +++ b/eval/src/vespa/eval/eval/node_types.cpp @@ -279,6 +279,7 @@ struct TypeResolver : public NodeVisitor, public NodeTraverser { void visit(const Elu &node) override { resolve_op1(node); } void visit(const Erf &node) override { resolve_op1(node); } void visit(const Bit &node) override { resolve_op2(node); } + void visit(const Hamming &node) override { resolve_op2(node); } //------------------------------------------------------------------------- diff --git a/eval/src/vespa/eval/eval/node_visitor.h b/eval/src/vespa/eval/eval/node_visitor.h index 475bbf5405c..b581a94f7ee 100644 --- a/eval/src/vespa/eval/eval/node_visitor.h +++ b/eval/src/vespa/eval/eval/node_visitor.h @@ -86,6 +86,7 @@ struct NodeVisitor { virtual void visit(const nodes::Elu &) = 0; virtual void visit(const nodes::Erf &) = 0; virtual void visit(const nodes::Bit &) = 0; + virtual void visit(const nodes::Hamming &) = 0; virtual ~NodeVisitor() {} }; @@ -156,6 +157,7 @@ struct EmptyNodeVisitor : NodeVisitor { void visit(const nodes::Elu &) override {} void visit(const nodes::Erf &) override {} void visit(const nodes::Bit &) override {} + void visit(const nodes::Hamming &) override {} }; } // namespace vespalib::eval diff --git a/eval/src/vespa/eval/eval/operation.cpp b/eval/src/vespa/eval/eval/operation.cpp index a82a79e6bc4..ddd188d250f 100644 --- a/eval/src/vespa/eval/eval/operation.cpp +++ b/eval/src/vespa/eval/eval/operation.cpp @@ -4,6 +4,7 @@ #include "function.h" #include "key_gen.h" #include "extract_bit.h" +#include "hamming_distance.h" #include <vespa/vespalib/util/approx.h> #include <algorithm> @@ -52,6 +53,7 @@ double Sigmoid::f(double a) { return 1.0 / (1.0 + std::exp(-1.0 * a)); } double Elu::f(double a) { return (a < 0) ? std::exp(a) - 1 : a; } double Erf::f(double a) { return std::erf(a); } double Bit::f(double a, double b) { return extract_bit(a, b); } +double Hamming::f(double a, double b) { return hamming_distance(a, b); } //----------------------------------------------------------------------------- double Inv::f(double a) { return (1.0 / a); } double Square::f(double a) { return (a * a); } @@ -146,6 +148,7 @@ std::map<vespalib::string,op2_t> make_op2_map() { add_op2(map, "min(a,b)", Min::f); add_op2(map, "max(a,b)", Max::f); add_op2(map, "bit(a,b)", Bit::f); + add_op2(map, "hamming(a,b)", Hamming::f); return map; } diff --git a/eval/src/vespa/eval/eval/operation.h b/eval/src/vespa/eval/eval/operation.h index 438b510b714..e2a524f318c 100644 --- a/eval/src/vespa/eval/eval/operation.h +++ b/eval/src/vespa/eval/eval/operation.h @@ -50,6 +50,7 @@ struct Sigmoid { static double f(double a); }; struct Elu { static double f(double a); }; struct Erf { static double f(double a); }; struct Bit { static double f(double a, double b); }; +struct Hamming { static double f(double a, double b); }; //----------------------------------------------------------------------------- struct Inv { static double f(double a); }; struct Square { static double f(double a); }; diff --git a/eval/src/vespa/eval/eval/test/eval_spec.cpp b/eval/src/vespa/eval/eval/test/eval_spec.cpp index 5d51a1d23b5..03b3af84fc9 100644 --- a/eval/src/vespa/eval/eval/test/eval_spec.cpp +++ b/eval/src/vespa/eval/eval/test/eval_spec.cpp @@ -8,6 +8,24 @@ namespace vespalib::eval::test { +namespace { + +double byte(const vespalib::string &bits) { + int8_t res = 0; + assert(bits.size() == 8); + for (const auto &c: bits) { + if (c == '1') { + res = (res << 1) | 1; + } else { + assert(c == '0'); + res = (res << 1); + } + } + return res; +} + +} // <unnamed> + constexpr double my_nan = std::numeric_limits<double>::quiet_NaN(); constexpr double my_inf = std::numeric_limits<double>::infinity(); @@ -169,6 +187,9 @@ EvalSpec::add_function_call_cases() { .add_case({85, 3}, 0.0).add_case({85, 2}, 1.0).add_case({85, 1}, 0.0).add_case({85, 0}, 1.0) .add_case({127, 7}, 0.0).add_case({127, 6}, 1.0).add_case({127, 5}, 1.0).add_case({127, 4}, 1.0) .add_case({127, 3}, 1.0).add_case({127, 2}, 1.0).add_case({127, 1}, 1.0).add_case({127, 0}, 1.0); + add_expression({"a", "b"}, "hamming(a,b)") + .add_case({0, 0}, 0.0).add_case({-1, -1}, 0.0).add_case({-1, 0}, 8.0).add_case({0, -1}, 8.0) + .add_case({byte("11001100"), byte("10101010")}, 4.0).add_case({byte("11001100"), byte("11110000")}, 4.0); } void diff --git a/eval/src/vespa/eval/eval/test/reference_evaluation.cpp b/eval/src/vespa/eval/eval/test/reference_evaluation.cpp index 58e4b91f6d9..def3f64c1a1 100644 --- a/eval/src/vespa/eval/eval/test/reference_evaluation.cpp +++ b/eval/src/vespa/eval/eval/test/reference_evaluation.cpp @@ -338,6 +338,9 @@ struct EvalNode : public NodeVisitor { void visit(const Bit &node) override { eval_join(node.get_child(0), node.get_child(1), operation::Bit::f); } + void visit(const Hamming &node) override { + eval_join(node.get_child(0), node.get_child(1), operation::Hamming::f); + } }; TensorSpec eval_node(const Node &node, const std::vector<TensorSpec> ¶ms) { diff --git a/eval/src/vespa/eval/eval/visit_stuff.cpp b/eval/src/vespa/eval/eval/visit_stuff.cpp index 786562d823f..1d684e1c340 100644 --- a/eval/src/vespa/eval/eval/visit_stuff.cpp +++ b/eval/src/vespa/eval/eval/visit_stuff.cpp @@ -60,6 +60,7 @@ vespalib::string name_of(join_fun_t fun) { if (fun == operation::Min::f) return "min"; if (fun == operation::Max::f) return "max"; if (fun == operation::Bit::f) return "bit"; + if (fun == operation::Hamming::f) return "hamming"; return "[other join function]"; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2e05bbf5d90..30110493da2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -79,7 +79,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag ENFORCE_RANK_PROFILE_INHERITANCE = defineFeatureFlag( - "enforce-rank-profile-inheritance", false, + "enforce-rank-profile-inheritance", true, List.of("baldersheim"), "2021-09-07", "2021-10-01", "Should we enforce verification of rank-profile inheritance.", "Takes effect at redeployment", @@ -297,14 +297,6 @@ public class Flags { APPLICATION_ID ); - public static final UnboundStringFlag ENDPOINT_CERTIFICATE_ALGORITHM = defineStringFlag( - "endpoint-certificate-algorithm", "", - // Acceptable values are: "rsa_2048", "rsa_4096", "ecdsa_p256" - List.of("andreer"), "2021-09-21", "2022-01-01", - "Selects algorithm used for an applications endpoint certificate, or use provider default if blank", - "Takes effect when a new endpoint certificate is requested (first deployment of new application/instance)", - APPLICATION_ID); - /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 9c06fe5fa7d..cdce5b03fea 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -169,6 +169,13 @@ public class PermanentFlags { APPLICATION_ID ); + public static final UnboundStringFlag ENDPOINT_CERTIFICATE_ALGORITHM = defineStringFlag( + "endpoint-certificate-algorithm", "", + // Acceptable values are: "rsa_2048", "rsa_4096", "ecdsa_p256" + "Selects algorithm used for an applications endpoint certificate, or use provider default if blank", + "Takes effect when a new endpoint certificate is requested (first deployment of new application/instance)", + APPLICATION_ID); + public static final UnboundDoubleFlag RESOURCE_LIMIT_DISK = defineDoubleFlag( "resource-limit-disk", 0.8, "Resource limit (between 0.0 and 1.0) for disk used by cluster controller for when to block feed", @@ -190,6 +197,12 @@ public class PermanentFlags { APPLICATION_ID, HOSTNAME ); + public static final UnboundStringFlag CONFIG_PROXY_JVM_ARGS = defineStringFlag( + "config-proxy-jvm-args", "", + "Sets jvm args for config proxy (added at the end of startup command, will override existing ones)", + "Takes effect on restart of Docker container", + ZONE_ID, APPLICATION_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java index 87e2f6db761..d8d58aee8c2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApi.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver; +import java.net.URI; import java.time.Duration; import java.util.Optional; @@ -19,13 +20,13 @@ public interface ConfigServerApi extends AutoCloseable { * @param <T> the type of the returned jackson response */ interface RetryPolicy<T> { - boolean tryNextConfigServer(T response); + boolean tryNextConfigServer(URI configServerEndpoint, T response); } class Params<T> { private Optional<Duration> connectionTimeout = Optional.empty(); - private RetryPolicy<T> retryPolicy = response -> false; + private RetryPolicy<T> retryPolicy = (configServerEndpoint, response) -> false; public Params() {} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 4a9c530d9c9..c41528c64ec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -125,7 +125,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { throw new UncheckedIOException("Failed parse response from config server", e); } - if (params.getRetryPolicy().tryNextConfigServer(result)) { + if (params.getRetryPolicy().tryNextConfigServer(configServer, result)) { lastResult = result; lastException = null; } else { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java index b8ea119c0be..8b74dd35f96 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/orchestrator/OrchestratorImpl.java @@ -9,9 +9,11 @@ import com.yahoo.vespa.orchestrator.restapi.wire.BatchOperationResult; import com.yahoo.vespa.orchestrator.restapi.wire.HostStateChangeDenialReason; import com.yahoo.vespa.orchestrator.restapi.wire.UpdateHostResponse; +import java.net.URI; import java.time.Duration; import java.util.List; import java.util.Optional; +import java.util.logging.Logger; /** * @author stiankri @@ -19,6 +21,8 @@ import java.util.Optional; * @author dybis */ public class OrchestratorImpl implements Orchestrator { + private static final Logger logger = Logger.getLogger(OrchestratorImpl.class.getName()); + // The server-side Orchestrator has an internal timeout of 10s. // // Note: A 409 has been observed to be returned after 33s in a case possibly involving @@ -68,14 +72,23 @@ public class OrchestratorImpl implements Orchestrator { private static ConfigServerApi.RetryPolicy<UpdateHostResponse> createRetryPolicyForSuspend() { return new ConfigServerApi.RetryPolicy<UpdateHostResponse>() { @Override - public boolean tryNextConfigServer(UpdateHostResponse response) { + public boolean tryNextConfigServer(URI configServerEndpoint, UpdateHostResponse response) { HostStateChangeDenialReason reason = response.reason(); if (reason == null) { return false; } // The config server has likely just bootstrapped, so try the next. - return "unknown-service-status".equals(reason.constraintName()); + if ("unknown-service-status".equals(reason.constraintName())) { + // Warn for now and until this feature has proven to work well + logger.warning("Config server at [" + configServerEndpoint + + "] failed with transient error (will try next): " + + reason.message()); + + return true; + } + + return false; } }; } diff --git a/storage/src/vespa/storage/distributor/bucket_space_state_map.h b/storage/src/vespa/storage/distributor/bucket_space_state_map.h index 6209f9f306c..ccf79e001f7 100644 --- a/storage/src/vespa/storage/distributor/bucket_space_state_map.h +++ b/storage/src/vespa/storage/distributor/bucket_space_state_map.h @@ -16,7 +16,6 @@ namespace storage::distributor { /** * Represents cluster state and distribution for a given bucket space. - * TODO STRIPE: Make DistributorBucketSpace inherit this class. */ class BucketSpaceState { private: diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp index 37e7dc86e43..9ec4d31eb32 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.cpp @@ -22,8 +22,8 @@ DistributorBucketSpace::DistributorBucketSpace() { } -DistributorBucketSpace::DistributorBucketSpace(uint16_t node_index, bool use_bucket_db) - : _bucketDatabase(use_bucket_db ? std::make_unique<BTreeBucketDatabase>() : std::unique_ptr<BTreeBucketDatabase>()), +DistributorBucketSpace::DistributorBucketSpace(uint16_t node_index) + : _bucketDatabase(std::make_unique<BTreeBucketDatabase>()), _clusterState(), _distribution(), _node_index(node_index), diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space.h b/storage/src/vespa/storage/distributor/distributor_bucket_space.h index 8898039eb02..794bb442400 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space.h @@ -47,8 +47,7 @@ class DistributorBucketSpace { bool owns_bucket_in_state(const lib::Distribution& distribution, const lib::ClusterState& cluster_state, document::BucketId bucket) const; public: explicit DistributorBucketSpace(); - // TODO STRIPE: Remove the use_bucket_db parameter when legacy mode is gone. - explicit DistributorBucketSpace(uint16_t node_index, bool use_bucket_db = true); + explicit DistributorBucketSpace(uint16_t node_index); ~DistributorBucketSpace(); DistributorBucketSpace(const DistributorBucketSpace&) = delete; diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index 368483d3f2d..4f64dab9a68 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -13,11 +13,11 @@ using document::BucketSpace; namespace storage::distributor { -DistributorBucketSpaceRepo::DistributorBucketSpaceRepo(uint16_t node_index, bool use_bucket_db) +DistributorBucketSpaceRepo::DistributorBucketSpaceRepo(uint16_t node_index) : _map() { - add(document::FixedBucketSpaces::default_space(), std::make_unique<DistributorBucketSpace>(node_index, use_bucket_db)); - add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>(node_index, use_bucket_db)); + add(document::FixedBucketSpaces::default_space(), std::make_unique<DistributorBucketSpace>(node_index)); + add(document::FixedBucketSpaces::global_space(), std::make_unique<DistributorBucketSpace>(node_index)); } DistributorBucketSpaceRepo::~DistributorBucketSpaceRepo() = default; diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h index e7552f058d8..f012b25e351 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.h @@ -19,8 +19,7 @@ private: BucketSpaceMap _map; public: - // TODO STRIPE: Remove the use_bucket_db parameter when legacy mode is gone. - explicit DistributorBucketSpaceRepo(uint16_t node_index, bool use_bucket_db = true); + explicit DistributorBucketSpaceRepo(uint16_t node_index); ~DistributorBucketSpaceRepo(); DistributorBucketSpaceRepo(const DistributorBucketSpaceRepo&&) = delete; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index 199291ac276..e5d5b8ba5b6 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -658,6 +658,7 @@ "public static void logAndDie(java.lang.String, boolean)", "public static void logAndDie(java.lang.String, java.lang.Throwable)", "public static void logAndDie(java.lang.String, java.lang.Throwable, boolean)", + "public static void dumpHeap(java.lang.String, boolean)", "public static void dumpThreads()" ], "fields": [] diff --git a/vespajlib/src/main/java/com/yahoo/protect/Process.java b/vespajlib/src/main/java/com/yahoo/protect/Process.java index 4d2fafd4665..f3674f665b2 100644 --- a/vespajlib/src/main/java/com/yahoo/protect/Process.java +++ b/vespajlib/src/main/java/com/yahoo/protect/Process.java @@ -1,7 +1,11 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.protect; +import com.sun.management.HotSpotDiagnosticMXBean; +import javax.management.MBeanServer; +import java.io.IOException; +import java.lang.management.ManagementFactory; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; @@ -70,6 +74,16 @@ public final class Process { } } + public static void dumpHeap(String filePath, boolean live) throws IOException { + log.log(Level.INFO, "Will dump the heap to '" + filePath + "', with the live = " + live); + getHotspotMXBean().dumpHeap(filePath, live); + } + + private static HotSpotDiagnosticMXBean getHotspotMXBean() throws IOException { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + return ManagementFactory.newPlatformMXBeanProxy( + server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + } public static void dumpThreads() { boolean alreadyDumpingThreads = busyDumpingThreads.getAndSet(true); |