diff options
257 files changed, 3983 insertions, 1911 deletions
diff --git a/abi-check-plugin/pom.xml b/abi-check-plugin/pom.xml index 059e593c0e9..bfa3af30cb5 100644 --- a/abi-check-plugin/pom.xml +++ b/abi-check-plugin/pom.xml @@ -45,7 +45,6 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-api</artifactId> - <version>5.3.1</version> <scope>test</scope> </dependency> <dependency> diff --git a/client/go/auth0/auth0.go b/client/go/auth0/auth0.go index f94a933d96a..bcde790f830 100644 --- a/client/go/auth0/auth0.go +++ b/client/go/auth0/auth0.go @@ -25,14 +25,24 @@ import ( const accessTokenExpThreshold = 5 * time.Minute -var errUnauthenticated = errors.New("not logged in. Try 'vespa login'") +var errUnauthenticated = errors.New("not logged in. Try 'vespa auth login'") + +type configJsonFormat struct { + Version int `json:"version"` + Providers providers `json:"providers"` +} + +type providers struct { + Config config `json:"auth0"` +} type config struct { + Version int `json:"version"` Systems map[string]System `json:"systems"` } type System struct { - Name string `json:"name"` + Name string `json:"-"` AccessToken string `json:"access_token,omitempty"` Scopes []string `json:"scopes,omitempty"` ExpiresAt time.Time `json:"expires_at"` @@ -217,7 +227,7 @@ func (a *Auth0) getSystem() (System, error) { s, ok := a.config.Systems[a.system] if !ok { - return System{}, fmt.Errorf("unable to find system: %s; run 'vespa login' to configure a new system", a.system) + return System{}, fmt.Errorf("unable to find system: %s; run 'vespa auth login' to configure a new system", a.system) } return s, nil @@ -272,7 +282,7 @@ func (a *Auth0) persistConfig() error { } } - buf, err := json.MarshalIndent(a.config, "", " ") + buf, err := a.configToJson(&a.config) if err != nil { return err } @@ -284,6 +294,32 @@ func (a *Auth0) persistConfig() error { return nil } +func (a *Auth0) configToJson(cfg *config) ([]byte, error) { + cfg.Version = 1 + r := configJsonFormat{ + Version: 1, + Providers: providers{ + Config: *cfg, + }, + } + return json.MarshalIndent(r, "", " ") +} + +func (a *Auth0) jsonToConfig(buf []byte) (*config, error) { + r := configJsonFormat{} + if err := json.Unmarshal(buf, &r); err != nil { + return nil, err + } + cfg := r.Providers.Config + systems := cfg.Systems + if systems != nil { + for n, s := range systems { + s.Name = n + } + } + return &cfg, nil +} + func (a *Auth0) init() error { a.initOnce.Do(func() { if a.errOnce = a.initContext(); a.errOnce != nil { @@ -303,10 +339,11 @@ func (a *Auth0) initContext() (err error) { return err } - if err := json.Unmarshal(buf, &a.config); err != nil { + cfg, err := a.jsonToConfig(buf) + if err != nil { return err } - + a.config = *cfg return nil } diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index 9832f04e3f0..f6113adf5d6 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -16,15 +16,24 @@ import ( var overwriteKey bool func init() { - rootCmd.AddCommand(apiKeyCmd) apiKeyCmd.Flags().BoolVarP(&overwriteKey, "force", "f", false, "Force overwrite of existing API key") apiKeyCmd.MarkPersistentFlagRequired(applicationFlag) } +var example string + +func apiKeyExample() string { + if vespa.Auth0AccessTokenEnabled() { + return "$ vespa auth api-key -a my-tenant.my-app.my-instance" + } else { + return "$ vespa api-key -a my-tenant.my-app.my-instance" + } +} + var apiKeyCmd = &cobra.Command{ Use: "api-key", Short: "Create a new user API key for authentication with Vespa Cloud", - Example: "$ vespa api-key -a my-tenant.my-app.my-instance", + Example: apiKeyExample(), DisableAutoGenTag: true, Args: cobra.ExactArgs(0), Run: func(cmd *cobra.Command, args []string) { diff --git a/client/go/cmd/auth.go b/client/go/cmd/auth.go new file mode 100644 index 00000000000..8f306356267 --- /dev/null +++ b/client/go/cmd/auth.go @@ -0,0 +1,32 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/vespa" +) + +func init() { + if vespa.Auth0AccessTokenEnabled() { + rootCmd.AddCommand(authCmd) + authCmd.AddCommand(certCmd) + authCmd.AddCommand(apiKeyCmd) + authCmd.AddCommand(loginCmd) + authCmd.AddCommand(logoutCmd) + } else { + rootCmd.AddCommand(certCmd) + rootCmd.AddCommand(apiKeyCmd) + } +} + +var authCmd = &cobra.Command{ + Use: "auth", + Short: "Manage Vespa Cloud credentials", + Long: `Manage Vespa Cloud credentials.`, + + DisableAutoGenTag: true, + Run: func(cmd *cobra.Command, args []string) { + // Root command does nothing + cmd.Help() + exitFunc(1) + }, +} diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index eaf3fc564dd..6fbe19b524d 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -16,15 +16,22 @@ import ( var overwriteCertificate bool func init() { - rootCmd.AddCommand(certCmd) certCmd.Flags().BoolVarP(&overwriteCertificate, "force", "f", false, "Force overwrite of existing certificate and private key") certCmd.MarkPersistentFlagRequired(applicationFlag) } +func certExample() string { + if vespa.Auth0AccessTokenEnabled() { + return "$ vespa auth cert -a my-tenant.my-app.my-instance" + } else { + return "$ vespa cert -a my-tenant.my-app.my-instance" + } +} + var certCmd = &cobra.Command{ Use: "cert", Short: "Create a new private key and self-signed certificate for Vespa Cloud deployment", - Example: "$ vespa cert -a my-tenant.my-app.my-instance", + Example: certExample(), DisableAutoGenTag: true, Args: cobra.MaximumNArgs(1), Run: func(cmd *cobra.Command, args []string) { @@ -66,8 +73,14 @@ var certCmd = &cobra.Command{ } } if pkg.IsZip() { + var msg string + if vespa.Auth0AccessTokenEnabled() { + msg = "Try running 'mvn clean' before 'vespa auth cert', and then 'mvn package'" + } else { + msg = "Try running 'mvn clean' before 'vespa cert', and then 'mvn package'" + } fatalErrHint(fmt.Errorf("Cannot add certificate to compressed application package %s", pkg.Path), - "Try running 'mvn clean' before 'vespa cert', and then 'mvn package'") + msg) return } diff --git a/client/go/cmd/command_tester.go b/client/go/cmd/command_tester.go index 2d2de6a201c..eb55021b536 100644 --- a/client/go/cmd/command_tester.go +++ b/client/go/cmd/command_tester.go @@ -127,4 +127,4 @@ func (c *mockHttpClient) Do(request *http.Request, timeout time.Duration) (*http nil } -func (c *mockHttpClient) UseCertificate(certificate tls.Certificate) {} +func (c *mockHttpClient) UseCertificate(certificates []tls.Certificate) {} diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 966080dbdc2..3a6e43e7ffe 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -155,7 +155,7 @@ func (c *Config) ReadAPIKey(tenantName string) ([]byte, error) { } func (c *Config) AuthConfigPath() string { - return filepath.Join(c.Home, "auth0.json") + return filepath.Join(c.Home, "auth.json") } func (c *Config) ReadSessionID(app vespa.ApplicationID) (int64, error) { diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index 89ea87f198e..f065ae0c680 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -205,7 +205,13 @@ func getTarget() vespa.Target { } kp, err := tls.LoadX509KeyPair(certificateFile, privateKeyFile) if err != nil { - fatalErrHint(err, "Deployment to cloud requires a certificate. Try 'vespa cert'") + var msg string + if vespa.Auth0AccessTokenEnabled() { + msg = "Deployment to cloud requires a certificate. Try 'vespa auth cert'" + } else { + msg = "Deployment to cloud requires a certificate. Try 'vespa cert'" + } + fatalErrHint(err, msg) } var cloudAuth string if vespa.Auth0AccessTokenEnabled() { @@ -262,7 +268,13 @@ func getDeploymentOpts(cfg *Config, pkg vespa.ApplicationPackage, target vespa.T if opts.IsCloud() { deployment := deploymentFromArgs() if !opts.ApplicationPackage.HasCertificate() { - fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", "Try 'vespa cert'") + var msg string + if vespa.Auth0AccessTokenEnabled() { + msg = "Try 'vespa auth cert'" + } else { + msg = "Try 'vespa cert'" + } + fatalErrHint(fmt.Errorf("Missing certificate in application package"), "Applications in Vespa Cloud require a certificate", msg) return opts } var err error diff --git a/client/go/cmd/login.go b/client/go/cmd/login.go index f7b412a4613..5011b290b9f 100644 --- a/client/go/cmd/login.go +++ b/client/go/cmd/login.go @@ -6,17 +6,11 @@ import ( "github.com/vespa-engine/vespa/client/go/vespa" ) -func init() { - if vespa.Auth0AccessTokenEnabled() { - rootCmd.AddCommand(loginCmd) - } -} - var loginCmd = &cobra.Command{ Use: "login", Args: cobra.NoArgs, Short: "Authenticate the Vespa CLI", - Example: "$ vespa login", + Example: "$ vespa auth login", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { ctx := cmd.Context() diff --git a/client/go/cmd/logout.go b/client/go/cmd/logout.go index e3cfe6733eb..ddc1d36d5e1 100644 --- a/client/go/cmd/logout.go +++ b/client/go/cmd/logout.go @@ -3,20 +3,13 @@ package cmd import ( "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/auth0" - "github.com/vespa-engine/vespa/client/go/vespa" ) -func init() { - if vespa.Auth0AccessTokenEnabled() { - rootCmd.AddCommand(logoutCmd) - } -} - var logoutCmd = &cobra.Command{ Use: "logout", Args: cobra.NoArgs, Short: "Log out of Vespa Cli", - Example: "$ vespa logout", + Example: "$ vespa auth logout", DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index d1d72362448..89dc4cb6094 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -116,7 +116,7 @@ For more information about production deployments in Vespa Cloud see: https://cloud.vespa.ai/en/getting-to-production https://cloud.vespa.ai/en/automated-deployments`, DisableAutoGenTag: true, - Example: `$ mvn package + Example: `$ mvn package # when adding custom Java components $ vespa prod submit`, Run: func(cmd *cobra.Command, args []string) { target := getTarget() @@ -139,10 +139,13 @@ $ vespa prod submit`, fatalErrHint(fmt.Errorf("No deployment.xml found"), "Try creating one with vespa prod init") return } - if !pkg.IsJava() { - // TODO: Loosen this requirement when we start supporting applications with Java in production - fatalErrHint(fmt.Errorf("No jar files found in %s", pkg.Path), "Only applications containing Java components are currently supported") + if pkg.TestPath == "" { + fatalErrHint(fmt.Errorf("No tests found"), + "The application must be a Java maven project, or include basic HTTP tests under src/test/application/", + "See https://cloud.vespa.ai/en/reference/getting-to-production") return + } else { + verifyTests(pkg.TestPath, target) } isCI := os.Getenv("CI") != "" if !isCI { @@ -347,3 +350,12 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu } return input } + +func verifyTests(testsParent string, target vespa.Target) { + runTests(filepath.Join(testsParent, "tests", "system-test"), target, true) + runTests(filepath.Join(testsParent, "tests", "staging-setup"), target, true) + runTests(filepath.Join(testsParent, "tests", "staging-test"), target, true) + if util.PathExists(filepath.Join(testsParent, "tests", "production-test")) { + runTests(filepath.Join(testsParent, "tests", "production-test"), target, true) + } +} diff --git a/client/go/cmd/prod_test.go b/client/go/cmd/prod_test.go index 4ce6112122a..a4f3ebd6b56 100644 --- a/client/go/cmd/prod_test.go +++ b/client/go/cmd/prod_test.go @@ -16,7 +16,7 @@ import ( func TestProdInit(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") - createApplication(t, pkgDir) + createApplication(t, pkgDir, false) answers := []string{ // Regions @@ -81,7 +81,7 @@ func readFileString(t *testing.T, filename string) string { return string(content) } -func createApplication(t *testing.T, pkgDir string) { +func createApplication(t *testing.T, pkgDir string, java bool) { appDir := filepath.Join(pkgDir, "src", "main", "application") targetDir := filepath.Join(pkgDir, "target") if err := os.MkdirAll(appDir, 0755); err != nil { @@ -120,7 +120,24 @@ func createApplication(t *testing.T, pkgDir string) { if err := os.MkdirAll(targetDir, 0755); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + if java { + if err := ioutil.WriteFile(filepath.Join(pkgDir, "pom.xml"), []byte(""), 0644); err != nil { + t.Fatal(err) + } + } else { + testsDir := filepath.Join(pkgDir, "src", "test", "application", "tests") + testBytes, _ := ioutil.ReadAll(strings.NewReader("{\"steps\":[{}]}")) + writeTest(filepath.Join(testsDir, "system-test", "test.json"), testBytes, t) + writeTest(filepath.Join(testsDir, "staging-setup", "test.json"), testBytes, t) + writeTest(filepath.Join(testsDir, "staging-test", "test.json"), testBytes, t) + } +} + +func writeTest(path string, content []byte, t *testing.T) { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := ioutil.WriteFile(path, content, 0644); err != nil { t.Fatal(err) } } @@ -128,7 +145,37 @@ func createApplication(t *testing.T, pkgDir string) { func TestProdSubmit(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") pkgDir := filepath.Join(t.TempDir(), "app") - createApplication(t, pkgDir) + createApplication(t, pkgDir, false) + + httpClient := &mockHttpClient{} + httpClient.NextResponse(200, `ok`) + execute(command{homeDir: homeDir, args: []string{"config", "set", "application", "t1.a1.i1"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"config", "set", "target", "cloud"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) + execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) + + // Zipping requires relative paths, so much let command run from pkgDir, then reset cwd for subsequent tests. + if cwd, err := os.Getwd(); err != nil { + t.Fatal(err) + } else { + defer os.Chdir(cwd) + } + if err := os.Chdir(pkgDir); err != nil { + t.Fatal(err) + } + if err := os.Setenv("CI", "true"); err != nil { + t.Fatal(err) + } + out, err := execute(command{homeDir: homeDir, args: []string{"prod", "submit"}}, t, httpClient) + assert.Equal(t, "", err) + assert.Contains(t, out, "Success: Submitted") + assert.Contains(t, out, "See https://console.vespa.oath.cloud/tenant/t1/application/a1/prod/deployment for deployment progress") +} + +func TestProdSubmitWithJava(t *testing.T) { + homeDir := filepath.Join(t.TempDir(), ".vespa") + pkgDir := filepath.Join(t.TempDir(), "app") + createApplication(t, pkgDir, true) httpClient := &mockHttpClient{} httpClient.NextResponse(200, `ok`) @@ -137,7 +184,7 @@ func TestProdSubmit(t *testing.T) { execute(command{homeDir: homeDir, args: []string{"api-key"}}, t, httpClient) execute(command{homeDir: homeDir, args: []string{"cert", pkgDir}}, t, httpClient) - // Copy an application package pre-assambled with mvn package + // Copy an application package pre-assembled with mvn package testAppDir := filepath.Join("testdata", "applications", "withDeployment", "target") zipFile := filepath.Join(testAppDir, "application.zip") copyFile(t, filepath.Join(pkgDir, "target", "application.zip"), zipFile) diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index dc75ca22729..7c49703595e 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -6,6 +6,7 @@ package cmd import ( "bytes" + "crypto/tls" "encoding/json" "fmt" "github.com/spf13/cobra" @@ -26,16 +27,15 @@ func init() { rootCmd.AddCommand(testCmd) } -// TODO: add link to test doc at cloud.vespa.ai var testCmd = &cobra.Command{ Use: "test [tests directory or test file]", Short: "Run a test suite, or a single test", Long: `Run a test suite, or a single test -Runs all JSON test files in the specified directory, or the single JSON -test file specified. +Runs all JSON test files in the specified directory (the working +directory by default), or the single JSON test file specified. -If no directory or file is specified, the working directory is used instead.`, +See https://cloud.vespa.ai/en/reference/testing.html for details.`, Example: `$ vespa test src/test/application/tests/system-test $ vespa test src/test/application/tests/system-test/feed-and-query.json`, Args: cobra.MaximumNArgs(1), @@ -46,108 +46,127 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, if len(args) > 0 { testPath = args[0] } - if count, failed := runTests(testPath, target); len(failed) != 0 { + if count, failed := runTests(testPath, target, false); len(failed) != 0 { fmt.Fprintf(stdout, "\nFailed %d of %d tests:\n", len(failed), count) for _, test := range failed { fmt.Fprintln(stdout, test) } exitFunc(3) - } else if count == 0 { - fmt.Fprintf(stdout, "Failed to find any tests at '%v'\n", testPath) - exitFunc(3) } else { - fmt.Fprintf(stdout, "%d tests completed successfully\n", count) + plural := "s" + if count == 1 { + plural = "" + } + fmt.Fprintf(stdout, "\n%d test%s completed successfully\n", count, plural) } }, } -func runTests(rootPath string, target vespa.Target) (int, []string) { +func runTests(rootPath string, target vespa.Target, dryRun bool) (int, []string) { count := 0 failed := make([]string, 0) if stat, err := os.Stat(rootPath); err != nil { - fatalErr(err, "Failed reading specified test path") + fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") } else if stat.IsDir() { tests, err := ioutil.ReadDir(rootPath) // TODO: Use os.ReadDir when >= 1.16 is required. if err != nil { - fatalErr(err, "Failed reading specified test directory") + fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") } + previousFailed := false for _, test := range tests { if !test.IsDir() && filepath.Ext(test.Name()) == ".json" { testPath := path.Join(rootPath, test.Name()) - failure := runTest(testPath, target) + if previousFailed { + fmt.Fprintln(stdout, "") + previousFailed = false + } + failure := runTest(testPath, target, dryRun) if failure != "" { failed = append(failed, failure) + previousFailed = true } count++ } } } else if strings.HasSuffix(stat.Name(), ".json") { - failure := runTest(rootPath, target) + failure := runTest(rootPath, target, dryRun) if failure != "" { failed = append(failed, failure) } count++ } + if count == 0 { + fatalErrHint(fmt.Errorf("Failed to find any tests at %s", rootPath), "See https://cloud.vespa.ai/en/reference/testing") + } return count, failed } // Runs the test at the given path, and returns the specified test name if the test fails -func runTest(testPath string, target vespa.Target) string { +func runTest(testPath string, target vespa.Target, dryRun bool) string { var test test testBytes, err := ioutil.ReadFile(testPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read test file at '%s'", testPath)) + fatalErrHint(err, "See https://cloud.vespa.ai/en/reference/testing") } if err = json.Unmarshal(testBytes, &test); err != nil { - fatalErr(err, fmt.Sprintf("Failed to parse test file at '%s", testPath)) + fatalErrHint(err, fmt.Sprintf("Failed parsing test at %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") } testName := test.Name if test.Name == "" { - testName = testPath + testName = filepath.Base(testPath) + } + if !dryRun { + fmt.Fprintf(stdout, "Running %s:", testName) } - fmt.Fprintf(stdout, "Running %s:", testName) defaultParameters, err := getParameters(test.Defaults.ParametersRaw, path.Dir(testPath)) if err != nil { - fatalErr(err, fmt.Sprintf("Invalid default parameters for '%s'", testName)) + fmt.Fprintln(stderr) + fatalErrHint(err, fmt.Sprintf("Invalid default parameters for %s", testName), "See https://cloud.vespa.ai/en/reference/testing") } - if len(test.Assertions) == 0 { - fatalErr(fmt.Errorf("a test must have at least one assertion, but none were found in '%s'", testPath)) + if len(test.Steps) == 0 { + fmt.Fprintln(stderr) + fatalErrHint(fmt.Errorf("a test must have at least one step, but none were found in %s", testPath), "See https://cloud.vespa.ai/en/reference/testing") } - for i, assertion := range test.Assertions { - assertionName := assertion.Name - if assertionName == "" { - assertionName = fmt.Sprintf("assertion %d", i) + for i, step := range test.Steps { + stepName := step.Name + if stepName == "" { + stepName = fmt.Sprintf("step %d", i+1) } - failure, err := verify(assertion, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target) + failure, longFailure, err := verify(step, path.Dir(testPath), test.Defaults.Cluster, defaultParameters, target, dryRun) if err != nil { - fatalErr(err, fmt.Sprintf("\nError verifying %s", assertionName)) + fmt.Fprintln(stderr) + fatalErrHint(err, fmt.Sprintf("Error in %s", stepName), "See https://cloud.vespa.ai/en/reference/testing") } - if failure != "" { - fmt.Fprintf(stdout, "\nFailed verifying %s:\n%s\n", assertionName, failure) - return fmt.Sprintf("%v: %v", testName, assertionName) - } - if i == 0 { - fmt.Fprintf(stdout, " ") + if !dryRun { + if failure != "" { + fmt.Fprintf(stdout, " Failed %s:\n%s\n", stepName, longFailure) + return fmt.Sprintf("%s: %s: %s", testName, stepName, failure) + } + if i == 0 { + fmt.Fprintf(stdout, " ") + } + fmt.Fprint(stdout, ".") } - fmt.Fprint(stdout, ".") } - fmt.Fprintln(stdout, " OK!") + if !dryRun { + fmt.Fprintln(stdout, " OK") + } return "" } // Asserts specified response is obtained for request, or returns a failure message, or an error if this fails -func verify(assertion assertion, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target) (string, error) { - requestBody, err := getBody(assertion.Request.BodyRaw, testsPath) +func verify(step step, testsPath string, defaultCluster string, defaultParameters map[string]string, target vespa.Target, dryRun bool) (string, string, error) { + requestBody, err := getBody(step.Request.BodyRaw, testsPath) if err != nil { - return "", err + return "", "", err } - parameters, err := getParameters(assertion.Request.ParametersRaw, testsPath) + parameters, err := getParameters(step.Request.ParametersRaw, testsPath) if err != nil { - return "", err + return "", "", err } for name, value := range defaultParameters { if _, present := parameters[name]; !present { @@ -155,28 +174,42 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul } } - cluster := assertion.Request.Cluster + cluster := step.Request.Cluster if cluster == "" { cluster = defaultCluster } - service, err := target.Service("query", 0, 0, cluster) - if err != nil { - return "", err + var service *vespa.Service + if !dryRun { + service, err = target.Service("query", 0, 0, cluster) + if err != nil { + return "", "", err + } } - method := assertion.Request.Method + method := step.Request.Method if method == "" { method = "GET" } - pathAndQuery := assertion.Request.URI - if pathAndQuery == "" { - pathAndQuery = "/search/" + requestUri := step.Request.URI + if requestUri == "" { + requestUri = "/search/" } - requestUrl, err := url.ParseRequestURI(service.BaseURL + pathAndQuery) + requestUrl, err := url.ParseRequestURI(requestUri) if err != nil { - return "", err + return "", "", err + } + externalEndpoint := requestUrl.IsAbs() + if !externalEndpoint { + baseURL := "http://dummy/" + if service != nil { + baseURL = service.BaseURL + } + requestUrl, err = url.ParseRequestURI(baseURL + requestUri) + if err != nil { + return "", "", err + } } query := requestUrl.Query() for name, value := range parameters { @@ -195,52 +228,72 @@ func verify(assertion assertion, testsPath string, defaultCluster string, defaul } defer request.Body.Close() - response, err := service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout - if err != nil { - return "", err - } - defer response.Body.Close() - - statusCode := assertion.Response.Code + statusCode := step.Response.Code if statusCode == 0 { statusCode = 200 } - if statusCode != response.StatusCode { - return fmt.Sprintf("Expected status code (%d) does not match actual (%d). Response body:\n%s", statusCode, response.StatusCode, util.ReaderToJSON(response.Body)), nil - } - responseBodySpecBytes, err := getBody(assertion.Response.BodyRaw, testsPath) + responseBodySpecBytes, err := getBody(step.Response.BodyRaw, testsPath) if err != nil { - return "", err - } - if responseBodySpecBytes == nil { - return "", nil + return "", "", err } var responseBodySpec interface{} - err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) + if responseBodySpecBytes != nil { + err = json.Unmarshal(responseBodySpecBytes, &responseBodySpec) + if err != nil { + return "", "", fmt.Errorf("invalid response body spec: %w", err) + } + } + + if dryRun { + return "", "", nil + } + + var response *http.Response + if externalEndpoint { + util.ActiveHttpClient.UseCertificate([]tls.Certificate{}) + response, err = util.ActiveHttpClient.Do(request, 60*time.Second) + } else { + response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout + } if err != nil { - return "", err + return "", "", err + } + defer response.Body.Close() + + if statusCode != response.StatusCode { + failure := fmt.Sprintf("Unexpected status code: %d", response.StatusCode) + return failure, fmt.Sprintf("%s\nExpected: %d\nActual response:\n%s", failure, statusCode, util.ReaderToJSON(response.Body)), nil + } + + if responseBodySpec == nil { + return "", "", nil } responseBodyBytes, err := ioutil.ReadAll(response.Body) if err != nil { - return "", err + return "", "", err } var responseBody interface{} err = json.Unmarshal(responseBodyBytes, &responseBody) if err != nil { - return "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) + return "", "", fmt.Errorf("got non-JSON response; %w:\n%s", err, string(responseBodyBytes)) } - failure, err := compare(responseBodySpec, responseBody, "") + failure, expected, err := compare(responseBodySpec, responseBody, "") if failure != "" { responsePretty, _ := json.MarshalIndent(responseBody, "", " ") - failure = failure + " Response body:\n" + string(responsePretty) + longFailure := failure + if expected != "" { + longFailure += "\n" + expected + } + longFailure += "\nActual response:\n" + string(responsePretty) + return failure, longFailure, err } - return failure, err + return "", "", err } -func compare(expected interface{}, actual interface{}, path string) (string, error) { +func compare(expected interface{}, actual interface{}, path string) (string, string, error) { typeMatch := false valueMatch := false switch u := expected.(type) { @@ -265,14 +318,14 @@ func compare(expected interface{}, actual interface{}, path string) (string, err if ok { if len(u) == len(v) { for i, e := range u { - result, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)) - if result != "" || err != nil { - return result, err + failure, expected, err := compare(e, v[i], fmt.Sprintf("%s/%d", path, i)) + if failure != "" || err != nil { + return failure, expected, err } } valueMatch = true } else { - return fmt.Sprintf("Expected number of elements at %s (%d) does not match actual (%d).", path, len(u), len(v)), nil + return fmt.Sprintf("Unexpected number of elements at %s: %d", path, len(v)), fmt.Sprintf("Expected: %d", len(u)), nil } } case map[string]interface{}: @@ -283,28 +336,32 @@ func compare(expected interface{}, actual interface{}, path string) (string, err childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1")) f, ok := v[n] if !ok { - return fmt.Sprintf("Expected field at %s not present in actual data.", childPath), nil + return fmt.Sprintf("Missing expected field at %s", childPath), "", nil } - result, err := compare(e, f, childPath) - if result != "" || err != nil { - return result, err + failure, expected, err := compare(e, f, childPath) + if failure != "" || err != nil { + return failure, expected, err } } valueMatch = true } default: - return "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected) + return "", "", fmt.Errorf("unexpected expected JSON type for value '%v'", expected) } - if !(typeMatch && valueMatch) { + if !valueMatch { if path == "" { path = "root" } - expectedJson, _ := json.MarshalIndent(expected, "", " ") - actualJson, _ := json.MarshalIndent(actual, "", " ") - return fmt.Sprintf("Expected JSON at %s (%s) does not match actual (%s).", path, expectedJson, actualJson), nil + mismatched := "type" + if typeMatch { + mismatched = "value" + } + expectedJson, _ := json.Marshal(expected) + actualJson, _ := json.Marshal(actual) + return fmt.Sprintf("Unexpected %s at %s: %s", mismatched, path, actualJson), fmt.Sprintf("Expected: %s", expectedJson), nil } - return "", nil + return "", "", nil } func getParameters(parametersRaw []byte, testsPath string) (map[string]string, error) { @@ -314,7 +371,7 @@ func getParameters(parametersRaw []byte, testsPath string) (map[string]string, e resolvedParametersPath := path.Join(testsPath, parametersPath) parametersRaw, err = ioutil.ReadFile(resolvedParametersPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read request parameters file at '%s'", resolvedParametersPath)) + return nil, fmt.Errorf("failed to read request parameters at %s: %w", resolvedParametersPath, err) } } var parameters map[string]string @@ -332,16 +389,16 @@ func getBody(bodyRaw []byte, testsPath string) ([]byte, error) { resolvedBodyPath := path.Join(testsPath, bodyPath) bodyRaw, err = ioutil.ReadFile(resolvedBodyPath) if err != nil { - fatalErr(err, fmt.Sprintf("Failed to read body file at '%s'", resolvedBodyPath)) + return nil, fmt.Errorf("failed to read body file at %s: %w", resolvedBodyPath, err) } } return bodyRaw, nil } type test struct { - Name string `json:"name"` - Defaults defaults `json:"defaults"` - Assertions []assertion `json:"assertions"` + Name string `json:"name"` + Defaults defaults `json:"defaults"` + Steps []step `json:"steps"` } type defaults struct { @@ -349,7 +406,7 @@ type defaults struct { ParametersRaw json.RawMessage `json:"parameters"` } -type assertion struct { +type step struct { Name string `json:"name"` Request request `json:"request"` Response response `json:"response"` diff --git a/client/go/cmd/test_test.go b/client/go/cmd/test_test.go index 9d92e285750..9a566beb10f 100644 --- a/client/go/cmd/test_test.go +++ b/client/go/cmd/test_test.go @@ -23,7 +23,7 @@ func TestSuite(t *testing.T) { searchResponse, _ := ioutil.ReadFile("testdata/tests/response.json") client.NextStatus(200) client.NextStatus(200) - for i := 0; i < 9; i++ { + for i := 0; i < 10; i++ { client.NextResponse(200, string(searchResponse)) } @@ -35,23 +35,31 @@ func TestSuite(t *testing.T) { baseUrl := "http://127.0.0.1:8080" urlWithQuery := baseUrl + "/search/?presentation.timing=true&query=artist%3A+foo&timeout=3.4s" requests := []*http.Request{createFeedRequest(baseUrl), createFeedRequest(baseUrl), createSearchRequest(urlWithQuery), createSearchRequest(urlWithQuery)} - for i := 0; i < 7; i++ { + for i := 0; i < 8; i++ { requests = append(requests, createSearchRequest(baseUrl+"/search/")) } assertRequests(requests, client, t) } +func TestProductionTest(t *testing.T) { + client := &mockHttpClient{} + client.NextStatus(200) + outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/production-test/external.json"}}, t, client) + assert.Equal(t, "Running external.json: . OK\n\n1 test completed successfully\n", outBytes) + assert.Equal(t, "", errBytes) + assertRequests([]*http.Request{createRequest("GET", "https://my.service:123/path?query=wohoo", "")}, client, t) +} + func TestTestWithoutAssertions(t *testing.T) { client := &mockHttpClient{} _, errBytes := execute(command{args: []string{"test", "testdata/tests/system-test/foo/query.json"}}, t, client) - assert.Equal(t, "a test must have at least one assertion, but none were found in 'testdata/tests/system-test/foo/query.json'\n", errBytes) + assert.Equal(t, "\nError: a test must have at least one step, but none were found in testdata/tests/system-test/foo/query.json\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) } func TestSuiteWithoutTests(t *testing.T) { client := &mockHttpClient{} - outBytes, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) - assert.Equal(t, "Failed to find any tests at 'testdata/tests/staging-test'\n", outBytes) - assert.Equal(t, "", errBytes) + _, errBytes := execute(command{args: []string{"test", "testdata/tests/staging-test"}}, t, client) + assert.Equal(t, "Error: Failed to find any tests at testdata/tests/staging-test\nHint: See https://cloud.vespa.ai/en/reference/testing\n", errBytes) } func TestSingleTest(t *testing.T) { diff --git a/client/go/cmd/testdata/tests/expected-suite.out b/client/go/cmd/testdata/tests/expected-suite.out index 0fb8b897f4f..bef3d678957 100644 --- a/client/go/cmd/testdata/tests/expected-suite.out +++ b/client/go/cmd/testdata/tests/expected-suite.out @@ -1,7 +1,8 @@ -Running testdata/tests/system-test/test.json: .... OK! -Running testdata/tests/system-test/wrong-bool-value.json: -Failed verifying assertion 0: -Expected JSON at /root/coverage/full (false) does not match actual (true). Response body: +Running my test: .... OK +Running wrong-bool-value.json: Failed step 1: +Unexpected value at /root/coverage/full: true +Expected: false +Actual response: { "root": { "children": [ @@ -36,9 +37,11 @@ Expected JSON at /root/coverage/full (false) does not match actual (true). Respo "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-element-count.json: -Failed verifying assertion 0: -Expected number of elements at /root/children (0) does not match actual (1). Response body: + +Running wrong-element-count.json: Failed step 1: +Unexpected number of elements at /root/children: 1 +Expected: 0 +Actual response: { "root": { "children": [ @@ -73,9 +76,10 @@ Expected number of elements at /root/children (0) does not match actual (1). Res "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-field-name.json: -Failed verifying assertion 0: -Expected field at /root/fields/totalCountDracula not present in actual data. Response body: + +Running wrong-field-name.json: Failed step 1: +Missing expected field at /root/fields/totalCountDracula +Actual response: { "root": { "children": [ @@ -110,9 +114,11 @@ Expected field at /root/fields/totalCountDracula not present in actual data. Res "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-float-value.json: -Failed verifying assertion 0: -Expected JSON at /root/children/0/relevance (0.381862373599) does not match actual (0.38186238359951247). Response body: + +Running wrong-float-value.json: Failed step 1: +Unexpected value at /root/children/0/relevance: 0.38186238359951247 +Expected: 0.381862373599 +Actual response: { "root": { "children": [ @@ -147,9 +153,11 @@ Expected JSON at /root/children/0/relevance (0.381862373599) does not match actu "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-int-value.json: -Failed verifying assertion 0: -Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response body: + +Running wrong-int-value.json: Failed step 1: +Unexpected value at /root/fields/totalCount: 1 +Expected: 2 +Actual response: { "root": { "children": [ @@ -184,9 +192,10 @@ Expected JSON at /root/fields/totalCount (2) does not match actual (1). Response "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-null-value.json: -Failed verifying assertion 0: -Expected field at /boot not present in actual data. Response body: + +Running wrong-null-value.json: Failed step 1: +Missing expected field at /boot +Actual response: { "root": { "children": [ @@ -221,9 +230,50 @@ Expected field at /boot not present in actual data. Response body: "summaryfetchtime": 0 } } -Running testdata/tests/system-test/wrong-string-value.json: -Failed verifying assertion 0: -Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match actual ("Foo Fighters"). Response body: + +Running wrong-string-value.json: Failed step 1: +Unexpected value at /root/children/0/fields/artist: "Foo Fighters" +Expected: "Boo Fighters" +Actual response: +{ + "root": { + "children": [ + { + "fields": { + "artist": "Foo Fighters", + "documentid": "id:test:music::doc", + "sddocname": "music" + }, + "id": "id:test:music::doc", + "relevance": 0.38186238359951247, + "source": "music" + } + ], + "coverage": { + "coverage": 100, + "documents": 1, + "full": true, + "nodes": 1, + "results": 1, + "resultsFull": 1 + }, + "fields": { + "totalCount": 1 + }, + "id": "toplevel", + "relevance": 1 + }, + "timing": { + "querytime": 0.003, + "searchtime": 0.004, + "summaryfetchtime": 0 + } +} + +Running wrong-type.json: Failed step 1: +Unexpected type at /root/fields/totalCount: 1 +Expected: "1" +Actual response: { "root": { "children": [ @@ -259,11 +309,12 @@ Expected JSON at /root/children/0/fields/artist ("Boo Fighters") does not match } } -Failed 7 of 8 tests: -testdata/tests/system-test/wrong-bool-value.json: assertion 0 -testdata/tests/system-test/wrong-element-count.json: assertion 0 -testdata/tests/system-test/wrong-field-name.json: assertion 0 -testdata/tests/system-test/wrong-float-value.json: assertion 0 -testdata/tests/system-test/wrong-int-value.json: assertion 0 -testdata/tests/system-test/wrong-null-value.json: assertion 0 -testdata/tests/system-test/wrong-string-value.json: assertion 0 +Failed 8 of 9 tests: +wrong-bool-value.json: step 1: Unexpected value at /root/coverage/full: true +wrong-element-count.json: step 1: Unexpected number of elements at /root/children: 1 +wrong-field-name.json: step 1: Missing expected field at /root/fields/totalCountDracula +wrong-float-value.json: step 1: Unexpected value at /root/children/0/relevance: 0.38186238359951247 +wrong-int-value.json: step 1: Unexpected value at /root/fields/totalCount: 1 +wrong-null-value.json: step 1: Missing expected field at /boot +wrong-string-value.json: step 1: Unexpected value at /root/children/0/fields/artist: "Foo Fighters" +wrong-type.json: step 1: Unexpected type at /root/fields/totalCount: 1 diff --git a/client/go/cmd/testdata/tests/expected.out b/client/go/cmd/testdata/tests/expected.out index f012ee30e95..d144dbe2cfa 100644 --- a/client/go/cmd/testdata/tests/expected.out +++ b/client/go/cmd/testdata/tests/expected.out @@ -1,2 +1,3 @@ -Running testdata/tests/system-test/test.json: .... OK! -1 tests completed successfully +Running my test: .... OK + +1 test completed successfully diff --git a/client/go/cmd/testdata/tests/production-test/external.json b/client/go/cmd/testdata/tests/production-test/external.json new file mode 100644 index 00000000000..af288bc8b1b --- /dev/null +++ b/client/go/cmd/testdata/tests/production-test/external.json @@ -0,0 +1,9 @@ +{ + "steps": [ + { + "request": { + "uri": "https://my.service:123/path?query=wohoo" + } + } + ] +}
\ No newline at end of file diff --git a/client/go/cmd/testdata/tests/system-test/test.json b/client/go/cmd/testdata/tests/system-test/test.json index 5aac76d29ff..f53df929dbd 100644 --- a/client/go/cmd/testdata/tests/system-test/test.json +++ b/client/go/cmd/testdata/tests/system-test/test.json @@ -1,11 +1,12 @@ { + "name": "my test", "defaults": { "cluster": "container", "parameters": { "timeout": "3.4s" } }, - "assertions": [ + "steps": [ { "name": "feed music", "request": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json index ae6f9de8de8..c594a206347 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-bool-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json index 77c687fa919..a772af67a78 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-element-count.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-element-count.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json index d020141ed12..6ce3d055584 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-field-name.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-field-name.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json index 804f2582176..6a1b221a91a 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-float-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-float-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json index 3cbf8acd1d8..d61a8b002c2 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-int-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-int-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json index 11425df7ad4..ea78357c99e 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-null-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-null-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json index 2cf0a5fdb38..5f56ebaab6d 100644 --- a/client/go/cmd/testdata/tests/system-test/wrong-string-value.json +++ b/client/go/cmd/testdata/tests/system-test/wrong-string-value.json @@ -1,5 +1,5 @@ { - "assertions": [ + "steps": [ { "response": { "body": { diff --git a/client/go/cmd/testdata/tests/system-test/wrong-type.json b/client/go/cmd/testdata/tests/system-test/wrong-type.json new file mode 100644 index 00000000000..6be28ff68ff --- /dev/null +++ b/client/go/cmd/testdata/tests/system-test/wrong-type.json @@ -0,0 +1,15 @@ +{ + "steps": [ + { + "response": { + "body": { + "root": { + "fields": { + "totalCount" : "1" + } + } + } + } + } + ] +} diff --git a/client/go/util/http.go b/client/go/util/http.go index acd9bb4f7ec..d5b8e3128ff 100644 --- a/client/go/util/http.go +++ b/client/go/util/http.go @@ -19,7 +19,7 @@ var ActiveHttpClient = CreateClient(time.Second * 10) type HttpClient interface { Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) - UseCertificate(certificate tls.Certificate) + UseCertificate(certificate []tls.Certificate) } type defaultHttpClient struct { @@ -33,9 +33,9 @@ func (c *defaultHttpClient) Do(request *http.Request, timeout time.Duration) (re return c.client.Do(request) } -func (c *defaultHttpClient) UseCertificate(certificate tls.Certificate) { +func (c *defaultHttpClient) UseCertificate(certificates []tls.Certificate) { c.client.Transport = &http.Transport{TLSClientConfig: &tls.Config{ - Certificates: []tls.Certificate{certificate}, + Certificates: certificates, }} } diff --git a/client/go/util/http_test.go b/client/go/util/http_test.go index 0a0de1fdd4c..e87a1e5ada4 100644 --- a/client/go/util/http_test.go +++ b/client/go/util/http_test.go @@ -36,7 +36,7 @@ func (c mockHttpClient) Do(request *http.Request, timeout time.Duration) (respon nil } -func (c mockHttpClient) UseCertificate(certificate tls.Certificate) {} +func (c mockHttpClient) UseCertificate(certificates []tls.Certificate) {} func TestHttpRequest(t *testing.T) { ActiveHttpClient = mockHttpClient{} diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index f2d4bf8c248..d52fc969c37 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -139,7 +139,7 @@ func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { tempZip.Close() os.Remove(tempZip.Name()) }() - if err := zipDir(ap.Path, tempZip.Name()); err != nil { + if err := zipDir(zipFile, tempZip.Name()); err != nil { return nil, err } zipFile = tempZip.Name() @@ -167,6 +167,10 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application } } if util.PathExists(filepath.Join(zipOrDir, "src", "main", "application")) { + if util.PathExists(filepath.Join(zipOrDir, "src", "test", "application")) { + return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application"), + TestPath: filepath.Join(zipOrDir, "src", "test", "application")}, nil + } return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application")}, nil } if util.PathExists(filepath.Join(zipOrDir, "services.xml")) { @@ -445,7 +449,10 @@ func zipDir(dir string, destination string) error { } defer file.Close() - zippath := strings.TrimPrefix(path, dir) + zippath, err := filepath.Rel(dir, path) + if err != nil { + return err + } zipfile, err := w.Create(zippath) if err != nil { return err diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index 0b3223c0d2e..204dda6538f 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -90,7 +90,7 @@ func (t *customTarget) PrepareApiRequest(req *http.Request, sigKeyId string) err // Do sends request to this service. Any required authentication happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if s.TLSOptions.KeyPair.Certificate != nil { - util.ActiveHttpClient.UseCertificate(s.TLSOptions.KeyPair) + util.ActiveHttpClient.UseCertificate([]tls.Certificate{s.TLSOptions.KeyPair}) } return util.HttpDo(request, timeout, s.Description()) } @@ -536,7 +536,7 @@ type requestFunc func() *http.Request func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) { if certificate != nil { - util.ActiveHttpClient.UseCertificate(*certificate) + util.ActiveHttpClient.UseCertificate([]tls.Certificate{*certificate}) } var ( httpErr error diff --git a/client/pom.xml b/client/pom.xml index 3dee909b932..ea33b9f3adf 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -28,15 +28,8 @@ <version>1.6</version> </dependency> <dependency> - <groupId>org.spockframework</groupId> - <artifactId>spock-core</artifactId> - <version>1.3-groovy-2.5</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.codehaus.groovy</groupId> - <artifactId>groovy</artifactId> - <version>3.0.8</version> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> <scope>test</scope> </dependency> </dependencies> @@ -44,19 +37,6 @@ <build> <plugins> <plugin> - <groupId>org.codehaus.gmavenplus</groupId> - <artifactId>gmavenplus-plugin</artifactId> - <version>1.13.0</version> - <executions> - <execution> - <goals> - <goal>addTestSources</goal> - <goal>compileTests</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy deleted file mode 100644 index 0d6e2ca3506..00000000000 --- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy +++ /dev/null @@ -1,677 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package ai.vespa.client.dsl - -import spock.lang.Specification - -class QTest extends Specification { - - def "select specific fields"() { - given: - def q = Q.select("f1", "f2") - .from("sd1") - .where("f1").contains("v1") - .semicolon() - .build() - - expect: - q == """yql=select f1, f2 from sd1 where f1 contains "v1";""" - } - - def "select from specific sources"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").contains("v1") - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 contains "v1";""" - } - - def "select from multiples sources"() { - given: - def q = Q.select("*") - .from("sd1", "sd2") - .where("f1").contains("v1") - .semicolon() - .build() - - expect: - q == """yql=select * from sources sd1, sd2 where f1 contains "v1";""" - } - - def "basic 'and', 'andnot', 'or', 'offset', 'limit', 'param', 'order by', and 'contains'"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").contains("v1") - .and("f2").contains("v2") - .or("f3").contains("v3") - .andnot("f4").contains("v4") - .offset(1) - .limit(2) - .timeout(3) - .orderByDesc("f1") - .orderByAsc("f2") - .semicolon() - .param("paramk1", "paramv1") - .build() - - expect: - q == """yql=select * from sd1 where f1 contains "v1" and f2 contains "v2" or f3 contains "v3" and !(f4 contains "v4") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;¶mk1=paramv1""" - } - - def "matches"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").matches("v1") - .and("f2").matches("v2") - .or("f3").matches("v3") - .andnot("f4").matches("v4") - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 matches "v1" and f2 matches "v2" or f3 matches "v3" and !(f4 matches "v4");""" - } - - def "numeric operations"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").le(1) - .and("f2").lt(2) - .and("f3").ge(3) - .and("f4").gt(4) - .and("f5").eq(5) - .and("f6").inRange(6, 7) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 <= 1 and f2 < 2 and f3 >= 3 and f4 > 4 and f5 = 5 and range(f6, 6, 7);""" - } - - def "long numeric operations"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").le(1L) - .and("f2").lt(2L) - .and("f3").ge(3L) - .and("f4").gt(4L) - .and("f5").eq(5L) - .and("f6").inRange(6L, 7L) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 <= 1L and f2 < 2L and f3 >= 3L and f4 > 4L and f5 = 5L and range(f6, 6L, 7L);""" - } - - def "float numeric operations"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").le(1.1) - .and("f2").lt(2.2) - .and("f3").ge(3.3) - .and("f4").gt(4.4) - .and("f5").eq(5.5) - .and("f6").inRange(6.6, 7.7) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7);""" - } - - def "double numeric operations"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").le(1.1D) - .and("f2").lt(2.2D) - .and("f3").ge(3.3D) - .and("f4").gt(4.4D) - .and("f5").eq(5.5D) - .and("f6").inRange(6.6D, 7.7D) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7);""" - } - - def "nested queries"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").contains("1") - .andnot(Q.p(Q.p("f2").contains("2").and("f3").contains("3")) - .or(Q.p("f2").contains("4").andnot("f3").contains("5"))) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 contains "1" and !((f2 contains "2" and f3 contains "3") or (f2 contains "4" and !(f3 contains "5")));""" - } - - def "userInput (with and with out defaultIndex)"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.ui("value")) - .and(Q.ui("index", "value2")) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where userInput(@_1) and ([{"defaultIndex":"index"}]userInput(@_2_index));&_2_index=value2&_1=value""" - } - - def "dot product"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.dotPdt("f1", [a: 1, b: 2, c: 3])) - .and("f2").contains("1") - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where dotProduct(f1, {"a":1,"b":2,"c":3}) and f2 contains "1";""" - } - - def "weighted set"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.wtdSet("f1", [a: 1, b: 2, c: 3])) - .and("f2").contains("1") - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where weightedSet(f1, {"a":1,"b":2,"c":3}) and f2 contains "1";""" - } - - def "non empty"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.nonEmpty(Q.p("f1").contains("v1"))) - .and("f2").contains("v2") - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where nonEmpty(f1 contains "v1") and f2 contains "v2";""" - } - - - def "wand (with and without annotation)"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.wand("f1", [a: 1, b: 2, c: 3])) - .and(Q.wand("f2", [[1, 1], [2, 2]])) - .and( - Q.wand("f3", [[1, 1], [2, 2]]) - .annotate(A.a("scoreThreshold", 0.13)) - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where wand(f1, {"a":1,"b":2,"c":3}) and wand(f2, [[1,1],[2,2]]) and ([{"scoreThreshold":0.13}]wand(f3, [[1,1],[2,2]]));""" - } - - def "weak and (with and without annotation)"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.weakand(Q.p("f1").contains("v1").and("f2").contains("v2"))) - .and(Q.weakand(Q.p("f1").contains("v1").and("f2").contains("v2")) - .annotate(A.a("scoreThreshold", 0.13)) - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where weakAnd(f1 contains "v1", f2 contains "v2") and ([{"scoreThreshold":0.13}]weakAnd(f1 contains "v1", f2 contains "v2"));""" - } - - def "geo location"() { - given: - def q = Q.select("*") - .from("sd1") - .where("a").contains("b").and(Q.geoLocation("taiwan", 25.105497, 121.597366, "200km")) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where a contains "b" and geoLocation(taiwan, 25.105497, 121.597366, "200km");""" - } - - def "nearest neighbor query"() { - when: - def q = Q.select("*") - .from("sd1") - .where("a").contains("b") - .and(Q.nearestNeighbor("vec1", "vec2") - .annotate(A.a("targetHits", 10, "approximate", false)) - ) - .semicolon() - .build() - - then: - q == """yql=select * from sd1 where a contains "b" and ([{"approximate":false,"targetHits":10}]nearestNeighbor(vec1, vec2));""" - } - - def "invalid nearest neighbor should throws an exception (targetHits annotation is required)"() { - when: - def q = Q.select("*") - .from("sd1") - .where("a").contains("b").and(Q.nearestNeighbor("vec1", "vec2")) - .semicolon() - .build() - - then: - thrown(IllegalArgumentException) - } - - - def "rank with only query"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.rank( - Q.p("f1").contains("v1") - ) - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where rank(f1 contains "v1");""" - } - - def "rank"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.rank( - Q.p("f1").contains("v1"), - Q.p("f2").contains("v2"), - Q.p("f3").eq(3)) - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where rank(f1 contains "v1", f2 contains "v2", f3 = 3);""" - } - - def "rank with rank query array"() { - given: - Query[] ranks = [Q.p("f2").contains("v2"), Q.p("f3").eq(3)].toArray() - def q = Q.select("*") - .from("sd1") - .where(Q.rank( - Q.p("f1").contains("v1"), - ranks) - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where rank(f1 contains "v1", f2 contains "v2", f3 = 3);""" - } - - def "string/function annotations"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").contains(annotation, "v1") - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where f1 contains (${expected}"v1");""" - - where: - annotation | expected - A.filter() | """[{"filter":true}]""" - A.defaultIndex("idx") | """[{"defaultIndex":"idx"}]""" - A.a([a1: [k1: "v1", k2: 2]]) | """[{"a1":{"k1":"v1","k2":2}}]""" - } - - def "sub-expression annotations"() { - given: - def q = Q.select("*") - .from("sd1") - .where("f1").contains("v1").annotate(A.a("ak1", "av1")) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where ([{"ak1":"av1"}](f1 contains "v1"));""" - } - - def "sub-expressions annotations (annotate in the middle of query)"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.p("f1").contains("v1").annotate(A.a("ak1", "av1")).and("f2").contains("v2")) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where ([{"ak1":"av1"}](f1 contains "v1" and f2 contains "v2"));""" - } - - def "sub-expressions annotations (annotate in nested queries)"() { - given: - def q = Q.select("*") - .from("sd1") - .where(Q.p( - Q.p("f1").contains("v1").annotate(A.a("ak1", "av1"))) - .and("f2").contains("v2") - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sd1 where (([{"ak1":"av1"}](f1 contains "v1")) and f2 contains "v2");""" - } - - def "build query which created from Q.b without select and sources"() { - given: - def q = Q.p("f1").contains("v1") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains "v1";""" - } - - def "order by"() { - given: - def q = Q.p("f1").contains("v1") - .orderByAsc("f2") - .orderByAsc(A.a([function: "uca", locale: "en_US", strength: "IDENTICAL"]), "f3") - .orderByDesc("f4") - .orderByDesc(A.a([function: "lowercase"]), "f5") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains "v1" order by f2 asc, [{"function":"uca","locale":"en_US","strength":"IDENTICAL"}]f3 asc, f4 desc, [{"function":"lowercase"}]f5 desc;""" - } - - def "contains sameElement"() { - given: - def q = Q.p("f1").containsSameElement(Q.p("stime").le(1).and("etime").gt(2)) - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains sameElement(stime <= 1, etime > 2);""" - } - - def "contains phrase/near/onear/equiv"() { - given: - def funcName = "contains${operator.capitalize()}" - def q1 = Q.p("f1")."$funcName"("p1", "p2", "p3") - .semicolon() - .build() - def q2 = Q.p("f1")."$funcName"(["p1", "p2", "p3"]) - .semicolon() - .build() - - expect: - q1 == """yql=select * from sources * where f1 contains ${operator}("p1", "p2", "p3");""" - q2 == """yql=select * from sources * where f1 contains ${operator}("p1", "p2", "p3");""" - - where: - operator | _ - "phrase" | _ - "near" | _ - "onear" | _ - "equiv" | _ - } - - def "contains uri"() { - given: - def q = Q.p("f1").containsUri("https://test.uri") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains uri("https://test.uri");""" - } - - def "contains uri with annotation"() { - given: - def q = Q.p("f1").containsUri(A.a("key", "value"), "https://test.uri") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains ([{"key":"value"}]uri("https://test.uri"));""" - } - - def "nearestNeighbor"() { - given: - def q = Q.p("f1").nearestNeighbor("query_vector") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where nearestNeighbor(f1, query_vector);""" - } - - def "nearestNeighbor with annotation"() { - given: - def q = Q.p("f1").nearestNeighbor(A.a("targetHits", 10), "query_vector") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where ([{"targetHits":10}]nearestNeighbor(f1, query_vector));""" - } - - def "use contains instead of contains equiv when input size is 1"() { - def q = Q.p("f1").containsEquiv(["p1"]) - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains "p1";""" - } - - def "contains phrase/near/onear/equiv empty list should throw illegal argument exception"() { - given: - def funcName = "contains${operator.capitalize()}" - - when: - def q = Q.p("f1")."$funcName"([]) - .semicolon() - .build() - - then: - thrown(IllegalArgumentException) - - where: - operator | _ - "phrase" | _ - "near" | _ - "onear" | _ - "equiv" | _ - } - - - def "contains near/onear with annotation"() { - given: - def funcName = "contains${operator.capitalize()}" - def q = Q.p("f1")."$funcName"(A.a("distance", 5), "p1", "p2", "p3") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains ([{"distance":5}]${operator}("p1", "p2", "p3"));""" - - where: - operator | _ - "near" | _ - "onear" | _ - } - - def "basic group syntax"() { - /* - example from vespa document: - https://docs.vespa.ai/en/grouping.html - all( group(a) max(5) each(output(count()) - all(max(1) each(output(summary()))) - all(group(b) each(output(count()) - all(max(1) each(output(summary()))) - all(group(c) each(output(count()) - all(max(1) each(output(summary())))))))) ); - */ - given: - def q = Q.p("f1").contains("v1") - .group( - G.all(G.group("a"), G.maxRtn(5), G.each(G.output(G.count()), - G.all(G.maxRtn(1), G.each(G.output(G.summary()))), - G.all(G.group("b"), G.each(G.output(G.count()), - G.all(G.maxRtn(1), G.each(G.output(G.summary()))), - G.all(G.group("c"), G.each(G.output(G.count()), - G.all(G.maxRtn(1), G.each(G.output(G.summary()))) - )) - )) - )) - ) - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains "v1" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))));""" - } - - def "set group syntax string directly"() { - /* - example from vespa document: - https://docs.vespa.ai/en/grouping.html - all( group(a) max(5) each(output(count()) - all(max(1) each(output(summary()))) - all(group(b) each(output(count()) - all(max(1) each(output(summary()))) - all(group(c) each(output(count()) - all(max(1) each(output(summary())))))))) ); - */ - given: - def q = Q.p("f1").contains("v1") - .group("all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))))") - .semicolon() - .build() - - expect: - q == """yql=select * from sources * where f1 contains "v1" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))));""" - } - - def "arbitrary annotations"() { - given: - def a = A.a("a1", "v1", "a2", 2, "a3", [k: "v", k2: 1], "a4", 4D, "a5", [1, 2, 3]) - expect: - a.toString() == """{"a1":"v1","a2":2,"a3":{"k":"v","k2":1},"a4":4.0,"a5":[1,2,3]}""" - } - - def "test programmability"() { - given: - def map = [a: "1", b: "2", c: "3"] - - when: - Query q = map - .entrySet() - .stream() - .map { entry -> Q.p(entry.key).contains(entry.value) } - .reduce { q1, q2 -> q1.and(q2) } - .get() - - then: - q.semicolon().build() == """yql=select * from sources * where a contains "1" and b contains "2" and c contains "3";""" - } - - def "test programmability 2"() { - given: - def map = [a: "1", b: "2", c: "3"] - def q = Q.p() - - when: - map.each { k, v -> - q.and(Q.p(k).contains(v)) - } - - then: - q.semicolon().build() == """yql=select * from sources * where a contains "1" and b contains "2" and c contains "3";""" - } - - def "empty queries should not print out"() { - given: - def q = Q.p(Q.p(Q.p().andnot(Q.p()).and(Q.p()))).and("a").contains("1").semicolon().build() - - expect: - q == """yql=select * from sources * where a contains "1";""" - } - - def "validate positive search term of strings"() { - given: - def q = Q.p(Q.p("k1").contains("v1").and("k2").contains("v2").andnot("k3").contains("v3")) - .andnot(Q.p("nk1").contains("nv1").and("nk2").contains("nv2").andnot("nk3").contains("nv3")) - .and(Q.p("k4").contains("v4") - .andnot(Q.p("k5").contains("v5").andnot("k6").contains("v6")) - ) - - expect: - q.hasPositiveSearchField("k1") - q.hasPositiveSearchField("k2") - q.hasPositiveSearchField("nk3") - q.hasPositiveSearchField("k6") - q.hasPositiveSearchField("k6", "v6") - !q.hasPositiveSearchField("k6", "v5") - - q.hasNegativeSearchField("k3") - q.hasNegativeSearchField("nk1") - q.hasNegativeSearchField("nk2") - q.hasNegativeSearchField("k5") - q.hasNegativeSearchField("k5", "v5") - !q.hasNegativeSearchField("k5", "v4") - } - - def "validate positive search term of user input"() { - given: - def q = Q.p(Q.ui("k1", "v1")).and(Q.ui("k2", "v2")).andnot(Q.ui("k3", "v3")) - .andnot(Q.p(Q.ui("nk1", "nv1")).and(Q.ui("nk2", "nv2")).andnot(Q.ui("nk3", "nv3"))) - .and(Q.p(Q.ui("k4", "v4")) - .andnot(Q.p(Q.ui("k5", "v5")).andnot(Q.ui("k6", "v6"))) - ) - - expect: - q.hasPositiveSearchField("k1") - q.hasPositiveSearchField("k2") - q.hasPositiveSearchField("nk3") - q.hasPositiveSearchField("k6") - q.hasPositiveSearchField("k6", "v6") - !q.hasPositiveSearchField("k6", "v5") - - q.hasNegativeSearchField("k3") - q.hasNegativeSearchField("nk1") - q.hasNegativeSearchField("nk2") - q.hasNegativeSearchField("k5") - q.hasNegativeSearchField("k5", "v5") - !q.hasNegativeSearchField("k5", "v4") - } -} diff --git a/client/src/test/java/ai/vespa/client/dsl/QTest.java b/client/src/test/java/ai/vespa/client/dsl/QTest.java new file mode 100644 index 00000000000..08ab603fa04 --- /dev/null +++ b/client/src/test/java/ai/vespa/client/dsl/QTest.java @@ -0,0 +1,727 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package ai.vespa.client.dsl; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author unknown contributor + * @author bjorncs + */ +class QTest { + + @Test + void select_specific_fields() { + String q = Q.select("f1", "f2") + .from("sd1") + .where("f1").contains("v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select f1, f2 from sd1 where f1 contains \"v1\";"); + } + + @Test + void select_from_specific_sources() { + String q = Q.select("*") + .from("sd1") + .where("f1").contains("v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 contains \"v1\";"); + } + + @Test + void select_from_multiples_sources() { + String q = Q.select("*") + .from("sd1", "sd2") + .where("f1").contains("v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources sd1, sd2 where f1 contains \"v1\";"); + } + + @Test + void basic_and_andnot_or_offset_limit_param_order_by_and_contains() { + String q = Q.select("*") + .from("sd1") + .where("f1").contains("v1") + .and("f2").contains("v2") + .or("f3").contains("v3") + .andnot("f4").contains("v4") + .offset(1) + .limit(2) + .timeout(3) + .orderByDesc("f1") + .orderByAsc("f2") + .semicolon() + .param("paramk1", "paramv1") + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 contains \"v1\" and f2 contains \"v2\" or f3 contains \"v3\" and !(f4 contains \"v4\") order by f1 desc, f2 asc limit 2 offset 1 timeout 3;¶mk1=paramv1"); + } + + @Test + void matches() { + String q = Q.select("*") + .from("sd1") + .where("f1").matches("v1") + .and("f2").matches("v2") + .or("f3").matches("v3") + .andnot("f4").matches("v4") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 matches \"v1\" and f2 matches \"v2\" or f3 matches \"v3\" and !(f4 matches \"v4\");"); + } + + @Test + void numeric_operations() { + String q = Q.select("*") + .from("sd1") + .where("f1").le(1) + .and("f2").lt(2) + .and("f3").ge(3) + .and("f4").gt(4) + .and("f5").eq(5) + .and("f6").inRange(6, 7) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 <= 1 and f2 < 2 and f3 >= 3 and f4 > 4 and f5 = 5 and range(f6, 6, 7);"); + } + + @Test + void long_numeric_operations() { + String q = Q.select("*") + .from("sd1") + .where("f1").le(1L) + .and("f2").lt(2L) + .and("f3").ge(3L) + .and("f4").gt(4L) + .and("f5").eq(5L) + .and("f6").inRange(6L, 7L) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 <= 1L and f2 < 2L and f3 >= 3L and f4 > 4L and f5 = 5L and range(f6, 6L, 7L);"); + } + + @Test + void float_numeric_operations() { + String q = Q.select("*") + .from("sd1") + .where("f1").le(1.1) + .and("f2").lt(2.2) + .and("f3").ge(3.3) + .and("f4").gt(4.4) + .and("f5").eq(5.5) + .and("f6").inRange(6.6, 7.7) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7);"); + } + + @Test + void double_numeric_operations() { + String q = Q.select("*") + .from("sd1") + .where("f1").le(1.1D) + .and("f2").lt(2.2D) + .and("f3").ge(3.3D) + .and("f4").gt(4.4D) + .and("f5").eq(5.5D) + .and("f6").inRange(6.6D, 7.7D) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 <= 1.1 and f2 < 2.2 and f3 >= 3.3 and f4 > 4.4 and f5 = 5.5 and range(f6, 6.6, 7.7);"); + } + + @Test + void nested_queries() { + String q = Q.select("*") + .from("sd1") + .where("f1").contains("1") + .andnot(Q.p(Q.p("f2").contains("2").and("f3").contains("3")) + .or(Q.p("f2").contains("4").andnot("f3").contains("5"))) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 contains \"1\" and !((f2 contains \"2\" and f3 contains \"3\") or (f2 contains \"4\" and !(f3 contains \"5\")));"); + } + + @Test + void userInput_with_and_with_out_defaultIndex() { + String q = Q.select("*") + .from("sd1") + .where(Q.ui("value")) + .and(Q.ui("index", "value2")) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where userInput(@_1) and ([{\"defaultIndex\":\"index\"}]userInput(@_2_index));&_2_index=value2&_1=value"); + } + + @Test + void dot_product() { + String q = Q.select("*") + .from("sd1") + .where(Q.dotPdt("f1", stringIntMap("a", 1, "b", 2, "c", 3))) + .and("f2").contains("1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where dotProduct(f1, {\"a\":1,\"b\":2,\"c\":3}) and f2 contains \"1\";"); + } + + @Test + void weighted_set() { + String q = Q.select("*") + .from("sd1") + .where(Q.wtdSet("f1", stringIntMap("a", 1, "b", 2, "c", 3))) + .and("f2").contains("1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where weightedSet(f1, {\"a\":1,\"b\":2,\"c\":3}) and f2 contains \"1\";"); + } + + @Test + void non_empty() { + String q = Q.select("*") + .from("sd1") + .where(Q.nonEmpty(Q.p("f1").contains("v1"))) + .and("f2").contains("v2") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where nonEmpty(f1 contains \"v1\") and f2 contains \"v2\";"); + } + + + @Test + void wand_with_and_without_annotation() { + String q = Q.select("*") + .from("sd1") + .where(Q.wand("f1", stringIntMap("a", 1, "b", 2, "c", 3))) + .and(Q.wand("f2", Arrays.asList(Arrays.asList(1, 1), Arrays.asList(2, 2)))) + .and( + Q.wand("f3", Arrays.asList(Arrays.asList(1, 1), Arrays.asList(2, 2))) + .annotate(A.a("scoreThreshold", 0.13)) + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where wand(f1, {\"a\":1,\"b\":2,\"c\":3}) and wand(f2, [[1,1],[2,2]]) and ([{\"scoreThreshold\":0.13}]wand(f3, [[1,1],[2,2]]));"); + } + + @Test + void weak_and_with_and_without_annotation() { + String q = Q.select("*") + .from("sd1") + .where(Q.weakand(Q.p("f1").contains("v1").and("f2").contains("v2"))) + .and(Q.weakand(Q.p("f1").contains("v1").and("f2").contains("v2")) + .annotate(A.a("scoreThreshold", 0.13)) + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where weakAnd(f1 contains \"v1\", f2 contains \"v2\") and ([{\"scoreThreshold\":0.13}]weakAnd(f1 contains \"v1\", f2 contains \"v2\"));"); + } + + @Test + void geo_location() { + String q = Q.select("*") + .from("sd1") + .where("a").contains("b").and(Q.geoLocation("taiwan", 25.105497, 121.597366, "200km")) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where a contains \"b\" and geoLocation(taiwan, 25.105497, 121.597366, \"200km\");"); + } + + @Test + void nearest_neighbor_query() { + String q = Q.select("*") + .from("sd1") + .where("a").contains("b") + .and(Q.nearestNeighbor("vec1", "vec2") + .annotate(A.a("targetHits", 10, "approximate", false)) + ) + .semicolon() + .build(); + assertEquals(q, "yql=select * from sd1 where a contains \"b\" and ([{\"approximate\":false,\"targetHits\":10}]nearestNeighbor(vec1, vec2));"); + } + + @Test + void invalid_nearest_neighbor_should_throws_an_exception_targetHits_annotation_is_required() { + assertThrows(IllegalArgumentException.class, + () -> Q.select("*") + .from("sd1") + .where("a").contains("b").and(Q.nearestNeighbor("vec1", "vec2")) + .semicolon() + .build()); + } + + + @Test + void rank_with_only_query() { + String q = Q.select("*") + .from("sd1") + .where(Q.rank( + Q.p("f1").contains("v1") + ) + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\");"); + } + + @Test + void rank() { + String q = Q.select("*") + .from("sd1") + .where(Q.rank( + Q.p("f1").contains("v1"), + Q.p("f2").contains("v2"), + Q.p("f3").eq(3)) + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\", f2 contains \"v2\", f3 = 3);"); + } + + @Test + void rank_with_rank_query_array() { + Query[] ranks = new Query[]{Q.p("f2").contains("v2"), Q.p("f3").eq(3)}; + String q = Q.select("*") + .from("sd1") + .where(Q.rank( + Q.p("f1").contains("v1"), + ranks) + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where rank(f1 contains \"v1\", f2 contains \"v2\", f3 = 3);"); + } + + @Test + void stringfunction_annotations() { + + { + Annotation annotation = A.filter(); + String expected = "[{\"filter\":true}]"; + String q = Q.select("*") + .from("sd1") + .where("f1").contains(annotation, "v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\");"); + } + { + Annotation annotation = A.defaultIndex("idx"); + String expected = "[{\"defaultIndex\":\"idx\"}]"; + String q = Q.select("*") + .from("sd1") + .where("f1").contains(annotation, "v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\");"); + } + { + Annotation annotation = A.a(stringObjMap("a1", stringObjMap("k1", "v1", "k2", 2))); + String expected = "[{\"a1\":{\"k1\":\"v1\",\"k2\":2}}]"; + String q = Q.select("*") + .from("sd1") + .where("f1").contains(annotation, "v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where f1 contains (" + expected + "\"v1\");"); + } + + } + + @Test + void sub_expression_annotations() { + String q = Q.select("*") + .from("sd1") + .where("f1").contains("v1").annotate(A.a("ak1", "av1")) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where ([{\"ak1\":\"av1\"}](f1 contains \"v1\"));"); + } + + @Test + void sub_expressions_annotations_annotate_in_the_middle_of_query() { + String q = Q.select("*") + .from("sd1") + .where(Q.p("f1").contains("v1").annotate(A.a("ak1", "av1")).and("f2").contains("v2")) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where ([{\"ak1\":\"av1\"}](f1 contains \"v1\" and f2 contains \"v2\"));"); + } + + @Test + void sub_expressions_annotations_annotate_in_nested_queries() { + String q = Q.select("*") + .from("sd1") + .where(Q.p( + Q.p("f1").contains("v1").annotate(A.a("ak1", "av1"))) + .and("f2").contains("v2") + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sd1 where (([{\"ak1\":\"av1\"}](f1 contains \"v1\")) and f2 contains \"v2\");"); + } + + @Test + void build_query_which_created_from_Q_b_without_select_and_sources() { + String q = Q.p("f1").contains("v1") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\";"); + } + + @Test + void order_by() { + String q = Q.p("f1").contains("v1") + .orderByAsc("f2") + .orderByAsc(A.a(stringObjMap("function", "uca", "locale", "en_US", "strength", "IDENTICAL")), "f3") + .orderByDesc("f4") + .orderByDesc(A.a(stringObjMap("function", "lowercase")), "f5") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" order by f2 asc, [{\"function\":\"uca\",\"locale\":\"en_US\",\"strength\":\"IDENTICAL\"}]f3 asc, f4 desc, [{\"function\":\"lowercase\"}]f5 desc;"); + } + + @Test + void contains_sameElement() { + String q = Q.p("f1").containsSameElement(Q.p("stime").le(1).and("etime").gt(2)) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains sameElement(stime <= 1, etime > 2);"); + } + + @Test + void contains_phrase_near_onear_equiv() { + { + String q1 = Q.p("f1").containsPhrase("p1", "p2", "p3") + .semicolon() + .build(); + String q2 = Q.p("f1").containsPhrase(Arrays.asList("p1", "p2", "p3")) + .semicolon() + .build(); + assertEquals(q1, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\");"); + assertEquals(q2, "yql=select * from sources * where f1 contains phrase(\"p1\", \"p2\", \"p3\");"); + } + { + String q1 = Q.p("f1").containsNear("p1", "p2", "p3") + .semicolon() + .build(); + String q2 = Q.p("f1").containsNear(Arrays.asList("p1", "p2", "p3")) + .semicolon() + .build(); + assertEquals(q1, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\");"); + assertEquals(q2, "yql=select * from sources * where f1 contains near(\"p1\", \"p2\", \"p3\");"); + } + { + String q1 = Q.p("f1").containsOnear("p1", "p2", "p3") + .semicolon() + .build(); + String q2 = Q.p("f1").containsOnear(Arrays.asList("p1", "p2", "p3")) + .semicolon() + .build(); + assertEquals(q1, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\");"); + assertEquals(q2, "yql=select * from sources * where f1 contains onear(\"p1\", \"p2\", \"p3\");"); + } + { + String q1 = Q.p("f1").containsEquiv("p1", "p2", "p3") + .semicolon() + .build(); + String q2 = Q.p("f1").containsEquiv(Arrays.asList("p1", "p2", "p3")) + .semicolon() + .build(); + assertEquals(q1, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\");"); + assertEquals(q2, "yql=select * from sources * where f1 contains equiv(\"p1\", \"p2\", \"p3\");"); + } + } + + @Test + void contains_uri() { + String q = Q.p("f1").containsUri("https://test.uri") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains uri(\"https://test.uri\");"); + } + + @Test + void contains_uri_with_annotation() { + String q = Q.p("f1").containsUri(A.a("key", "value"), "https://test.uri") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains ([{\"key\":\"value\"}]uri(\"https://test.uri\"));"); + } + + @Test + void nearestNeighbor() { + String q = Q.p("f1").nearestNeighbor("query_vector") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where nearestNeighbor(f1, query_vector);"); + } + + @Test + void nearestNeighbor_with_annotation() { + String q = Q.p("f1").nearestNeighbor(A.a("targetHits", 10), "query_vector") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where ([{\"targetHits\":10}]nearestNeighbor(f1, query_vector));"); + } + + @Test + void use_contains_instead_of_contains_equiv_when_input_size_is_1() { + String q = Q.p("f1").containsEquiv(Collections.singletonList("p1")) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains \"p1\";"); + } + + @Test + void contains_phrase_near_onear_equiv_empty_list_should_throw_illegal_argument_exception() { + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsPhrase(Collections.emptyList()) + .semicolon() + .build()); + + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsNear(Collections.emptyList()) + .semicolon() + .build()); + + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsOnear(Collections.emptyList()) + .semicolon() + .build()); + + assertThrows(IllegalArgumentException.class, () -> Q.p("f1").containsEquiv(Collections.emptyList()) + .semicolon() + .build()); + } + + + @Test + void contains_near_onear_with_annotation() { + { + String q = Q.p("f1").containsNear(A.a("distance", 5), "p1", "p2", "p3") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains ([{\"distance\":5}]near(\"p1\", \"p2\", \"p3\"));"); + } + { + String q = Q.p("f1").containsOnear(A.a("distance", 5), "p1", "p2", "p3") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains ([{\"distance\":5}]onear(\"p1\", \"p2\", \"p3\"));"); + } + } + + @Test + void basic_group_syntax() { + /* + example from vespa document: + https://docs.vespa.ai/en/grouping.html + all( group(a) max(5) each(output(count()) + all(max(1) each(output(summary()))) + all(group(b) each(output(count()) + all(max(1) each(output(summary()))) + all(group(c) each(output(count()) + all(max(1) each(output(summary())))))))) ); + */ + String q = Q.p("f1").contains("v1") + .group( + G.all(G.group("a"), G.maxRtn(5), G.each(G.output(G.count()), + G.all(G.maxRtn(1), G.each(G.output(G.summary()))), + G.all(G.group("b"), G.each(G.output(G.count()), + G.all(G.maxRtn(1), G.each(G.output(G.summary()))), + G.all(G.group("c"), G.each(G.output(G.count()), + G.all(G.maxRtn(1), G.each(G.output(G.summary()))) + )) + )) + )) + ) + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))));"); + } + + @Test + void set_group_syntax_string_directly() { + /* + example from vespa document: + https://docs.vespa.ai/en/grouping.html + all( group(a) max(5) each(output(count()) + all(max(1) each(output(summary()))) + all(group(b) each(output(count()) + all(max(1) each(output(summary()))) + all(group(c) each(output(count()) + all(max(1) each(output(summary())))))))) ); + */ + String q = Q.p("f1").contains("v1") + .group("all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))))") + .semicolon() + .build(); + + assertEquals(q, "yql=select * from sources * where f1 contains \"v1\" | all(group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) all(max(1) each(output(summary()))) all(group(c) each(output(count()) all(max(1) each(output(summary())))))))));"); + } + +@Test + void arbitrary_annotations() { + Annotation a = A.a("a1", "v1", "a2", 2, "a3", stringObjMap("k", "v", "k2", 1), "a4", 4D, "a5", Arrays.asList(1, 2, 3)); + assertEquals(a.toString(), "{\"a1\":\"v1\",\"a2\":2,\"a3\":{\"k\":\"v\",\"k2\":1},\"a4\":4.0,\"a5\":[1,2,3]}"); + } + + @Test + void test_programmability() { + Map<String, String> map = stringStringMap("a", "1", "b", "2", "c", "3"); + + Query q = map + .entrySet() + .stream() + .map(entry -> Q.p(entry.getKey()).contains(entry.getValue())) + .reduce(Query::and) + .get(); + + assertEquals(q.semicolon().build(), "yql=select * from sources * where a contains \"1\" and b contains \"2\" and c contains \"3\";"); + } + + @Test + void test_programmability_2() { + Map<String, String> map = stringStringMap("a", "1", "b", "2", "c", "3"); + Query q = Q.p(); + + map.forEach((k, v) -> q.and(Q.p(k).contains(v))); + + assertEquals(q.semicolon().build(), "yql=select * from sources * where a contains \"1\" and b contains \"2\" and c contains \"3\";"); + } + + @Test + void empty_queries_should_not_print_out() { + String q = Q.p(Q.p(Q.p().andnot(Q.p()).and(Q.p()))).and("a").contains("1").semicolon().build(); + + assertEquals(q, "yql=select * from sources * where a contains \"1\";"); + } + + @Test + void validate_positive_search_term_of_strings() { + Query q = Q.p(Q.p("k1").contains("v1").and("k2").contains("v2").andnot("k3").contains("v3")) + .andnot(Q.p("nk1").contains("nv1").and("nk2").contains("nv2").andnot("nk3").contains("nv3")) + .and(Q.p("k4").contains("v4") + .andnot(Q.p("k5").contains("v5").andnot("k6").contains("v6")) + ); + + assertTrue(q.hasPositiveSearchField("k1")); + assertTrue(q.hasPositiveSearchField("k2")); + assertTrue(q.hasPositiveSearchField("nk3")); + assertTrue(q.hasPositiveSearchField("k6")); + assertTrue(q.hasPositiveSearchField("k6", "v6")); + assertFalse(q.hasPositiveSearchField("k6", "v5")); + + assertTrue(q.hasNegativeSearchField("k3")); + assertTrue(q.hasNegativeSearchField("nk1")); + assertTrue(q.hasNegativeSearchField("nk2")); + assertTrue(q.hasNegativeSearchField("k5")); + assertTrue(q.hasNegativeSearchField("k5", "v5")); + assertFalse(q.hasNegativeSearchField("k5", "v4")); + } + + @Test + void validate_positive_search_term_of_user_input() { + Query q = Q.p(Q.ui("k1", "v1")).and(Q.ui("k2", "v2")).andnot(Q.ui("k3", "v3")) + .andnot(Q.p(Q.ui("nk1", "nv1")).and(Q.ui("nk2", "nv2")).andnot(Q.ui("nk3", "nv3"))) + .and(Q.p(Q.ui("k4", "v4")) + .andnot(Q.p(Q.ui("k5", "v5")).andnot(Q.ui("k6", "v6"))) + ); + + assertTrue(q.hasPositiveSearchField("k1")); + assertTrue(q.hasPositiveSearchField("k2")); + assertTrue(q.hasPositiveSearchField("nk3")); + assertTrue(q.hasPositiveSearchField("k6")); + assertTrue(q.hasPositiveSearchField("k6", "v6")); + assertFalse(q.hasPositiveSearchField("k6", "v5")); + + assertTrue(q.hasNegativeSearchField("k3")); + assertTrue(q.hasNegativeSearchField("nk1")); + assertTrue(q.hasNegativeSearchField("nk2")); + assertTrue(q.hasNegativeSearchField("k5")); + assertTrue(q.hasNegativeSearchField("k5", "v5")); + assertFalse(q.hasNegativeSearchField("k5", "v4")); + } + + private static Map<String, Integer> stringIntMap(String k1, int v1, String k2, int v2, String k3, int v3) { + HashMap<String, Integer> m = new HashMap<>(); + m.put(k1, v1); + m.put(k2, v2); + m.put(k3, v3); + return m; + } + + private static Map<String, Object> stringObjMap(String k, Object v) { + HashMap<String, Object> m = new HashMap<>(); + m.put(k, v); + return m; + } + + private static Map<String, Object> stringObjMap(String k1, Object v1, String k2, Object v2) { + Map<String, Object> m = new LinkedHashMap<>(); + m.put(k1, v1); + m.put(k2, v2); + return m; + } + + private static Map<String, Object> stringObjMap(String k1, Object v1, String k2, Object v2, String k3, Object v3) { + Map<String, Object> m = new LinkedHashMap<>(); + m.put(k1, v1); + m.put(k2, v2); + m.put(k3, v3); + return m; + } + + private static Map<String, String> stringStringMap(String k1, String v1, String k2, String v2, String k3, String v3) { + Map<String, String> m = new LinkedHashMap<>(); + m.put(k1, v1); + m.put(k2, v2); + m.put(k3, v3); + return m; + } +}
\ No newline at end of file diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index e2cc7085353..d2ebb7ba9eb 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -32,8 +32,8 @@ <jaxb.version>2.3.0</jaxb.version> <jetty.version>9.4.44.v20210927</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> - <junit5.version>5.7.0</junit5.version> - <junit5.platform.version>1.7.0</junit5.platform.version> + <junit5.version>5.8.1</junit5.version> + <junit5.platform.version>1.8.1</junit5.platform.version> <onnxruntime.version>1.8.0</onnxruntime.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20090211</org.json.version> diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index 0154e5d3b13..08ec615e4c0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -4,6 +4,7 @@ package com.yahoo.config.model.api; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; import java.util.List; import java.util.Objects; @@ -17,9 +18,21 @@ import java.util.stream.Stream; * @author mortent */ public class ApplicationClusterEndpoint { + @Override + public String toString() { + return "ApplicationClusterEndpoint{" + + "dnsName=" + dnsName + + ", scope=" + scope + + ", routingMethod=" + routingMethod + + ", weight=" + weight + + ", hostNames=" + hostNames + + ", clusterId='" + clusterId + '\'' + + '}'; + } + public enum Scope {application, global, zone} - public enum RoutingMethod {shared, sharedLayer4} + public enum RoutingMethod {shared, sharedLayer4, exclusive} private final DnsName dnsName; private final Scope scope; @@ -99,6 +112,11 @@ public class ApplicationClusterEndpoint { return this; } + public Builder routingMethod(RoutingMethod routingMethod) { + this.routingMethod = routingMethod; + return this; + } + public Builder weight(int weigth) { this.weigth = weigth; return this; @@ -132,16 +150,25 @@ public class ApplicationClusterEndpoint { return name; } - // TODO: remove + // TODO: remove when 7.508 is latest version public static DnsName sharedNameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(cluster, applicationId) + return sharedNameFrom(SystemName.main, cluster, applicationId, suffix); + } + + public static DnsName sharedNameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { + String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" .collect(Collectors.joining("--")); return new DnsName(sanitize(name) + suffix); // Need to sanitize name since it is considered one label } + // TODO remove this method when 7.508 is latest version public static DnsName sharedL4NameFrom(ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { - String name = dnsParts(cluster, applicationId) + return sharedL4NameFrom(SystemName.main, cluster, applicationId, suffix); + } + + public static DnsName sharedL4NameFrom(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId, String suffix) { + String name = dnsParts(systemName, cluster, applicationId) .filter(Objects::nonNull) // remove null values that were "default" .map(DnsName::sanitize) .collect(Collectors.joining(".")); @@ -152,9 +179,10 @@ public class ApplicationClusterEndpoint { return new DnsName(name); } - private static Stream<String> dnsParts(ClusterSpec.Id cluster, ApplicationId applicationId) { + private static Stream<String> dnsParts(SystemName systemName, ClusterSpec.Id cluster, ApplicationId applicationId) { return Stream.of( nullIfDefault(cluster.value()), + systemPart(systemName), nullIfDefault(applicationId.instance().value()), applicationId.application().value(), applicationId.tenant().value() @@ -180,5 +208,16 @@ public class ApplicationClusterEndpoint { private static String nullIfDefault(String string) { return Optional.of(string).filter(s -> !s.equals("default")).orElse(null); } + + private static String systemPart(SystemName systemName) { + return "cd".equals(systemName.value()) ? systemName.value() : null; + } + + @Override + public String toString() { + return "DnsName{" + + "name='" + name + '\'' + + '}'; + } } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java index a114f9d40ef..78da750fb5b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java @@ -3,6 +3,9 @@ package com.yahoo.config.model.api; import java.util.List; import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.OptionalLong; /** * ContainerEndpoint tracks the service names that a Container Cluster should be @@ -16,11 +19,23 @@ public class ContainerEndpoint { private final String clusterId; private final ApplicationClusterEndpoint.Scope scope; private final List<String> names; + private final OptionalInt weight; + private final ApplicationClusterEndpoint.RoutingMethod routingMethod; public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names) { + this(clusterId, scope, names, OptionalInt.empty()); + } + + public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names, OptionalInt weight) { + this(clusterId, scope, names, weight, ApplicationClusterEndpoint.RoutingMethod.sharedLayer4); + } + + public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names, OptionalInt weight, ApplicationClusterEndpoint.RoutingMethod routingMethod) { this.clusterId = Objects.requireNonNull(clusterId); this.scope = Objects.requireNonNull(scope); this.names = List.copyOf(Objects.requireNonNull(names)); + this.weight = weight; + this.routingMethod = routingMethod; } public String clusterId() { @@ -35,6 +50,14 @@ public class ContainerEndpoint { return scope; } + public OptionalInt weight() { + return weight; + } + + public ApplicationClusterEndpoint.RoutingMethod routingMethod() { + return routingMethod; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -42,17 +65,18 @@ public class ContainerEndpoint { ContainerEndpoint that = (ContainerEndpoint) o; return Objects.equals(clusterId, that.clusterId) && Objects.equals(scope, that.scope) && - Objects.equals(names, that.names); + Objects.equals(names, that.names) && + Objects.equals(weight, that.weight) && + Objects.equals(routingMethod, that.routingMethod); } @Override public int hashCode() { - return Objects.hash(clusterId, names, scope); + return Objects.hash(clusterId, names, scope, weight, routingMethod); } @Override public String toString() { - return String.format("container endpoint %s -> %s [scope=%s]", clusterId, names, scope); + return String.format("container endpoint %s -> %s [scope=%s, weight=%s, routingMetod=%s]", clusterId, names, scope, weight, routingMethod); } - } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java index 2890f1cc019..04ef85856cd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java @@ -3,11 +3,11 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Ranking; import com.yahoo.searchdefinition.document.Sorting; @@ -69,8 +69,7 @@ public class AttributeFields extends Derived implements AttributesConfig.Produce private static boolean unsupportedFieldType(ImmutableSDField field) { return (field.usesStructOrMap() && !isSupportedComplexField(field) && - !field.getDataType().equals(PositionDataType.INSTANCE) && - !field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + !GeoPos.isAnyPos(field)); } /** Returns an attribute by name, or null if it doesn't exist */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java index a63b88f9445..3b8c0a9cff2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java @@ -2,9 +2,9 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; @@ -60,9 +60,8 @@ public class ImportedFields extends Derived implements ImportedFieldsConfig.Prod private static void considerComplexField(ImportedFieldsConfig.Builder builder, ImportedComplexField field) { ImmutableSDField targetField = field.targetField(); - if (targetField.getDataType().equals(PositionDataType.INSTANCE) || - targetField.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { - + if (GeoPos.isAnyPos(targetField)) { + // no action needed } else if (isArrayOfSimpleStruct(targetField)) { considerNestedFields(builder, field); } else if (isMapOfSimpleStruct(targetField)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 879ad570c26..495c3da5d3a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -6,7 +6,6 @@ import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.NumericDataType; -import com.yahoo.document.PositionDataType; import com.yahoo.document.PrimitiveDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.searchdefinition.Index; @@ -15,6 +14,7 @@ import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.BooleanIndexDefinition; import com.yahoo.searchdefinition.document.Case; import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.Stemming; @@ -91,12 +91,8 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } } - private static boolean isPositionArrayField(ImmutableSDField field) { - return field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE)); - } - private static boolean isPositionField(ImmutableSDField field) { - return field.getDataType().equals(PositionDataType.INSTANCE) || isPositionArrayField(field); + return GeoPos.isAnyPos(field); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java index cabe8d001bd..23409729dbb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java @@ -2,8 +2,8 @@ package com.yahoo.searchdefinition.derived; import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; @@ -58,9 +58,7 @@ public final class IndexingScript extends Derived implements IlscriptsConfig.Pro if (field.hasFullIndexingDocprocRights()) docFields.add(field.getName()); - if (field.usesStructOrMap() && - ! field.getDataType().equals(PositionDataType.INSTANCE) && - ! field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { + if (field.usesStructOrMap() && ! GeoPos.isAnyPos(field)) { return; // unsupported } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java index 4ce486e13ba..03b9e795317 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.derived; -import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; @@ -37,7 +37,7 @@ public class VsmSummary extends Derived implements VsmsummaryConfig.Producer { if (doMapField(schema, summaryField)) { SDField sdField = schema.getConcreteField(summaryField.getName()); - if (sdField != null && PositionDataType.INSTANCE.equals(sdField.getDataType())) { + if (sdField != null && GeoPos.isPos(sdField)) { summaryMap.put(summaryField, Collections.singletonList(summaryField.getName())); } else { summaryMap.put(summaryField, from); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java index 24a40154494..feac6b9618e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java @@ -59,8 +59,7 @@ public class ComplexAttributeFieldUtils { } private static boolean isStructWithPrimitiveStructFieldAttributes(DataType type, ImmutableSDField field) { - if (type instanceof StructDataType && - !(type.equals(PositionDataType.INSTANCE))) { + if (type instanceof StructDataType && ! GeoPos.isPos(type)) { for (ImmutableSDField structField : field.getStructFields()) { Attribute attribute = structField.getAttributes().get(structField.getName()); if (attribute != null) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java new file mode 100644 index 00000000000..956d63a1cdf --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; + +/** + * Common utilities for recognizing fields with the built-in "position" datatype, + * possibly in array form. + * @author arnej + */ +public class GeoPos { + static public boolean isPos(DataType type) { + return PositionDataType.INSTANCE.equals(type); + } + static public boolean isPosArray(DataType type) { + return DataType.getArray(PositionDataType.INSTANCE).equals(type); + } + static public boolean isAnyPos(DataType type) { + return isPos(type) || isPosArray(type); + } + + static public boolean isPos(ImmutableSDField field) { return isPos(field.getDataType()); } + static public boolean isPosArray(ImmutableSDField field) { return isPosArray(field.getDataType()); } + static public boolean isAnyPos(ImmutableSDField field) { return isAnyPos(field.getDataType()); } +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java index 983942f87c3..766b6ed3fec 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java @@ -8,6 +8,7 @@ import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; @@ -34,7 +35,7 @@ public class AdjustPositionSummaryFields extends Processor { private void scanSummary(DocumentSummary summary) { for (SummaryField summaryField : summary.getSummaryFields().values()) { - if ( ! isPositionDataType(summaryField.getDataType())) continue; + if ( ! GeoPos.isAnyPos(summaryField.getDataType())) continue; String originalSource = summaryField.getSingleSource(); if (originalSource.indexOf('.') == -1) { // Eliminate summary fields with pos.x or pos.y as source @@ -112,10 +113,6 @@ public class AdjustPositionSummaryFields extends Processor { return name.length() > suffix.length() && name.substring(name.length() - suffix.length()).equals(suffix); } - private static boolean isPositionDataType(DataType dataType) { - return dataType.equals(PositionDataType.INSTANCE) || dataType.equals(DataType.getArray(PositionDataType.INSTANCE)); - } - private static DataType makeZCurveDataType(DataType dataType) { return dataType instanceof ArrayDataType ? DataType.getArray(DataType.LONG) : DataType.LONG; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java index 699abb1e792..f5c1d8d8197 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java @@ -8,6 +8,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -142,10 +143,7 @@ public class CreatePositionZCurve extends Processor { } private static boolean isSupportedPositionType(DataType dataType) { - if (dataType instanceof ArrayDataType) { - dataType = ((ArrayDataType)dataType).getNestedType(); - } - return dataType.equals(PositionDataType.INSTANCE); + return GeoPos.isAnyPos(dataType); } private static class RemoveSummary extends ExpressionConverter { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java index f8a28061897..e836caac10d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java @@ -9,6 +9,7 @@ import com.yahoo.searchdefinition.DocumentReferences; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.ImportedComplexField; import com.yahoo.searchdefinition.document.ImportedField; @@ -49,8 +50,7 @@ public class ImportedFieldsResolver extends Processor { private void resolveImportedField(TemporaryImportedField importedField, boolean validate) { DocumentReference reference = validateDocumentReference(importedField); ImmutableSDField targetField = getTargetField(importedField, reference); - if (targetField.getDataType().equals(PositionDataType.INSTANCE) || - targetField.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))) { + if (GeoPos.isAnyPos(targetField)) { resolveImportedPositionField(importedField, reference, targetField, validate); } else if (isArrayOfSimpleStruct(targetField)) { resolveImportedArrayOfStructField(importedField, reference, targetField, validate); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java index eb9b561da73..242f5dab308 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java @@ -5,11 +5,11 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; -import com.yahoo.document.PositionDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; @@ -153,7 +153,7 @@ public class IndexingValidation extends Processor { createCompatType(mapType.getValueType())); } else if (origType instanceof WeightedSetDataType) { return DataType.getWeightedSet(createCompatType(((WeightedSetDataType)origType).getNestedType())); - } else if (origType == PositionDataType.INSTANCE) { + } else if (GeoPos.isPos(origType)) { return DataType.LONG; } else { return origType; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 5c21f3f5a8d..973f7f5cc40 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -165,7 +165,7 @@ public class VespaMetricSet { metrics.add(new Metric("httpapi_succeeded.rate")); metrics.add(new Metric("httpapi_failed.rate")); metrics.add(new Metric("httpapi_parse_error.rate")); - addMetric(metrics, "httpapi_test_and_set_condition_not_met", List.of("rate")); + addMetric(metrics, "httpapi_condition_not_met", List.of("rate")); metrics.add(new Metric("mem.heap.total.average")); metrics.add(new Metric("mem.heap.free.average")); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index 43bf8133c74..e2b08a621d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.DataType; -import com.yahoo.document.PositionDataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; +import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; @@ -62,8 +62,7 @@ public class ComplexAttributeFieldsValidator extends Validator { private static boolean isSupportedComplexField(ImmutableSDField field) { return (ComplexAttributeFieldUtils.isSupportedComplexField(field) || - field.getDataType().equals(PositionDataType.INSTANCE) || - field.getDataType().equals(DataType.getArray(PositionDataType.INSTANCE))); + GeoPos.isAnyPos(field)); } private static String toString(ImmutableSDField field) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index f7e8afc2d94..df617bf3eed 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -51,6 +51,9 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; + /** * A container cluster that is typically set up from the user application. * @@ -210,6 +213,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat for(String suffix : deployState.getProperties().zoneDnsSuffixes()) { // L4 ApplicationClusterEndpoint.DnsName l4Name = ApplicationClusterEndpoint.DnsName.sharedL4NameFrom( + deployState.zone().system(), ClusterSpec.Id.from(getName()), deployState.getProperties().applicationId(), suffix); @@ -223,6 +227,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat // L7 ApplicationClusterEndpoint.DnsName l7Name = ApplicationClusterEndpoint.DnsName.sharedNameFrom( + deployState.zone().system(), ClusterSpec.Id.from(getName()), deployState.getProperties().applicationId(), suffix); @@ -235,14 +240,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .build()); } - // Then get all endpoints provided by controller. Can be created with L4 routing only + // Then get all endpoints provided by controller. + Set<ApplicationClusterEndpoint.RoutingMethod> supportedRoutingMethods = Set.of(shared, sharedLayer4); Set<ContainerEndpoint> endpointsFromController = deployState.getEndpoints(); endpointsFromController.stream() .filter(ce -> ce.clusterId().equals(getName())) + .filter(ce -> supportedRoutingMethods.contains(ce.routingMethod())) .forEach(ce -> ce.names().forEach( name -> endpoints.add(ApplicationClusterEndpoint.builder() .scope(ce.scope()) - .sharedL4Routing() + .weight(Long.valueOf(ce.weight().orElse(1)).intValue()) // Default to weight=1 if not set + .routingMethod(ce.routingMethod()) .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) .hosts(hosts) .clusterId(getName()) diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 2016cea02a9..560ac28b6f7 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -39,9 +39,18 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Objects; +import java.util.OptionalInt; +import java.util.OptionalLong; import java.util.Set; import java.util.stream.Collectors; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.exclusive; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.shared; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.application; +import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global; +import static com.yahoo.config.provision.SystemName.cd; +import static com.yahoo.config.provision.SystemName.main; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.hasKey; @@ -358,32 +367,78 @@ public class ContainerClusterTest { @Test public void generatesCorrectRoutingInfo() { + // main system: + assertNames(main, + ApplicationId.from("t1", "a1", "i1"), + Set.of(), + List.of("search-cluster.i1.a1.t1.endpoint.suffix"), + List.of("search-cluster--i1--a1--t1.endpoint.suffix")); + + assertNames(main, + ApplicationId.from("t1", "a1", "default"), + Set.of(), + List.of("search-cluster.a1.t1.endpoint.suffix"), + List.of("search-cluster--a1--t1.endpoint.suffix")); - assertNames(ApplicationId.from("t1", "a1", "i1"), + assertNames(main, + ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.i1.a1.t1.endpoint.suffix", "search-cluster--i1--a1--t1.endpoint.suffix")); + List.of("search-cluster.default.t1.endpoint.suffix"), + List.of("search-cluster--default--t1.endpoint.suffix")); + + assertNames(main, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + List.of("search-cluster.a1.t1.endpoint.suffix"), + List.of("search-cluster--a1--t1.endpoint.suffix")); + + assertNames(main, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4)), + List.of("search-cluster.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), + List.of("search-cluster--a1--t1.endpoint.suffix")); + + // cd system: + assertNames(cd, + ApplicationId.from("t1", "a1", "i1"), + Set.of(), + List.of("search-cluster.cd.i1.a1.t1.endpoint.suffix"), + List.of("search-cluster--cd--i1--a1--t1.endpoint.suffix")); - assertNames(ApplicationId.from("t1", "a1", "default"), + assertNames(cd, + ApplicationId.from("t1", "a1", "default"), Set.of(), - List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix")); + List.of("search-cluster.cd.a1.t1.endpoint.suffix"), + List.of("search-cluster--cd--a1--t1.endpoint.suffix")); - assertNames(ApplicationId.from("t1", "default", "default"), + assertNames(cd, + ApplicationId.from("t1", "default", "default"), Set.of(), - List.of("search-cluster.default.t1.endpoint.suffix", "search-cluster--default--t1.endpoint.suffix")); + List.of("search-cluster.cd.default.t1.endpoint.suffix"), + List.of("search-cluster--cd--default--t1.endpoint.suffix")); + + assertNames(cd, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("not-in-this-cluster", global, List.of("foo", "bar"))), + List.of("search-cluster.cd.a1.t1.endpoint.suffix"), + List.of("search-cluster--cd--a1--t1.endpoint.suffix")); - assertNames(ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("not-in-this-cluster", ApplicationClusterEndpoint.Scope.global, List.of("foo", "bar"))), - List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix")); + assertNames(cd, + ApplicationId.from("t1", "a1", "default"), + Set.of(new ContainerEndpoint("search-cluster", global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), sharedLayer4), + new ContainerEndpoint("search-cluster", global, List.of("a--b.x.y.z", "rotation-2.x.y.z"), OptionalInt.empty(), shared), + new ContainerEndpoint("search-cluster", application, List.of("app-rotation.x.y.z"), OptionalInt.of(3), sharedLayer4), + new ContainerEndpoint("not-supported", global, List.of("not.supported"), OptionalInt.empty(), exclusive)), + List.of("search-cluster.cd.a1.t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z"), + List.of("search-cluster--cd--a1--t1.endpoint.suffix", "a--b.x.y.z", "rotation-2.x.y.z")); - assertNames(ApplicationId.from("t1", "a1", "default"), - Set.of(new ContainerEndpoint("search-cluster", ApplicationClusterEndpoint.Scope.global, List.of("rotation-1.x.y.z", "rotation-2.x.y.z")), - new ContainerEndpoint("search-cluster", ApplicationClusterEndpoint.Scope.application, List.of("app-rotation.x.y.z"))), - List.of("search-cluster.a1.t1.endpoint.suffix", "search-cluster--a1--t1.endpoint.suffix", "rotation-1.x.y.z", "rotation-2.x.y.z", "app-rotation.x.y.z")); } - private void assertNames(ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedNames) { + private void assertNames(SystemName systemName, ApplicationId appId, Set<ContainerEndpoint> globalEndpoints, List<String> expectedSharedL4Names, List<String> expectedSharedNames) { + Zone zone = new Zone(systemName, Environment.defaultEnvironment(), RegionName.defaultName()); DeployState state = new DeployState.Builder() - .zone(Zone.defaultZone()) + .zone(zone) .endpoints(globalEndpoints) .properties(new TestProperties() .setHostedVespa(true) @@ -395,8 +450,26 @@ public class ContainerClusterTest { addContainer(root, cluster, "c1", "host-c1"); cluster.doPrepare(state); List<ApplicationClusterEndpoint> endpoints = cluster.endpoints(); + + assertNames(expectedSharedNames, endpoints.stream().filter(e -> e.routingMethod() == shared).collect(Collectors.toList())); + assertNames(expectedSharedL4Names, endpoints.stream().filter(e -> e.routingMethod() == sharedLayer4).collect(Collectors.toList())); + + List<ContainerEndpoint> endpointsWithWeight = + globalEndpoints.stream().filter(endpoint -> endpoint.weight().isPresent()).collect(Collectors.toList()); + endpointsWithWeight.stream() + .filter(ce -> ce.weight().isPresent()) + .forEach(ce -> assertTrue(endpointsMatch(ce, endpoints))); + } + + private void assertNames(List<String> expectedNames, List<ApplicationClusterEndpoint> endpoints) { assertEquals(expectedNames.size(), endpoints.size()); - expectedNames.forEach(expected -> assertTrue("Endpoint not matched " + expected, endpoints.stream().anyMatch(e -> Objects.equals(e.dnsName().value(), expected)))); + expectedNames.forEach(expected -> assertTrue("Endpoint not matched " + expected + " was: " + endpoints, endpoints.stream().anyMatch(e -> Objects.equals(e.dnsName().value(), expected)))); + } + + private boolean endpointsMatch(ContainerEndpoint configuredEndpoint, List<ApplicationClusterEndpoint> clusterEndpoints) { + return clusterEndpoints.stream().anyMatch(e -> + configuredEndpoint.names().contains(e.dnsName().value()) && + configuredEndpoint.weight().getAsInt() == e.weight()); } private void verifyTesterApplicationInstalledBundles(Zone zone, List<String> expectedBundleNames) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java index f2f52dca9fa..625f1b5fe17 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionAndUrlDownload.java @@ -4,7 +4,9 @@ package com.yahoo.vespa.config.proxy.filedistribution; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.subscription.ConfigSourceSet; import com.yahoo.jrt.Supervisor; +import com.yahoo.vespa.config.ConnectionPool; import com.yahoo.vespa.config.JRTConnectionPool; +import com.yahoo.vespa.filedistribution.FileDistributionConnectionPool; import com.yahoo.vespa.filedistribution.FileDownloader; import java.time.Duration; @@ -20,6 +22,7 @@ import java.util.concurrent.TimeUnit; public class FileDistributionAndUrlDownload { private static final Duration delay = Duration.ofMinutes(1); + private final FileDistributionRpcServer fileDistributionRpcServer; private final UrlDownloadRpcServer urlDownloadRpcServer; private final ScheduledExecutorService cleanupExecutor = @@ -28,7 +31,7 @@ public class FileDistributionAndUrlDownload { public FileDistributionAndUrlDownload(Supervisor supervisor, ConfigSourceSet source) { fileDistributionRpcServer = new FileDistributionRpcServer(supervisor, - new FileDownloader(new JRTConnectionPool(source, supervisor), supervisor, Duration.ofMinutes(5))); + new FileDownloader(createConnectionPool(supervisor, source), supervisor, Duration.ofMinutes(5))); urlDownloadRpcServer = new UrlDownloadRpcServer(supervisor); cleanupExecutor.scheduleAtFixedRate(new CachedFilesMaintainer(), delay.toSeconds(), delay.toSeconds(), TimeUnit.SECONDS); } @@ -45,4 +48,12 @@ public class FileDistributionAndUrlDownload { } } + private static ConnectionPool createConnectionPool(Supervisor supervisor, ConfigSourceSet source) { + String useFileDistributionConnectionPool = System.getenv("VESPA_CONFIG_PROXY_USE_FILE_DISTRIBUTION_CONNECTION_POOL"); + if (useFileDistributionConnectionPool != null && useFileDistributionConnectionPool.equalsIgnoreCase("true")) + return new FileDistributionConnectionPool(source, supervisor); + else + return new JRTConnectionPool(source, supervisor); + } + } diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java index 8b9d1f34154..dfbd605ab50 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileDistributionRpcServer.java @@ -9,6 +9,7 @@ import com.yahoo.jrt.Request; import com.yahoo.jrt.StringArray; import com.yahoo.jrt.StringValue; import com.yahoo.jrt.Supervisor; +import com.yahoo.net.HostName; import com.yahoo.vespa.filedistribution.FileDownloader; import java.io.File; @@ -101,7 +102,7 @@ class FileDistributionRpcServer { private void downloadFile(Request req) { FileReference fileReference = new FileReference(req.parameters().get(0).asString()); log.log(Level.FINE, () -> "getFile() called for file reference '" + fileReference.value() + "'"); - Optional<File> file = downloader.getFile(fileReference); + Optional<File> file = downloader.getFile(fileReference, HostName.getLocalhost()); if (file.isPresent()) { new RequestTracker().trackRequest(file.get().getParentFile()); req.returnValues().add(new StringValue(file.get().getAbsolutePath())); diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java index a3265671d50..d107d1e30b5 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/ConfigSubscription.java @@ -329,6 +329,8 @@ public abstract class ConfigSubscription<T extends ConfigInstance> { state = State.CLOSED; } + public boolean isClosed() { return state == State.CLOSED; } + State getState() { return state; } diff --git a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java index 4c1d7b39755..0c4ac005bb6 100644 --- a/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java +++ b/config/src/main/java/com/yahoo/config/subscription/impl/JRTConfigRequester.java @@ -29,7 +29,7 @@ import static java.util.logging.Level.SEVERE; import static java.util.logging.Level.WARNING; /** - * This class fetches config payload using JRT, and acts as the callback target. + * Requests configs using RPC, and acts as the callback target. * It uses the {@link JRTConfigSubscription} and {@link JRTClientConfigRequest} * as context, and puts the request objects on a queue on the subscription, * for handling by the user thread. @@ -53,8 +53,9 @@ public class JRTConfigRequester implements RequestWaiter { private final ConnectionPool connectionPool; private final ConfigSourceSet configSourceSet; - private Instant noApplicationWarningLogged = Instant.MIN; + private Instant timeForLastLogWarning; private int failures = 0; + private volatile boolean closed = false; /** * Returns a new requester @@ -68,6 +69,8 @@ public class JRTConfigRequester implements RequestWaiter { this.scheduler = scheduler; this.connectionPool = connectionPool; this.timingValues = timingValues; + // Adjust so that we wait 1 second with logging warning in case there are some errors just when starting up + timeForLastLogWarning = Instant.now().minus(delayBetweenWarnings.plus(Duration.ofSeconds(1))); } /** @@ -93,14 +96,15 @@ public class JRTConfigRequester implements RequestWaiter { private <T extends ConfigInstance> void doRequest(JRTConfigSubscription<T> sub, JRTClientConfigRequest req) { Connection connection = connectionPool.getCurrent(); - req.getRequest().setContext(new RequestContext(sub, req, connection)); + Request request = req.getRequest(); + request.setContext(new RequestContext(sub, req, connection)); if (!req.validateParameters()) throw new ConfigurationRuntimeException("Error in parameters for config request: " + req); double jrtClientTimeout = getClientTimeout(req); log.log(FINE, () -> "Requesting config for " + sub + " on connection " + connection + " with client timeout " + jrtClientTimeout + (log.isLoggable(FINEST) ? (",defcontent=" + req.getDefContent().asString()) : "")); - connection.invokeAsync(req.getRequest(), jrtClientTimeout, this); + connection.invokeAsync(request, jrtClientTimeout, this); } @SuppressWarnings("unchecked") @@ -124,7 +128,7 @@ public class JRTConfigRequester implements RequestWaiter { } private void doHandle(JRTConfigSubscription<ConfigInstance> sub, JRTClientConfigRequest jrtReq, Connection connection) { - if (subscriptionIsClosed(sub)) return; // Avoid error messages etc. after closing + if (sub.isClosed()) return; // Avoid error messages etc. after closing boolean validResponse = jrtReq.validateResponse(); log.log(FINE, () -> "Request callback " + (validResponse ? "valid" : "invalid") + ". Req: " + jrtReq + "\nSpec: " + connection); @@ -145,12 +149,7 @@ public class JRTConfigRequester implements RequestWaiter { break; case ErrorCode.APPLICATION_NOT_LOADED: case ErrorCode.UNKNOWN_VESPA_VERSION: - if (noApplicationWarningLogged.isBefore(Instant.now().minus(delayBetweenWarnings))) { - log.log(WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + - ". Connection spec: " + connection.getAddress() + - ", error message: " + jrtReq.errorMessage()); - noApplicationWarningLogged = Instant.now(); - } + logWarning(jrtReq, connection); break; default: log.log(WARNING, "Request callback failed. Req: " + jrtReq + "\nSpec: " + connection.getAddress() + @@ -159,6 +158,15 @@ public class JRTConfigRequester implements RequestWaiter { } } + private void logWarning(JRTClientConfigRequest jrtReq, Connection connection) { + if ( ! closed && timeForLastLogWarning.isBefore(Instant.now().minus(delayBetweenWarnings))) { + log.log(WARNING, "Request callback failed: " + ErrorCode.getName(jrtReq.errorCode()) + + ". Connection spec: " + connection.getAddress() + + ", error message: " + jrtReq.errorMessage()); + timeForLastLogWarning = Instant.now(); + } + } + private void handleFailedRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<ConfigInstance> sub, Connection connection) { logError(jrtReq, connection); @@ -190,7 +198,6 @@ public class JRTConfigRequester implements RequestWaiter { private void handleOKRequest(JRTClientConfigRequest jrtReq, JRTConfigSubscription<ConfigInstance> sub) { failures = 0; - noApplicationWarningLogged = Instant.MIN; sub.setLastCallBackOKTS(Instant.now()); log.log(FINE, () -> "OK response received in handleOkRequest: " + jrtReq); if (jrtReq.hasUpdatedGeneration()) { @@ -199,10 +206,6 @@ public class JRTConfigRequester implements RequestWaiter { scheduleNextRequest(jrtReq, sub, calculateSuccessDelay(), calculateSuccessTimeout()); } - private boolean subscriptionIsClosed(JRTConfigSubscription<ConfigInstance> sub) { - return sub.getState() == ConfigSubscription.State.CLOSED; - } - private long calculateSuccessTimeout() { return timingValues.getPlusMinusFractionRandom(timingValues.getSuccessTimeout(), randomFraction); } @@ -237,8 +240,7 @@ public class JRTConfigRequester implements RequestWaiter { } public void close() { - // Fake that we have logged to avoid printing warnings after this - noApplicationWarningLogged = Instant.now(); + closed = true; if (configSourceSet != null) { managedPool.release(configSourceSet); } diff --git a/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java index c1fc50f6a82..f731d49941a 100644 --- a/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java +++ b/config/src/main/java/com/yahoo/vespa/config/JRTConnection.java @@ -53,6 +53,7 @@ public class JRTConnection implements Connection { if (target == null || !target.isValid()) { logger.log(Level.INFO, "Connecting to " + address); target = supervisor.connect(new Spec(address)); + logger.log(Level.FINE, "Connected to " + address); } return target; } diff --git a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java index ea73a6cbef1..270c618ee1b 100644 --- a/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java +++ b/config/src/test/java/com/yahoo/config/subscription/ConfigSubscriptionTest.java @@ -3,19 +3,21 @@ package com.yahoo.config.subscription; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigurationRuntimeException; -import com.yahoo.foo.SimpletypesConfig; -import com.yahoo.foo.AppConfig; import com.yahoo.config.subscription.impl.ConfigSubscription; +import com.yahoo.foo.AppConfig; +import com.yahoo.foo.SimpletypesConfig; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.TimingValues; - -import org.junit.Ignore; import org.junit.Test; import java.util.Collections; import java.util.List; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author hmusum @@ -26,13 +28,10 @@ public class ConfigSubscriptionTest { @Test public void testEquals() { ConfigSubscriber sub = new ConfigSubscriber(); - final String payload = "boolval true"; - ConfigSubscription<SimpletypesConfig> a = ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, "test"), - sub, new RawSource(payload), new TimingValues()); - ConfigSubscription<SimpletypesConfig> b = ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, "test"), - sub, new RawSource(payload), new TimingValues()); - ConfigSubscription<SimpletypesConfig> c = ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, "test2"), - sub, new RawSource(payload), new TimingValues()); + + ConfigSubscription<SimpletypesConfig> a = createSubscription(sub, "test"); + ConfigSubscription<SimpletypesConfig> b = createSubscription(sub, "test"); + ConfigSubscription<SimpletypesConfig> c = createSubscription(sub, "test2"); assertEquals(b, a); assertEquals(a, a); assertEquals(b, b); @@ -68,16 +67,13 @@ public class ConfigSubscriptionTest { ConfigSubscriber sub = new ConfigSubscriber(); ConfigHandle<SimpletypesConfig> handle = sub.subscribe(SimpletypesConfig.class, "raw:boolval true", 10000); assertNotNull(handle); - sub.nextConfig(false); + assertTrue(sub.nextConfig(false)); assertTrue(handle.getConfig().boolval()); - //assertTrue(sub.getSource() instanceof RawSource); sub.close(); } - // Test that subscription is closed and subscriptionHandles is empty if we get an exception - // (only the last is possible to test right now). + // Test that exception is thrown if subscribe fails and that subscription is closed if we close the subscriber @Test - @Ignore public void testSubscribeWithException() { TestConfigSubscriber sub = new TestConfigSubscriber(); ConfigSourceSet configSourceSet = new ConfigSourceSet(Collections.singletonList("tcp/localhost:99999")); @@ -85,10 +81,16 @@ public class ConfigSubscriptionTest { sub.subscribe(SimpletypesConfig.class, "configid", configSourceSet, new TimingValues().setSubscribeTimeout(100)); fail(); } catch (ConfigurationRuntimeException e) { - assertEquals(0, sub.getSubscriptionHandles().size()); + sub.close(); + assertTrue(sub.getSubscriptionHandles().get(0).subscription().isClosed()); } } + private ConfigSubscription<SimpletypesConfig> createSubscription(ConfigSubscriber sub, String configId) { + return ConfigSubscription.get(new ConfigKey<>(SimpletypesConfig.class, configId), + sub, new RawSource("boolval true"), new TimingValues()); + } + private static class TestConfigSubscriber extends ConfigSubscriber { List<ConfigHandle<? extends ConfigInstance>> getSubscriptionHandles() { return subscriptionHandles; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java index f4801c5a7ea..02fad2357c3 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileServer.java @@ -103,7 +103,7 @@ public class FileServer { try { return root.getFile(reference).exists(); } catch (IllegalArgumentException e) { - log.log(Level.FINE, () -> "Failed locating file reference '" + reference + "' with error " + e.toString()); + log.log(Level.FINE, () -> "Failed locating " + reference + ": " + e.getMessage()); } return false; } @@ -121,7 +121,7 @@ public class FileServer { private void serveFile(FileReference reference, Receiver target) { File file = root.getFile(reference); - log.log(Level.FINE, () -> "Start serving reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, () -> "Start serving " + reference + " with file '" + file.getAbsolutePath() + "'"); boolean success = false; String errorDescription = "OK"; FileReferenceData fileData = EmptyFileReferenceData.empty(reference, file.getName()); @@ -129,15 +129,15 @@ public class FileServer { fileData = readFileReferenceData(reference); success = true; } catch (IOException e) { - errorDescription = "For file reference '" + reference.value() + "': failed reading file '" + file.getAbsolutePath() + "'"; + errorDescription = "For" + reference.value() + ": failed reading file '" + file.getAbsolutePath() + "'"; log.warning(errorDescription + " for sending to '" + target.toString() + "'. " + e.toString()); } try { target.receive(fileData, new ReplayStatus(success ? 0 : 1, success ? "OK" : errorDescription)); - log.log(Level.FINE, () -> "Done serving file reference '" + reference.value() + "' with file '" + file.getAbsolutePath() + "'"); + log.log(Level.FINE, () -> "Done serving " + reference.value() + " with file '" + file.getAbsolutePath() + "'"); } catch (Exception e) { - log.log(Level.WARNING, "Failed serving file reference '" + reference.value() + "': " + Exceptions.toMessageString(e)); + log.log(Level.WARNING, "Failed serving " + reference + ": " + Exceptions.toMessageString(e)); } finally { fileData.close(); } @@ -157,12 +157,12 @@ public class FileServer { public void serveFile(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { if (executor instanceof ThreadPoolExecutor) - log.log(Level.FINE, () -> "Active threads is now " + ((ThreadPoolExecutor) executor).getActiveCount()); + log.log(Level.FINE, () -> "Active threads: " + ((ThreadPoolExecutor) executor).getActiveCount()); executor.execute(() -> serveFileInternal(fileReference, downloadFromOtherSourceIfNotFound, request, receiver)); } private void serveFileInternal(String fileReference, boolean downloadFromOtherSourceIfNotFound, Request request, Receiver receiver) { - log.log(Level.FINE, () -> "Received request for reference '" + fileReference + "' from " + request.target()); + log.log(Level.FINE, () -> "Received request for file reference '" + fileReference + "' from " + request.target()); boolean fileExists; try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java new file mode 100644 index 00000000000..bc44093b89a --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandler.java @@ -0,0 +1,277 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v1; + +import com.google.inject.Inject; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.jdisc.http.HttpRequest; +import com.yahoo.path.Path; +import com.yahoo.restapi.RestApi; +import com.yahoo.restapi.RestApiException; +import com.yahoo.restapi.RestApiRequestHandler; +import com.yahoo.restapi.SlimeJsonResponse; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.yolean.Exceptions; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * This implements the /routing/v1/status REST API on the config server, providing explicit control over the routing + * status of a deployment or zone (all deployments). The routing status manipulated by this is only respected by the + * shared routing layer. + * + * @author bjorncs + * @author mpolden + */ +public class RoutingStatusApiHandler extends RestApiRequestHandler<RoutingStatusApiHandler> { + + private static final Logger log = Logger.getLogger(RoutingStatusApiHandler.class.getName()); + + private static final Path ROUTING_ROOT = Path.fromString("/routing/v1/"); + private static final Path DEPLOYMENT_STATUS_ROOT = ROUTING_ROOT.append("status"); + private static final Path ZONE_STATUS_ROOT = ROUTING_ROOT.append("zone-inactive"); + + private final Curator curator; + private final Clock clock; + private final Deployer deployer; + + @Inject + public RoutingStatusApiHandler(Context context, Curator curator, Deployer deployer) { + this(context, curator, Clock.systemUTC(), deployer); + } + + RoutingStatusApiHandler(Context context, Curator curator, Clock clock, Deployer deployer) { + super(context, RoutingStatusApiHandler::createRestApiDefinition); + this.curator = Objects.requireNonNull(curator); + this.clock = Objects.requireNonNull(clock); + this.deployer = Objects.requireNonNull(deployer); + } + + private static RestApi createRestApiDefinition(RoutingStatusApiHandler self) { + return RestApi.builder() + .addRoute(RestApi.route("/routing/v1/status") + .get(self::listInactiveDeployments)) + .addRoute(RestApi.route("/routing/v1/status/zone") + .get(self::zoneStatus) + .put(self::changeZoneStatus) + .delete(self::changeZoneStatus)) + .addRoute(RestApi.route("/routing/v1/status/{upstreamName}") + .get(self::getDeploymentStatus) + .put(self::changeDeploymentStatus)) + .build(); + } + + /** Get upstream of all deployments with status OUT */ + private SlimeJsonResponse listInactiveDeployments(RestApi.RequestContext context) { + List<String> inactiveDeployments = curator.getChildren(DEPLOYMENT_STATUS_ROOT).stream() + .filter(upstreamName -> deploymentStatus(upstreamName).status() == RoutingStatus.out) + .collect(Collectors.toUnmodifiableList()); + Slime slime = new Slime(); + Cursor rootArray = slime.setArray(); + inactiveDeployments.forEach(rootArray::addString); + return new SlimeJsonResponse(slime); + } + + /** Get the routing status of a deployment */ + private SlimeJsonResponse getDeploymentStatus(RestApi.RequestContext context) { + String upstreamName = upstreamName(context); + DeploymentRoutingStatus deploymentRoutingStatus = deploymentStatus(upstreamName); + // If the entire zone is out, we always return OUT regardless of the actual routing status + if (zoneStatus() == RoutingStatus.out) { + String reason = String.format("Rotation is OUT because the zone is OUT (actual deployment status is %s)", + deploymentRoutingStatus.status().name().toUpperCase(Locale.ENGLISH)); + deploymentRoutingStatus = new DeploymentRoutingStatus(RoutingStatus.out, "operator", reason, + clock.instant()); + } + return new SlimeJsonResponse(toSlime(deploymentRoutingStatus)); + } + + /** Change routing status of a deployment */ + private SlimeJsonResponse changeDeploymentStatus(RestApi.RequestContext context) { + String upstreamName = upstreamName(context); + ApplicationId instance = instance(context); + Path path = deploymentStatusPath(upstreamName); + + RestApi.RequestContext.RequestContent requestContent = context.requestContentOrThrow(); + Slime requestBody = Exceptions.uncheck(() -> SlimeUtils.jsonToSlime(requestContent.content().readAllBytes())); + DeploymentRoutingStatus wantedStatus = deploymentRoutingStatusFromSlime(requestBody, clock.instant()); + DeploymentRoutingStatus currentStatus = deploymentStatus(upstreamName); + if (wantedStatus.status() == currentStatus.status()) { // No change + return new SlimeJsonResponse(toSlime(currentStatus)); + } + + // Redeploy application so that a new LbServicesConfig containing the updated status is generated and consumed + // by routing layer. This is required to update weights for application endpoints when routing status for a + // deployment is changed + curator.set(path, toJsonBytes(wantedStatus)); + try { + deployer.deployFromLocalActive(instance, Duration.ofMinutes(1)); + } catch (Exception e) { + log.log(Level.SEVERE, "Failed to redeploy " + instance + ". Reverting routing status to " + + currentStatus.status(), e); + curator.set(path, toJsonBytes(currentStatus)); + throw new RestApiException.InternalServerError("Failed to change status to " + + wantedStatus.status() + ", reverting to " + + currentStatus.status() + + " because redeployment of " + + instance + " failed: " + + Exceptions.toMessageString(e)); + } + return new SlimeJsonResponse(toSlime(wantedStatus)); + } + + /** Change routing status of a zone */ + private SlimeJsonResponse changeZoneStatus(RestApi.RequestContext context) { + boolean in = context.request().getMethod() == HttpRequest.Method.DELETE; + if (in) { + curator.delete(ZONE_STATUS_ROOT); + return new SlimeJsonResponse(toSlime(RoutingStatus.in)); + } else { + curator.create(ZONE_STATUS_ROOT); + return new SlimeJsonResponse(toSlime(RoutingStatus.out)); + } + } + + /** Read the status for zone */ + private SlimeJsonResponse zoneStatus(RestApi.RequestContext context) { + return new SlimeJsonResponse(toSlime(zoneStatus())); + } + + /** Read the status for a deployment */ + private DeploymentRoutingStatus deploymentStatus(String upstreamName) { + Instant changedAt = clock.instant(); + Path path = deploymentStatusPath(upstreamName); + Optional<byte[]> data = curator.getData(path); + if (data.isEmpty()) { + return new DeploymentRoutingStatus(RoutingStatus.in, "", "", changedAt); + } + String agent = ""; + String reason = ""; + RoutingStatus status = RoutingStatus.out; + if (data.get().length > 0) { // Compatibility with old format, where no data is stored + Slime slime = SlimeUtils.jsonToSlime(data.get()); + Cursor root = slime.get(); + status = asRoutingStatus(root.field("status").asString()); + agent = root.field("agent").asString(); + reason = root.field("cause").asString(); + changedAt = Instant.ofEpochSecond(root.field("lastUpdate").asLong()); + } + return new DeploymentRoutingStatus(status, agent, reason, changedAt); + } + + private RoutingStatus zoneStatus() { + return curator.exists(ZONE_STATUS_ROOT) ? RoutingStatus.out : RoutingStatus.in; + } + + protected Path deploymentStatusPath(String upstreamName) { + return DEPLOYMENT_STATUS_ROOT.append(upstreamName); + } + + private static String upstreamName(RestApi.RequestContext context) { + String upstreamName = context.pathParameters().getStringOrThrow("upstreamName"); + if (upstreamName.contains(" ")) { + throw new RestApiException.BadRequest("Invalid upstream name: '" + upstreamName + "'"); + } + return upstreamName; + } + + private static ApplicationId instance(RestApi.RequestContext context) { + return context.queryParameters().getString("application") + .map(ApplicationId::fromSerializedForm) + .orElseThrow(() -> new RestApiException.BadRequest("Missing application parameter")); + } + + private byte[] toJsonBytes(DeploymentRoutingStatus status) { + return Exceptions.uncheck(() -> SlimeUtils.toJsonBytes(toSlime(status))); + } + + private Slime toSlime(DeploymentRoutingStatus status) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("status", asString(status.status())); + root.setString("cause", status.reason()); + root.setString("agent", status.agent()); + root.setLong("lastUpdate", status.changedAt().getEpochSecond()); + return slime; + } + + private static Slime toSlime(RoutingStatus status) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString("status", asString(status)); + return slime; + } + + private static RoutingStatus asRoutingStatus(String s) { + switch (s) { + case "IN": return RoutingStatus.in; + case "OUT": return RoutingStatus.out; + } + throw new IllegalArgumentException("Unknown status: '" + s + "'"); + } + + private static String asString(RoutingStatus status) { + switch (status) { + case in: return "IN"; + case out: return "OUT"; + } + throw new IllegalArgumentException("Unknown status: " + status); + } + + private static DeploymentRoutingStatus deploymentRoutingStatusFromSlime(Slime slime, Instant changedAt) { + Cursor root = slime.get(); + return new DeploymentRoutingStatus(asRoutingStatus(root.field("status").asString()), + root.field("agent").asString(), + root.field("cause").asString(), + changedAt); + } + + private static class DeploymentRoutingStatus { + + private final RoutingStatus status; + private final String agent; + private final String reason; + private final Instant changedAt; + + public DeploymentRoutingStatus(RoutingStatus status, String agent, String reason, Instant changedAt) { + this.status = Objects.requireNonNull(status); + this.agent = Objects.requireNonNull(agent); + this.reason = Objects.requireNonNull(reason); + this.changedAt = Objects.requireNonNull(changedAt); + } + + public RoutingStatus status() { + return status; + } + + public String agent() { + return agent; + } + + public String reason() { + return reason; + } + + public Instant changedAt() { + return changedAt; + } + + } + + private enum RoutingStatus { + in, out + } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 08c300220df..9cc475a56a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -100,7 +100,8 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { ? new FileDistributionConnectionPool(configSourceSet, supervisor) : new JRTConnectionPool(configSourceSet, supervisor), supervisor, - downloadDirectory); + downloadDirectory, + Duration.ofSeconds(30)); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java index 55986e71b3d..b813d56b345 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -30,16 +30,19 @@ public class ContainerEndpointSerializer { private static final String clusterIdField = "clusterId"; private static final String scopeField = "scope"; private static final String namesField = "names"; + private static final String weightField = "weight"; + private static final String routingMethodField = "routingMethod"; private ContainerEndpointSerializer() {} public static ContainerEndpoint endpointFromSlime(Inspector inspector) { final var clusterId = inspector.field(clusterIdField).asString(); - // Currently assigned endpoints that do not have scope should be interpreted as global endpoints - // TODO: Remove default assignment after 7.500 - final var scope = SlimeUtils.optionalString(inspector.field(scopeField)).orElse(ApplicationClusterEndpoint.Scope.global.name()); + final var scope = inspector.field(scopeField).asString(); final var namesInspector = inspector.field(namesField); - + final var weight = SlimeUtils.optionalInteger(inspector.field(weightField)); + // assign default routingmethod. Remove when 7.507 is latest version + // Cannot be used before all endpoints are assigned explicit routingmethod (from controller) + final var routingMethod = SlimeUtils.optionalString(inspector.field(routingMethodField)).orElse(ApplicationClusterEndpoint.RoutingMethod.sharedLayer4.name()); if (clusterId.isEmpty()) { throw new IllegalStateException("'clusterId' missing on serialized ContainerEndpoint"); } @@ -52,6 +55,10 @@ public class ContainerEndpointSerializer { throw new IllegalStateException("'names' missing on serialized ContainerEndpoint"); } + if(routingMethod.isEmpty()) { + throw new IllegalStateException("'routingMethod' missing on serialized ContainerEndpoint"); + } + final var names = new ArrayList<String>(); namesInspector.traverse((ArrayTraverser) (idx, nameInspector) -> { @@ -59,7 +66,8 @@ public class ContainerEndpointSerializer { names.add(containerName); }); - return new ContainerEndpoint(clusterId, ApplicationClusterEndpoint.Scope.valueOf(scope), names); + return new ContainerEndpoint(clusterId, ApplicationClusterEndpoint.Scope.valueOf(scope), names, weight, + ApplicationClusterEndpoint.RoutingMethod.valueOf(routingMethod)); } public static List<ContainerEndpoint> endpointListFromSlime(Slime slime) { @@ -81,9 +89,10 @@ public class ContainerEndpointSerializer { public static void endpointToSlime(Cursor cursor, ContainerEndpoint endpoint) { cursor.setString(clusterIdField, endpoint.clusterId()); cursor.setString(scopeField, endpoint.scope().name()); - + endpoint.weight().ifPresent(w -> cursor.setLong(weightField, w)); final var namesInspector = cursor.setArray(namesField); endpoint.names().forEach(namesInspector::addString); + cursor.setString(routingMethodField, endpoint.routingMethod().name()); } public static Slime endpointListToSlime(List<ContainerEndpoint> endpoints) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java new file mode 100644 index 00000000000..3eed93ce131 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v1/RoutingStatusApiHandlerTest.java @@ -0,0 +1,204 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.http.v1; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Deployer; +import com.yahoo.config.provision.Deployment; +import com.yahoo.container.jdisc.HttpRequestBuilder; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.jdisc.http.HttpRequest.Method; +import com.yahoo.path.Path; +import com.yahoo.restapi.RestApiTestDriver; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.curator.Curator; +import com.yahoo.vespa.curator.mock.MockCurator; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.yolean.Exceptions.uncheck; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author bjorncs + * @author mpolden + */ +public class RoutingStatusApiHandlerTest { + + private static final ApplicationId instance = ApplicationId.from("t1", "a1", "i1"); + private static final String upstreamName = "test-upstream-name"; + + private final Curator curator = new MockCurator(); + private final ManualClock clock = new ManualClock(); + private final MockDeployer deployer = new MockDeployer(clock); + + private RestApiTestDriver testDriver; + + @Before + public void before() { + RoutingStatusApiHandler requestHandler = new RoutingStatusApiHandler(RestApiTestDriver.createHandlerTestContext(), + curator, + clock, + deployer); + testDriver = RestApiTestDriver.newBuilder(requestHandler).build(); + } + + @Test + public void list_deployment_status() { + List<String> expected = List.of("foo", "bar"); + for (String upstreamName : expected) { + executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), + statusOut()); + } + String actual = responseAsString(executeRequest(Method.GET, "/routing/v1/status", null)); + assertEquals("[\"foo\",\"bar\"]", actual); + } + + @Test + public void get_deployment_status() { + String response = responseAsString(executeRequest(Method.GET, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), null)); + assertEquals(response("IN", "", "", clock.instant()), response); + } + + @Test + public void set_deployment_status() { + String response = responseAsString(executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), + statusOut())); + assertEquals(response("OUT", "issue-XXX", "operator", clock.instant()), response); + assertTrue("Re-deployed " + instance, deployer.lastDeployed.containsKey(instance)); + + // Status is reverted if redeployment fails + deployer.failNextDeployment(true); + response = responseAsString(executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), + requestContent("IN", "all good"))); + assertEquals("{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Failed to change status to in, reverting to out because redeployment of t1.a1.i1 failed: Deployment failed\"}", + response); + + // Read status stored in old format (path exists, but without content) + curator.set(Path.fromString("/routing/v1/status/" + upstreamName), new byte[0]); + response = responseAsString(executeRequest(Method.GET, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), null)); + + assertEquals(response("OUT", "", "", clock.instant()), response); + } + + @Test + public void fail_on_invalid_upstream_name() { + HttpResponse response = executeRequest(Method.GET, "/routing/v1/status/" + upstreamName + "%20invalid", null); + assertEquals(400, response.getStatus()); + } + + @Test + public void fail_on_changing_routing_status_without_request_content() { + HttpResponse response = executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), null); + assertEquals(400, response.getStatus()); + } + + @Test + public void zone_status_out_overrides_deployment_status() { + // Setting zone out overrides deployment status + executeRequest(Method.PUT, "/routing/v1/status/zone", null); + String response = responseAsString(executeRequest(Method.GET, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), null)); + assertEquals(response("OUT", "Rotation is OUT because the zone is OUT (actual deployment status is IN)", "operator", clock.instant()), response); + + // Setting zone back in falls back to deployment status, which is also out + executeRequest(Method.DELETE, "/routing/v1/status/zone", null); + String response2 = responseAsString(executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), + statusOut())); + assertEquals(response("OUT", "issue-XXX", "operator", clock.instant()), response2); + + // Deployment status is changed to in + String response3 = responseAsString(executeRequest(Method.PUT, "/routing/v1/status/" + upstreamName + "?application=" + instance.serializedForm(), + requestContent("IN", "all good"))); + assertEquals(response("IN", "all good", "operator", clock.instant()), response3); + } + + @Test + public void set_zone_status() { + executeRequest(Method.PUT, "/routing/v1/status/zone", null); + String response = responseAsString(executeRequest(Method.GET, "/routing/v1/status/zone", null)); + assertEquals("{\"status\":\"OUT\"}", response); + executeRequest(Method.DELETE, "/routing/v1/status/zone", null); + response = responseAsString(executeRequest(Method.GET, "/routing/v1/status/zone", null)); + assertEquals("{\"status\":\"IN\"}", response); + } + + private HttpResponse executeRequest(Method method, String path, String requestContent) { + var builder = HttpRequestBuilder.create(method, path); + if (requestContent != null) { + builder.withRequestContent(new ByteArrayInputStream(requestContent.getBytes(StandardCharsets.UTF_8))); + } + return testDriver.executeRequest(builder.build()); + } + + private static String responseAsString(HttpResponse response) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + uncheck(() -> response.render(out)); + return out.toString(StandardCharsets.UTF_8); + } + + private static String statusOut() { + return requestContent("OUT", "issue-XXX"); + } + + private static String requestContent(String status, String cause) { + return "{\"status\": \"" + status + "\", \"agent\":\"operator\", \"cause\": \"" + cause + "\"}"; + } + + private static String response(String status, String reason, String agent, Instant instant) { + return "{\"status\":\"" + status + "\",\"cause\":\"" + reason + "\",\"agent\":\"" + agent + "\",\"lastUpdate\":" + instant.getEpochSecond() + "}"; + } + + private static class MockDeployer implements Deployer { + + private final Map<ApplicationId, Instant> lastDeployed = new HashMap<>(); + private final Clock clock; + + private boolean failNextDeployment = false; + + public MockDeployer(Clock clock) { + this.clock = clock; + } + + public MockDeployer failNextDeployment(boolean fail) { + this.failNextDeployment = fail; + return this; + } + + @Override + public Optional<Deployment> deployFromLocalActive(ApplicationId application, boolean bootstrap) { + return deployFromLocalActive(application, Duration.ZERO, false); + } + + @Override + public Optional<Deployment> deployFromLocalActive(ApplicationId application, Duration timeout, boolean bootstrap) { + if (failNextDeployment) { + throw new RuntimeException("Deployment failed"); + } + lastDeployed.put(application, clock.instant()); + return Optional.empty(); + } + + @Override + public Optional<Instant> lastDeployTime(ApplicationId application) { + return Optional.ofNullable(lastDeployed.get(application)); + } + + @Override + public Duration serverDeployTimeout() { + return Duration.ZERO; + } + + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 2c97d0b9382..cd824967fc3 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -43,6 +43,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; +import java.util.OptionalLong; import java.util.stream.Collectors; import static org.hamcrest.CoreMatchers.is; @@ -90,18 +92,21 @@ public class PrepareParamsTest { public void testCorrectParsingWithContainerEndpoints() throws IOException { var endpoints = List.of(new ContainerEndpoint("qrs1", ApplicationClusterEndpoint.Scope.global, List.of("c1.example.com", - "c2.example.com")), + "c2.example.com"), OptionalInt.of(3)), new ContainerEndpoint("qrs2",ApplicationClusterEndpoint.Scope.global, List.of("c3.example.com", "c4.example.com"))); var param = "[\n" + " {\n" + " \"clusterId\": \"qrs1\",\n" + - " \"names\": [\"c1.example.com\", \"c2.example.com\"]\n" + + " \"names\": [\"c1.example.com\", \"c2.example.com\"],\n" + + " \"scope\": \"global\",\n" + + " \"weight\": 3\n" + " },\n" + " {\n" + " \"clusterId\": \"qrs2\",\n" + - " \"names\": [\"c3.example.com\", \"c4.example.com\"]\n" + + " \"names\": [\"c3.example.com\", \"c4.example.com\"],\n" + + " \"scope\": \"global\"\n" + " }\n" + "]"; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 08e6a353fbb..79632b8446b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -63,6 +63,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.logging.Level; @@ -248,14 +249,18 @@ public class SessionPreparerTest { " \"names\": [\n" + " \"foo.app1.tenant1.global.vespa.example.com\",\n" + " \"rotation-042.vespa.global.routing\"\n" + - " ]\n" + + " ],\n" + + " \"scope\": \"global\", \n" + + " \"routingMethod\": \"shared\"\n" + " },\n" + " {\n" + " \"clusterId\": \"bar\",\n" + " \"names\": [\n" + " \"bar.app1.tenant1.global.vespa.example.com\",\n" + " \"rotation-043.vespa.global.routing\"\n" + - " ]\n" + + " ],\n" + + " \"scope\": \"global\",\n" + + " \"routingMethod\": \"sharedLayer4\"\n" + " }\n" + "]"; var applicationId = applicationId("test"); @@ -267,11 +272,15 @@ public class SessionPreparerTest { var expected = List.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("foo.app1.tenant1.global.vespa.example.com", - "rotation-042.vespa.global.routing")), + "rotation-042.vespa.global.routing"), + OptionalInt.empty(), + ApplicationClusterEndpoint.RoutingMethod.shared), new ContainerEndpoint("bar", ApplicationClusterEndpoint.Scope.global, List.of("bar.app1.tenant1.global.vespa.example.com", - "rotation-043.vespa.global.routing"))); + "rotation-043.vespa.global.routing"), + OptionalInt.empty(), + ApplicationClusterEndpoint.RoutingMethod.sharedLayer4)); assertEquals(expected, readContainerEndpoints(applicationId)); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java index 2d767cfded4..c8f31697c5e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -7,6 +7,8 @@ import com.yahoo.slime.Slime; import org.junit.Test; import java.util.List; +import java.util.OptionalInt; +import java.util.OptionalLong; import static org.junit.Assert.assertEquals; @@ -33,24 +35,8 @@ public class ContainerEndpointSerializerTest { } @Test - public void readEndpointWithoutScope() { - final var slime = new Slime(); - final var entry = slime.setObject(); - - entry.setString("clusterId", "foobar"); - final var entryNames = entry.setArray("names"); - entryNames.addString("a"); - entryNames.addString("b"); - - final var endpoint = ContainerEndpointSerializer.endpointFromSlime(slime.get()); - assertEquals("foobar", endpoint.clusterId()); - assertEquals(ApplicationClusterEndpoint.Scope.global, endpoint.scope()); - assertEquals(List.of("a", "b"), endpoint.names()); - } - - @Test public void writeReadSingleEndpoint() { - final var endpoint = new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b")); + final var endpoint = new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b"), OptionalInt.of(1)); final var serialized = new Slime(); ContainerEndpointSerializer.endpointToSlime(serialized.setObject(), endpoint); final var deserialized = ContainerEndpointSerializer.endpointFromSlime(serialized.get()); @@ -60,7 +46,7 @@ public class ContainerEndpointSerializerTest { @Test public void writeReadEndpoints() { - final var endpoints = List.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b"))); + final var endpoints = List.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b"), OptionalInt.of(3), ApplicationClusterEndpoint.RoutingMethod.shared)); final var serialized = ContainerEndpointSerializer.endpointListToSlime(endpoints); final var deserialized = ContainerEndpointSerializer.endpointListFromSlime(serialized); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java index bfd412700ea..387290065c9 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/HttpRequest.java @@ -74,7 +74,7 @@ public class HttpRequest extends Request implements ServletOrJdiscHttpRequest { private final long jvmRelativeCreatedAt = System.nanoTime(); private final HeaderFields trailers = new HeaderFields(); private final Map<String, List<String>> parameters = new HashMap<>(); - private Principal principal; + private volatile Principal principal; private final long connectedAt; private Method method; private Version version; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index 46738c1501b..2eea7f155ee 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -6,6 +6,7 @@ import com.yahoo.container.logging.AccessLog; import com.yahoo.container.logging.AccessLogEntry; import com.yahoo.container.logging.RequestLog; import com.yahoo.container.logging.RequestLogEntry; +import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.jdisc.http.ServerConfig; import com.yahoo.jdisc.http.servlet.ServletRequest; import org.eclipse.jetty.http2.HTTP2Stream; @@ -17,7 +18,6 @@ import org.eclipse.jetty.server.Response; import org.eclipse.jetty.util.component.AbstractLifeCycle; import javax.servlet.http.HttpServletRequest; -import java.security.Principal; import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; @@ -81,8 +81,10 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty addNonNullValue(builder, request.getHeader("Referer"), RequestLogEntry.Builder::referer); addNonNullValue(builder, request.getQueryString(), RequestLogEntry.Builder::rawQuery); - Principal principal = (Principal) request.getAttribute(ServletRequest.JDISC_REQUEST_PRINCIPAL); - addNonNullValue(builder, principal, RequestLogEntry.Builder::userPrincipal); + HttpRequest jdiscRequest = (HttpRequest) request.getAttribute(HttpRequest.class.getName()); + if (jdiscRequest != null) { + addNonNullValue(builder, jdiscRequest.getUserPrincipal(), RequestLogEntry.Builder::userPrincipal); + } String requestFilterId = (String) request.getAttribute(ServletRequest.JDISC_REQUEST_CHAIN); addNonNullValue(builder, requestFilterId, (b, chain) -> b.addExtraAttribute("request-chain", chain)); diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java index 64cfbc96b17..52c2a83563e 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HttpRequestFactory.java @@ -33,6 +33,7 @@ class HttpRequestFactory { new InetSocketAddress(servletRequest.getRemoteAddr(), servletRequest.getRemotePort()), getConnection((Request) servletRequest).getCreatedTimeStamp()); httpRequest.context().put(ServletRequest.JDISC_REQUEST_X509CERT, getCertChain(servletRequest)); + servletRequest.setAttribute(HttpRequest.class.getName(), httpRequest); return httpRequest; } catch (Utf8Appendable.NotUtf8Exception e) { throw createBadQueryException(e); diff --git a/container-disc/src/main/sh/vespa-start-container-daemon.sh b/container-disc/src/main/sh/vespa-start-container-daemon.sh index d465edb3c39..eb446f9a251 100755 --- a/container-disc/src/main/sh/vespa-start-container-daemon.sh +++ b/container-disc/src/main/sh/vespa-start-container-daemon.sh @@ -39,10 +39,6 @@ CP="${VESPA_HOME}/lib/jars/jdisc_core-jar-with-dependencies.jar" mkdir -p $bundlecachedir || exit 1 printenv > $cfpfile || exit 1 -# ??? TODO ??? XXX ??? -# LANG=en_US.utf8 -# LC_ALL=C - getconfig() { qrstartcfg="" @@ -244,6 +240,13 @@ import_cfg_var () { fi } +# TODO Vespa 8: Remove when all containers use JDK 17 +configure_illegal_access() { + if [[ "$VESPA_JDK_VERSION" = "11" ]]; then + illegal_access_option="--illegal-access=debug" + fi +} + getconfig configure_memory configure_gcopts @@ -252,6 +255,7 @@ configure_classpath configure_numactl configure_cpu configure_preload +configure_illegal_access exec $numactlcmd $envcmd java \ -Dconfig.id="${VESPA_CONFIG_ID}" \ @@ -265,6 +269,7 @@ exec $numactlcmd $envcmd java \ -XX:HeapDumpPath="${VESPA_HOME}/var/crash" \ -XX:ErrorFile="${VESPA_HOME}/var/crash/hs_err_pid%p.log" \ -XX:+ExitOnOutOfMemoryError \ + ${illegal_access_option} \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ diff --git a/container-search/src/test/java/com/yahoo/search/handler/SearchHandlerTest.java b/container-search/src/test/java/com/yahoo/search/handler/SearchHandlerTest.java index 2c76383bff4..a6ae5f2b1b9 100644 --- a/container-search/src/test/java/com/yahoo/search/handler/SearchHandlerTest.java +++ b/container-search/src/test/java/com/yahoo/search/handler/SearchHandlerTest.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.net.URI; import java.util.concurrent.Executors; +import static com.yahoo.yolean.Exceptions.uncheckInterrupted; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; @@ -45,6 +46,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author bratseth @@ -304,22 +306,19 @@ public class SearchHandlerTest { assertOkResult(driver.sendRequest(request), jsonResult); } - private boolean waitForMetric(String key) { - try { - for (int i = 0; i < 10; i++) { - if (metric.metrics().containsKey(key)) return true; - Thread.sleep(20); - } - } catch (InterruptedException e) { + private void assertMetricPresent(String key) { + for (int i = 0; i < 200; i++) { + if (metric.metrics().containsKey(key)) return; + uncheckInterrupted(() -> Thread.sleep(1)); } - return false; + fail(String.format("Could not find metric with key '%s' in '%s'", key, metric)); } private void assertOkResult(RequestHandlerTestDriver.MockResponseHandler response, String expected) { assertEquals(expected, response.readAll()); assertEquals(200, response.getStatus()); assertEquals(selfHostname, response.getResponse().headers().get(myHostnameHeader).get(0)); - assertTrue(waitForMetric(SearchHandler.RENDER_LATENCY_METRIC)); + assertMetricPresent(SearchHandler.RENDER_LATENCY_METRIC); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index b9cb0d773c6..d4e11163343 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; +import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.ChangeRequestClient; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; @@ -108,4 +109,6 @@ public interface ServiceRegistry { HorizonClient horizonClient(); PlanRegistry planRegistry(); + + RoleMaintainer roleMaintainer(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 561475caa54..4679f660319 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -252,6 +252,10 @@ public class ZmsClientMock implements ZmsClient { } @Override + public void deleteRole(AthenzRole athenzRole) { + athenz.domains.get(athenzRole.domain()).roles.removeIf(role -> role.name().equals(athenzRole.roleName())); + } + @Override public void close() {} private static AthenzDomain getTenantDomain(AthenzResourceName resource) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java index 84e36ea75d1..bd4c3c1a56f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/Plan.java @@ -23,4 +23,7 @@ public interface Plan { /** Is this a plan that is billed */ boolean isBilled(); + + /** Is this a plan that gets on-call support */ + boolean isSupported(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java index 60eddbd24ff..5fb4d853e67 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java @@ -11,9 +11,9 @@ import java.util.stream.Stream; public class PlanRegistryMock implements PlanRegistry { - public static final Plan freeTrial = new MockPlan("trial", false, 0, 0, 0, 200, "Free Trial - for testing purposes"); - public static final Plan paidPlan = new MockPlan("paid", true, "0.09", "0.009", "0.0003", 500, "Paid Plan - for testing purposes"); - public static final Plan nonePlan = new MockPlan("none", false, 0, 0, 0, 0, "None Plan - for testing purposes"); + public static final Plan freeTrial = new MockPlan("trial", false, false, 0, 0, 0, 200, "Free Trial - for testing purposes"); + public static final Plan paidPlan = new MockPlan("paid", true, true, "0.09", "0.009", "0.0003", 500, "Paid Plan - for testing purposes"); + public static final Plan nonePlan = new MockPlan("none", false, false, 0, 0, 0, 0, "None Plan - for testing purposes"); @Override public Plan defaultPlan() { @@ -33,18 +33,20 @@ public class PlanRegistryMock implements PlanRegistry { private final CostCalculator costCalculator; private final QuotaCalculator quotaCalculator; private final boolean billed; + private final boolean supported; - public MockPlan(String planId, boolean billed, double cpuPrice, double memPrice, double dgbPrice, int quota, String description) { - this(PlanId.from(planId), billed, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description); + public MockPlan(String planId, boolean billed, boolean supported, double cpuPrice, double memPrice, double dgbPrice, int quota, String description) { + this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description); } - public MockPlan(String planId, boolean billed, String cpuPrice, String memPrice, String dgbPrice, int quota, String description) { - this(PlanId.from(planId), billed, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description); + public MockPlan(String planId, boolean billed, boolean supported, String cpuPrice, String memPrice, String dgbPrice, int quota, String description) { + this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description); } - public MockPlan(PlanId planId, boolean billed, MockCostCalculator calculator, QuotaCalculator quota, String description) { + public MockPlan(PlanId planId, boolean billed, boolean supported, MockCostCalculator calculator, QuotaCalculator quota, String description) { this.planId = planId; this.billed = billed; + this.supported = supported; this.costCalculator = calculator; this.quotaCalculator = quota; this.description = description; @@ -74,6 +76,11 @@ public class PlanRegistryMock implements PlanRegistry { public boolean isBilled() { return billed; } + + @Override + public boolean isSupported() { + return supported; + } } private static class MockCostCalculator implements CostCalculator { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java index bac34e73dc5..7246903a51b 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java @@ -1,8 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; +import com.yahoo.config.provision.zone.RoutingMethod; + import java.util.List; import java.util.Objects; +import java.util.OptionalInt; /** * This represents a list of one or more names for a container cluster. @@ -14,11 +17,15 @@ public class ContainerEndpoint { private final String clusterId; private final String scope; private final List<String> names; + private final OptionalInt weight; + private final RoutingMethod routingMethod; - public ContainerEndpoint(String clusterId, String scope, List<String> names) { - this.clusterId = nonEmpty(clusterId, "message must be non-empty"); + public ContainerEndpoint(String clusterId, String scope, List<String> names, OptionalInt weight, RoutingMethod routingMethod) { + this.clusterId = nonEmpty(clusterId, "clusterId must be non-empty"); this.scope = Objects.requireNonNull(scope, "scope must be non-null"); this.names = List.copyOf(Objects.requireNonNull(names, "names must be non-null")); + this.weight = Objects.requireNonNull(weight, "weight must be non-null"); + this.routingMethod = Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); } /** ID of the cluster to which this points */ @@ -39,22 +46,34 @@ public class ContainerEndpoint { return names; } + /** The relative weight of this endpoint */ + public OptionalInt weight() { + return weight; + } + + /** The routing method used by this endpoint */ + public RoutingMethod routingMethod() { + return routingMethod; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ContainerEndpoint that = (ContainerEndpoint) o; - return clusterId.equals(that.clusterId) && scope.equals(that.scope) && names.equals(that.names); + return clusterId.equals(that.clusterId) && scope.equals(that.scope) && names.equals(that.names) && weight.equals(that.weight) && routingMethod == that.routingMethod; } @Override public int hashCode() { - return Objects.hash(clusterId, scope, names); + return Objects.hash(clusterId, scope, names, weight, routingMethod); } @Override public String toString() { - return "container endpoint for " + clusterId + ": " + names + " [scope=" + scope + "]"; + return "container endpoint for cluster " + clusterId + ": " + String.join(", ", names) + + " [method=" + routingMethod + ",scope=" + scope + ",weight=" + + weight.stream().boxed().map(Object::toString).findFirst().orElse("<none>") + "]"; } private static String nonEmpty(String s, String message) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleMaintainer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleMaintainer.java new file mode 100644 index 00000000000..97a15b421c5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleMaintainer.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.user; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; + +import java.util.List; + +/** + * @author olaa + */ +public interface RoleMaintainer { + + /** Given the set of all existing tenants and applications, delete any superflous roles */ + void deleteLeftoverRoles(List<Tenant> tenants, List<ApplicationId> applications); + + /** Finds the subset of tenants that should be deleted based on role/domain existence */ + List<Tenant> tenantsToDelete(List<Tenant> tenants); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleMaintainerMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleMaintainerMock.java new file mode 100644 index 00000000000..df39f51b6fe --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/user/RoleMaintainerMock.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.user; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.tenant.Tenant; + +import java.util.List; + +/** + * @author olaa + */ +public class RoleMaintainerMock implements RoleMaintainer { + + @Override + public void deleteLeftoverRoles(List<Tenant> tenants, List<ApplicationId> applications) { + + } + + @Override + public List<Tenant> tenantsToDelete(List<Tenant> tenants) { + return List.of(); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 4772dbeaab1..6ef0df9f099 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -35,9 +35,10 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.ExclusiveDeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.SharedDeploymentRoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.ExclusiveRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.ExclusiveZoneRoutingContext; import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; -import com.yahoo.vespa.hosted.controller.routing.context.SharedRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.SharedZoneRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.rotation.Rotation; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; @@ -53,6 +54,8 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -95,9 +98,9 @@ public class RoutingController { /** Create a routing context for given zone */ public RoutingContext of(ZoneId zone) { if (usesSharedRouting(zone)) { - return new SharedRoutingContext(zone, controller.serviceRegistry().configServer()); + return new SharedZoneRoutingContext(zone, controller.serviceRegistry().configServer()); } - return new ExclusiveRoutingContext(zone, routingPolicies); + return new ExclusiveZoneRoutingContext(zone, routingPolicies); } public RoutingPolicies policies() { @@ -257,7 +260,6 @@ public class RoutingController { EndpointList endpoints = declaredEndpointsOf(application.get()).targets(deployment); EndpointList globalEndpoints = endpoints.scope(Endpoint.Scope.global); for (var assignedRotation : instance.rotations()) { - var names = new ArrayList<String>(); EndpointList rotationEndpoints = globalEndpoints.named(assignedRotation.endpointId()) .requiresRotation(); @@ -272,21 +274,21 @@ public class RoutingController { } // Register names in DNS - var rotation = rotationRepository.getRotation(assignedRotation.rotationId()); - if (rotation.isPresent()) { - rotationEndpoints.forEach(endpoint -> { - controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), - RecordData.fqdn(rotation.get().name()), - Priority.normal); - names.add(endpoint.dnsName()); - }); + Rotation rotation = rotationRepository.requireRotation(assignedRotation.rotationId()); + for (var endpoint : rotationEndpoints) { + controller.nameServiceForwarder().createCname(RecordName.from(endpoint.dnsName()), + RecordData.fqdn(rotation.name()), + Priority.normal); + List<String> names = List.of(endpoint.dnsName(), + // Include rotation ID as a valid name of this container endpoint + // (required by global routing health checks) + assignedRotation.rotationId().asString()); + containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), + asString(Endpoint.Scope.global), + names, + OptionalInt.empty(), + endpoint.routingMethod())); } - - // Include rotation ID as a valid name of this container endpoint (required by global routing health checks) - names.add(assignedRotation.rotationId().asString()); - containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), - asString(Endpoint.Scope.global), - names)); } // Add endpoints not backed by a rotation (i.e. other routing methods so that the config server always knows // about global names, even when not using rotations) @@ -295,7 +297,9 @@ public class RoutingController { .forEach((clusterId, clusterEndpoints) -> { containerEndpoints.add(new ContainerEndpoint(clusterId.value(), asString(Endpoint.Scope.global), - clusterEndpoints.mapToList(Endpoint::dnsName))); + clusterEndpoints.mapToList(Endpoint::dnsName), + OptionalInt.empty(), + RoutingMethod.exclusive)); }); // Add application endpoints EndpointList applicationEndpoints = endpoints.scope(Endpoint.Scope.application); @@ -313,12 +317,22 @@ public class RoutingController { RecordData.fqdn(vipHostname), Priority.normal); } - applicationEndpoints.groupingBy(Endpoint::cluster) - .forEach((clusterId, clusterEndpoints) -> { - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - asString(Endpoint.Scope.application), - clusterEndpoints.mapToList(Endpoint::dnsName))); - }); + Map<ClusterSpec.Id, EndpointList> applicationEndpointsByCluster = applicationEndpoints.groupingBy(Endpoint::cluster); + for (var kv : applicationEndpointsByCluster.entrySet()) { + ClusterSpec.Id clusterId = kv.getKey(); + EndpointList clusterEndpoints = kv.getValue(); + for (var endpoint : clusterEndpoints) { + Optional<Endpoint.Target> matchingTarget = endpoint.targets().stream() + .filter(t -> t.routesTo(deployment)) + .findFirst(); + if (matchingTarget.isEmpty()) throw new IllegalStateException("No target found routing to " + deployment + " in " + endpoint); + containerEndpoints.add(new ContainerEndpoint(clusterId.value(), + asString(Endpoint.Scope.application), + List.of(endpoint.dnsName()), + OptionalInt.of(matchingTarget.get().weight()), + endpoint.routingMethod())); + } + } return Collections.unmodifiableSet(containerEndpoints); } @@ -376,8 +390,8 @@ public class RoutingController { var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>(); for (var deployment : deployments) { for (var method : controller.zoneRegistry().routingMethods(deployment.zoneId())) { - deploymentsByMethod.putIfAbsent(method, new LinkedHashSet<>()); - deploymentsByMethod.get(method).add(deployment); + deploymentsByMethod.computeIfAbsent(method, k -> new LinkedHashSet<>()) + .add(deployment); } } var routingMethods = new ArrayList<RoutingMethod>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index aee7c1052be..544822e3be3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -482,6 +482,11 @@ public class Endpoint { return weight; } + /** Returns whether this routes to given deployment */ + public boolean routesTo(DeploymentId deployment) { + return this.deployment.equals(deployment); + } + } public static class EndpointBuilder { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index f9fd02fbf56..7fe8d554998 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -53,7 +53,7 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.deployments().containsAll(deployments)); } - /** Returns the subset of endpoints which target the given deployments */ + /** Returns the subset of endpoints which target the given deployment */ public EndpointList targets(DeploymentId deployment) { return targets(List.of(deployment)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 09eac53f218..e28273870d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -349,9 +349,15 @@ public class InternalStepRunner implements StepRunner { String failureReason = null; - NodeList suspendedTooLong = nodeList.suspendedSince(controller.clock().instant().minus(timeouts.nodesDown())); + NodeList suspendedTooLong = nodeList + .isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statefulNodesDown())) + .and(nodeList + .not().isStateful() + .suspendedSince(controller.clock().instant().minus(timeouts.statelessNodesDown())) + ); if ( ! suspendedTooLong.isEmpty()) { - failureReason = "Some nodes have been suspended for more than " + timeouts.nodesDown().toMinutes() + " minutes:\n" + + failureReason = "Some nodes have been suspended for more than the allowed threshold:\n" + suspendedTooLong.asList().stream().map(node -> node.node().hostname().value()).collect(joining("\n")); } @@ -1042,7 +1048,8 @@ public class InternalStepRunner implements StepRunner { Duration endpoint() { return Duration.ofMinutes(15); } Duration endpointCertificate() { return Duration.ofMinutes(20); } Duration tester() { return Duration.ofMinutes(30); } - Duration nodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } + Duration statelessNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 60); } + Duration statefulNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 720); } Duration noNodesDown() { return Duration.ofMinutes(system.isCd() ? 30 : 240); } Duration testerCertificate() { return Duration.ofMinutes(300); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java index 12c226241e1..cb0ff0644fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeList.java @@ -87,6 +87,10 @@ public class NodeList extends AbstractFilteringList<NodeWithServices, NodeList> return matching(NodeWithServices::needsNewConfig); } + public NodeList isStateful() { + return matching(NodeWithServices::isStateful); + } + /** The nodes that are retiring. */ public NodeList retiring() { return matching(node -> node.node().retired()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java index bd589af190e..d8f88d31759 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/NodeWithServices.java @@ -82,6 +82,10 @@ public class NodeWithServices { return services.stream().anyMatch(service -> wantedConfigGeneration > service.currentGeneration()); } + public boolean isStateful() { + return node.clusterType() == Node.ClusterType.content || node.clusterType() == Node.ClusterType.combined; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index f11cd78c303..913d6dfeab8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -74,7 +74,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new VcmrMaintainer(controller, intervals.vcmrMaintainer)); maintainers.add(new CloudTrialExpirer(controller, intervals.defaultInterval)); maintainers.add(new RetriggerMaintainer(controller, intervals.retriggerMaintainer)); - maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, userManagement)); + maintainers.add(new UserManagementMaintainer(controller, intervals.userManagementMaintainer, controller.serviceRegistry().roleMaintainer())); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 5f6f917bc75..33012763f97 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -1,17 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.role.ApplicationRole; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import com.yahoo.vespa.hosted.controller.api.role.TenantRole; +import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; import java.time.Duration; -import java.util.List; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -23,43 +19,32 @@ import java.util.stream.Collectors; */ public class UserManagementMaintainer extends ControllerMaintainer { - private final UserManagement userManagement; - + private final RoleMaintainer roleMaintainer; private static final Logger logger = Logger.getLogger(UserManagementMaintainer.class.getName()); - public UserManagementMaintainer(Controller controller, Duration interval, UserManagement userManagement) { - super(controller, interval, UserManagementMaintainer.class.getSimpleName(), SystemName.allOf(SystemName::isPublic)); - this.userManagement = userManagement; - + public UserManagementMaintainer(Controller controller, Duration interval, RoleMaintainer roleMaintainer) { + super(controller, interval); + this.roleMaintainer = roleMaintainer; } @Override protected double maintain() { - findLeftoverRoles().forEach(role -> { - logger.warning(String.format("Found unexpected %s - Deleting", role.toString())); - userManagement.deleteRole(role); - }); - return 1.0; - } - - // protected for testing - protected List<Role> findLeftoverRoles() { - var tenantRoles = controller().tenants().asList() + var tenants = controller().tenants().asList(); + var applications = controller().applications().idList() .stream() - .flatMap(tenant -> Roles.tenantRoles(tenant.name()).stream()) + .map(appId -> ApplicationId.from(appId.tenant(), appId.application(), InstanceName.defaultName())) .collect(Collectors.toList()); + roleMaintainer.deleteLeftoverRoles(tenants, applications); - var applicationRoles = controller().applications().asList() - .stream() - .map(Application::id) - .flatMap(applicationId -> Roles.applicationRoles(applicationId.tenant(), applicationId.application()).stream()) - .collect(Collectors.toList()); + if (!controller().system().isPublic()) { + roleMaintainer.tenantsToDelete(tenants) + .forEach(tenant -> { + // TODO: controller().tenants().delete(tenant.name()); + logger.fine("Want to delete tenant " + tenant.name()); + }); + } - return userManagement.listRoles().stream() - .peek(role -> logger.fine(role::toString)) - .filter(role -> role instanceof TenantRole || role instanceof ApplicationRole) - .filter(role -> !tenantRoles.contains(role) && !applicationRoles.contains(role)) - .collect(Collectors.toList()); + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 0d12b283543..7e9ae036cc7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -25,6 +25,7 @@ import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.User; @@ -176,6 +177,7 @@ public class UserApiHandler extends LoggingRequestHandler { .sorted() .forEach(tenant -> { Cursor tenantObject = tenants.setObject(tenant.value()); + tenantObject.setBool("supported", hasSupportedPlan(tenant)); Cursor tenantRolesObject = tenantObject.setArray("roles"); tenantRolesByTenantName.getOrDefault(tenant, List.of()) @@ -405,4 +407,11 @@ public class UserApiHandler extends LoggingRequestHandler { .map(clazz::cast) .orElseThrow(() -> new IllegalArgumentException("Attribute '" + attributeName + "' was not set on request")); } + + private boolean hasSupportedPlan(TenantName tenantName) { + var planId = controller.serviceRegistry().billingController().getPlan(tenantName); + return controller.serviceRegistry().planRegistry().plan(planId) + .map(Plan::isSupported) + .orElse(false); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveZoneRoutingContext.java index e949c45f2fd..e29fb5ab404 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveZoneRoutingContext.java @@ -13,12 +13,12 @@ import java.util.Objects; * * @author mpolden */ -public class ExclusiveRoutingContext implements RoutingContext { +public class ExclusiveZoneRoutingContext implements RoutingContext { private final RoutingPolicies policies; private final ZoneId zone; - public ExclusiveRoutingContext(ZoneId zone, RoutingPolicies policies) { + public ExclusiveZoneRoutingContext(ZoneId zone, RoutingPolicies policies) { this.policies = Objects.requireNonNull(policies); this.zone = Objects.requireNonNull(zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java index e38212d7f80..2923c8dff5c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedZoneRoutingContext.java @@ -15,12 +15,12 @@ import java.util.Objects; * * @author mpolden */ -public class SharedRoutingContext implements RoutingContext { +public class SharedZoneRoutingContext implements RoutingContext { private final ConfigServer configServer; private final ZoneId zone; - public SharedRoutingContext(ZoneId zone, ConfigServer configServer) { + public SharedZoneRoutingContext(ZoneId zone, ConfigServer configServer) { this.configServer = Objects.requireNonNull(configServer); this.zone = Objects.requireNonNull(zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java index 961fdc6dd9c..39a0b6a8858 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java @@ -21,7 +21,6 @@ import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.logging.Logger; @@ -56,9 +55,11 @@ public class RotationRepository { return new RotationLock(curator.lockRotations()); } - /** Get rotation by given rotationId */ - public Optional<Rotation> getRotation(RotationId rotationId) { - return Optional.of(allRotations.get(rotationId)); + /** Get rotation with given id */ + public Rotation requireRotation(RotationId id) { + Rotation rotation = allRotations.get(id); + if (rotation == null) throw new IllegalArgumentException("No such rotation: '" + id.asString() + "'"); + return rotation; } /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 30cdd1b8466..1215ddbc2ad 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -37,10 +36,10 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; @@ -50,6 +49,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -233,22 +233,41 @@ public class ControllerTest { public void testDnsUpdatesForGlobalEndpoint() { var betaContext = tester.newDeploymentContext("tenant1", "app1", "beta"); var defaultContext = tester.newDeploymentContext("tenant1", "app1", "default"); + + ZoneId usWest = ZoneId.from("prod.us-west-1"); + ZoneId usCentral = ZoneId.from("prod.us-central-1"); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) .instances("beta,default") .endpoint("default", "foo") - .region("us-west-1") - .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .region(usWest.region()) + .region(usCentral.region()) // Two deployments should result in each DNS alias being registered once .build(); + tester.controllerTester().zoneRegistry().setRoutingMethod(List.of(ZoneApiMock.from(usWest), ZoneApiMock.from(usCentral)), + RoutingMethod.shared, + RoutingMethod.sharedLayer4); betaContext.submit(applicationPackage).deploy(); { // Expected rotation names are passed to beta instance deployments Collection<Deployment> betaDeployments = betaContext.instance().deployments().values(); assertFalse(betaDeployments.isEmpty()); + Set<ContainerEndpoint> containerEndpoints = Set.of(new ContainerEndpoint("foo", + "global", + List.of("beta--app1--tenant1.global.vespa.oath.cloud", + "rotation-id-01"), + OptionalInt.empty(), + RoutingMethod.shared), + new ContainerEndpoint("foo", + "global", + List.of("beta.app1.tenant1.global.vespa.oath.cloud", + "rotation-id-01"), + OptionalInt.empty(), + RoutingMethod.sharedLayer4)); + for (Deployment deployment : betaDeployments) { - assertEquals("Rotation names are passed to config server in " + deployment.zone(), - Set.of("rotation-id-01", - "beta--app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpointNames(betaContext.deploymentIdIn(deployment.zone()))); + assertEquals(containerEndpoints, + tester.configServer().containerEndpoints() + .get(betaContext.deploymentIdIn(deployment.zone()))); } betaContext.flushDnsUpdates(); } @@ -256,11 +275,21 @@ public class ControllerTest { { // Expected rotation names are passed to default instance deployments Collection<Deployment> defaultDeployments = defaultContext.instance().deployments().values(); assertFalse(defaultDeployments.isEmpty()); + Set<ContainerEndpoint> containerEndpoints = Set.of(new ContainerEndpoint("foo", + "global", + List.of("app1--tenant1.global.vespa.oath.cloud", + "rotation-id-02"), + OptionalInt.empty(), + RoutingMethod.shared), + new ContainerEndpoint("foo", + "global", + List.of("app1.tenant1.global.vespa.oath.cloud", + "rotation-id-02"), + OptionalInt.empty(), + RoutingMethod.sharedLayer4)); for (Deployment deployment : defaultDeployments) { - assertEquals("Rotation names are passed to config server in " + deployment.zone(), - Set.of("rotation-id-02", - "app1--tenant1.global.vespa.oath.cloud"), - tester.configServer().containerEndpointNames(defaultContext.deploymentIdIn(deployment.zone()))); + assertEquals(containerEndpoints, + tester.configServer().containerEndpoints().get(defaultContext.deploymentIdIn(deployment.zone()))); } defaultContext.flushDnsUpdates(); } @@ -274,13 +303,17 @@ public class ControllerTest { assertEquals(data, record.get().data().asString()); }); - Map<ApplicationId, List<String>> globalDnsNamesByInstance = Map.of(betaContext.instanceId(), List.of("beta--app1--tenant1.global.vespa.oath.cloud"), - defaultContext.instanceId(), List.of("app1--tenant1.global.vespa.oath.cloud")); + Map<ApplicationId, Set<String>> globalDnsNamesByInstance = Map.of(betaContext.instanceId(), Set.of("beta--app1--tenant1.global.vespa.oath.cloud", + "beta.app1.tenant1.global.vespa.oath.cloud"), + defaultContext.instanceId(), Set.of("app1--tenant1.global.vespa.oath.cloud", + "app1.tenant1.global.vespa.oath.cloud")); globalDnsNamesByInstance.forEach((instance, dnsNames) -> { - List<String> actualDnsNames = tester.controller().routing().readDeclaredEndpointsOf(instance) - .scope(Endpoint.Scope.global) - .mapToList(Endpoint::dnsName); + Set<String> actualDnsNames = tester.controller().routing().readDeclaredEndpointsOf(instance) + .scope(Endpoint.Scope.global) + .asList().stream() + .map(Endpoint::dnsName) + .collect(Collectors.toSet()); assertEquals("Global DNS names for " + instance, dnsNames, actualDnsNames); }); } @@ -617,33 +650,46 @@ public class ControllerTest { @Test public void testDnsUpdatesForApplicationEndpoint() { - var context = tester.newDeploymentContext("tenant1", "app1", "beta"); + ApplicationId beta = ApplicationId.from("tenant1", "app1", "beta"); + ApplicationId main = ApplicationId.from("tenant1", "app1", "main"); + var context = tester.newDeploymentContext(beta); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .instances("beta,main") .region("us-west-1") .region("us-east-3") .applicationEndpoint("a", "default", "us-west-1", - Map.of(InstanceName.from("beta"), 2, - InstanceName.from("main"), 8)) + Map.of(beta.instance(), 2, + main.instance(), 8)) .applicationEndpoint("b", "default", "us-west-1", - Map.of(InstanceName.from("beta"), 1, - InstanceName.from("main"), 1)) + Map.of(beta.instance(), 1, + main.instance(), 1)) .applicationEndpoint("c", "default", "us-east-3", - Map.of(InstanceName.from("beta"), 4, - InstanceName.from("main"), 6)) + Map.of(beta.instance(), 4, + main.instance(), 6)) .build(); context.submit(applicationPackage).deploy(); - // Endpoint names are passed to each deployment - DeploymentId usWest = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); - DeploymentId usEast = context.deploymentIdIn(ZoneId.from("prod", "us-east-3")); - Map<DeploymentId, List<String>> deploymentEndpoints = Map.of(usWest, List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", "b.app1.tenant1.us-west-1-r.vespa.oath.cloud"), - usEast, List.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud")); - deploymentEndpoints.forEach((zone, endpointNames) -> { - assertEquals("Endpoint names are passed to config server in " + zone, - Set.of(new ContainerEndpoint("default", "application", - endpointNames)), - tester.configServer().containerEndpoints().get(zone)); + ZoneId usWest = ZoneId.from("prod", "us-west-1"); + ZoneId usEast = ZoneId.from("prod", "us-east-3"); + // Expected container endpoints are passed to each deployment + Map<DeploymentId, Map<String, Integer>> deploymentEndpoints = Map.of( + new DeploymentId(beta, usWest), Map.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", 2, + "b.app1.tenant1.us-west-1-r.vespa.oath.cloud", 1), + new DeploymentId(main, usWest), Map.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", 8, + "b.app1.tenant1.us-west-1-r.vespa.oath.cloud", 1), + new DeploymentId(beta, usEast), Map.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud", 4), + new DeploymentId(main, usEast), Map.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud", 6) + ); + deploymentEndpoints.forEach((deployment, endpoints) -> { + Set<ContainerEndpoint> expected = endpoints.entrySet().stream() + .map(kv -> new ContainerEndpoint("default", "application", + List.of(kv.getKey()), + OptionalInt.of(kv.getValue()), + RoutingMethod.sharedLayer4)) + .collect(Collectors.toSet()); + assertEquals("Endpoint names for " + deployment + " are passed to config server", + expected, + tester.configServer().containerEndpoints().get(deployment)); }); context.flushDnsUpdates(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 5cf554f2c01..ae92fd46f26 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -210,7 +210,7 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(failed, tester.jobs().last(app.instanceId(), JobType.stagingTest).get().stepStatuses().get(Step.installInitialReal)); - tester.clock().advance(InternalStepRunner.Timeouts.of(system()).nodesDown().minus(Duration.ofSeconds(3))); + tester.clock().advance(InternalStepRunner.Timeouts.of(system()).statelessNodesDown().minus(Duration.ofSeconds(3))); tester.runner().run(); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 43ef9daa178..b1311b8081c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -44,6 +44,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainer; +import com.yahoo.vespa.hosted.controller.api.integration.user.RoleMaintainerMock; import com.yahoo.vespa.hosted.controller.api.integration.vcmr.MockChangeRequestClient; /** @@ -86,6 +88,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final PlanRegistry planRegistry = new PlanRegistryMock(); private final ResourceDatabaseClient resourceDb = new ResourceDatabaseClientMock(planRegistry); private final BillingDatabaseClient billingDb = new BillingDatabaseClientMock(clock, planRegistry); + private final RoleMaintainer roleMaintainer = new RoleMaintainerMock(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -267,6 +270,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return billingDb; } + @Override + public RoleMaintainer roleMaintainer() { + return roleMaintainer; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java deleted file mode 100644 index 52cb3ce121f..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainerTest.java +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockUserManagement; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; -import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.role.Role; -import org.junit.Test; - -import java.time.Duration; - -import static org.junit.Assert.*; - -/** - * @author olaa - */ -public class UserManagementMaintainerTest { - - private final ControllerTester tester = new ControllerTester(); - private final UserManagement userManagement = new MockUserManagement(); - private final UserManagementMaintainer userManagementMaintainer = new UserManagementMaintainer(tester.controller(), Duration.ofMinutes(1), userManagement); - - private final TenantName tenant = TenantName.from("tenant1"); - private final ApplicationName app = ApplicationName.from("app1"); - private final TenantName deletedTenant = TenantName.from("deleted-tenant"); - - @Test - public void finds_superfluous_roles() { - tester.createTenant(tenant.value()); - tester.createApplication(tenant.value(), app.value()); - - Roles.tenantRoles(tenant).forEach(userManagement::createRole); - Roles.applicationRoles(tenant, app).forEach(userManagement::createRole); - Roles.tenantRoles(deletedTenant).forEach(userManagement::createRole); - userManagement.createRole(Role.hostedSupporter()); - - var expectedRoles = Roles.tenantRoles(deletedTenant); - var actualRoles = userManagementMaintainer.findLeftoverRoles(); - - assertEquals(expectedRoles.size(), actualRoles.size()); - assertTrue(expectedRoles.containsAll(actualRoles) && actualRoles.containsAll(expectedRoles)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 1d41beb8a99..537f6c48bdf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.api.integration.user.User; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; @@ -270,4 +271,27 @@ public class UserApiTest extends ControllerContainerCloudTest { new File("user-without-trial-capacity-cloud.json")); } } + + @Test + public void supportTenant() { + try (Flags.Replacer ignored = Flags.clearFlagsForTesting(PermanentFlags.MAX_TRIAL_TENANTS.id(), PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id())) { + ContainerTester tester = new ContainerTester(container, responseFiles); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withIntFlag(PermanentFlags.MAX_TRIAL_TENANTS.id(), 10) + .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true); + ControllerTester controller = new ControllerTester(tester); + User user = new User("dev@domail", "Joe Developer", "dev", null); + + var tenant1 = controller.createTenant("tenant1", Tenant.Type.cloud); + var tenant2 = controller.createTenant("tenant2", Tenant.Type.cloud); + controller.serviceRegistry().billingController().setPlan(tenant2, PlanId.from("paid"), false); + + tester.assertResponse( + request("/user/v1/user") + .roles(Role.reader(tenant1), Role.reader(tenant2)) + .user(user), + new File("user-with-supported-tenant.json")); + } + + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json index 006c3b98a4d..0211f595ce7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-athenz.json @@ -10,6 +10,7 @@ }, "tenants": { "sandbox": { + "supported": (ignore), "roles": [ "administrator", "developer", @@ -17,6 +18,7 @@ ] }, "tenant1": { + "supported": (ignore), "roles": [ "administrator", "developer", @@ -24,6 +26,7 @@ ] }, "tenant2": { + "supported": (ignore), "roles": [ "administrator", "developer", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json index 4ae55e97baa..76904bf9bb4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-applications-cloud.json @@ -10,17 +10,20 @@ }, "tenants": { "sandbox": { + "supported": false, "roles": [ "developer", "reader" ] }, "tenant1": { + "supported": false, "roles": [ "administrator" ] }, "tenant2": { + "supported": false, "roles": [ "developer" ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json new file mode 100644 index 00000000000..a40354a9e71 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/user-with-supported-tenant.json @@ -0,0 +1,35 @@ +{ + "isPublic": true, + "isCd": false, + "hasTrialCapacity": true, + "user": { + "name": "Joe Developer", + "email": "dev@domail", + "nickname": "dev", + "verified": false + }, + "tenants": { + "tenant1": { + "supported": false, + "roles": [ + "reader" + ] + }, + "tenant2": { + "supported": true, + "roles": [ + "reader" + ] + } + }, + "flags": [ + { + "id": "enable-public-signup-flow", + "rules": [ + { + "value": false + } + ] + } + ] +}
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index 9a3ac8b547d..9a56123e8e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -205,10 +205,9 @@ public class RotationRepositoryTest { private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) { assertEquals(1, assignedRotations.size()); - var rotationId = assignedRotations.get(0).rotationId(); - var rotation = repository.getRotation(rotationId); - assertTrue(rotationId + " exists", rotation.isPresent()); - assertEquals(expected, rotation.get()); + RotationId rotationId = assignedRotations.get(0).rotationId(); + Rotation rotation = repository.requireRotation(rotationId); + assertEquals(expected, rotation); } private static List<RotationId> rotationIds(List<AssignedRotation> assignedRotations) { @@ -9,7 +9,7 @@ fi VERSION="$1" mkdir -p ~/rpmbuild/{SOURCES,SPECS} -GZIP=-1 tar -zcf ~/rpmbuild/SOURCES/vespa-$VERSION.tar.gz --exclude target --exclude cmake-build-debug --transform "flags=r;s,^,vespa-$VERSION/," * +git archive --format=tar.gz --prefix=vespa-$VERSION/ -o ~/rpmbuild/SOURCES/vespa-$VERSION.tar.gz HEAD DIST_FILE="dist/vespa.spec" # When checking out relase tags, the vespa.spec is in the source root folder. This is a workaround to be able to build rpms from a release tag. diff --git a/dist/vespa.spec b/dist/vespa.spec index edae861fd64..8620a26d945 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -187,7 +187,12 @@ BuildRequires: java-11-openjdk-devel %endif BuildRequires: rpm-build BuildRequires: make +%if 0%{?el7} && ! 0%{?amzn2} +BuildRequires: rh-git227 +%define _rhgit227_enable /opt/rh/rh-git227/enable +%else BuildRequires: git +%endif BuildRequires: golang BuildRequires: systemd BuildRequires: flex >= 2.5.0 @@ -503,6 +508,9 @@ source %{_devtoolset_enable} || true %if 0%{?_rhmaven35_enable:1} source %{_rhmaven35_enable} || true %endif +%if 0%{?_rhgit227_enable:1} +source %{_rhgit227_enable} || true +%endif %if 0%{?_java_home:1} export JAVA_HOME=%{?_java_home} @@ -529,6 +537,7 @@ mvn --batch-mode -e -N io.takari:maven:wrapper -Dmaven=3.6.3 . make %{_smp_mflags} +env GOTMPDIR=$(pwd)/client/go make -C client/go %endif %install @@ -538,6 +547,8 @@ rm -rf %{buildroot} cp -r %{installdir} %{buildroot} %else make install DESTDIR=%{buildroot} +# TODO: Include the vespa program +#cp client/go/bin/vespa %{buildroot}%{_prefix}/bin/vespa %endif %if %{_create_vespa_service} @@ -750,10 +761,13 @@ fi %defattr(-,%{_vespa_user},%{_vespa_group},-) %endif %dir %{_prefix} +%dir %{_prefix}/bin %dir %{_prefix}/conf %dir %{_prefix}/conf/vespa-feed-client %dir %{_prefix}/lib %dir %{_prefix}/lib/jars +# TODO: Include the vespa program +#%{_prefix}/bin/vespa %{_prefix}/bin/vespa-feed-client %{_prefix}/conf/vespa-feed-client/logging.properties %{_prefix}/lib/jars/vespa-http-client-jar-with-dependencies.jar diff --git a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java index 8d2b19b2818..91274144710 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/CompositeReader.java @@ -3,6 +3,7 @@ package com.yahoo.document.json.readers; import com.fasterxml.jackson.core.JsonToken; import com.yahoo.document.DataType; +import com.yahoo.document.PositionDataType; import com.yahoo.document.datatypes.CollectionFieldValue; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.MapFieldValue; @@ -35,6 +36,8 @@ public class CompositeReader { } } else if (fieldValue instanceof MapFieldValue) { MapReader.fillMap(buffer, (MapFieldValue) fieldValue); + } else if (PositionDataType.INSTANCE.equals(fieldValue.getDataType())) { + GeoPositionReader.fillGeoPosition(buffer, fieldValue); } else if (fieldValue instanceof StructuredFieldValue) { StructReader.fillStruct(buffer, (StructuredFieldValue) fieldValue); } else if (fieldValue instanceof TensorFieldValue) { diff --git a/document/src/main/java/com/yahoo/document/json/readers/GeoPositionReader.java b/document/src/main/java/com/yahoo/document/json/readers/GeoPositionReader.java new file mode 100644 index 00000000000..eb3919e07d7 --- /dev/null +++ b/document/src/main/java/com/yahoo/document/json/readers/GeoPositionReader.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.document.json.readers; + +import com.fasterxml.jackson.core.JsonToken; +import com.yahoo.document.PositionDataType; +import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.document.json.TokenBuffer; + +import static com.yahoo.document.json.readers.JsonParserHelpers.*; + +/** + * @author arnej + */ +public class GeoPositionReader { + + static void fillGeoPosition(TokenBuffer buffer, FieldValue positionFieldValue) { + Double latitude = null; + Double longitude = null; + expectObjectStart(buffer.currentToken()); + int initNesting = buffer.nesting(); + for (buffer.next(); buffer.nesting() >= initNesting; buffer.next()) { + String curName = buffer.currentName(); + if ("lat".equals(curName) || "latitude".equals(curName)) { + latitude = readDouble(buffer) * 1.0e6; + } else if ("lng".equals(curName) || "longitude".equals(curName)) { + longitude = readDouble(buffer) * 1.0e6; + } else if ("x".equals(curName)) { + longitude = readDouble(buffer); + } else if ("y".equals(curName)) { + latitude = readDouble(buffer); + } else { + throw new IllegalArgumentException("Unexpected attribute "+curName+" in geo position field"); + } + } + expectObjectEnd(buffer.currentToken()); + if (latitude == null) { + throw new IllegalArgumentException("Missing 'lat' attribute in geo position field"); + } + if (longitude == null) { + throw new IllegalArgumentException("Missing 'lng' attribute in geo position field"); + } + int y = (int) Math.round(latitude); + int x = (int) Math.round(longitude); + var geopos = PositionDataType.valueOf(x, y); + positionFieldValue.assign(geopos); + } + + private static double readDouble(TokenBuffer buffer) { + try { + return Double.parseDouble(buffer.currentText()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Expected a number but got '" + buffer.currentText()); + } + } + +} diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 527159dbc10..a1c1669ffa1 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -504,18 +504,22 @@ public class JsonReaderTestCase { assertEquals("smoke", docType.getName()); } + private Document docFromJson(String json) throws IOException { + JsonReader r = createReader(json); + DocumentParseInfo parseInfo = r.parseDocument().get(); + DocumentType docType = r.readDocumentType(parseInfo.documentId); + DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); + new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); + return put.getDocument(); + } + @Test public void testWeightedSet() throws IOException { - JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testset::whee',", + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testset::whee',", " 'fields': {", " 'actualset': {", " 'nalle': 2,", " 'tralle': 7 }}}")); - DocumentParseInfo parseInfo = r.parseDocument().get(); - DocumentType docType = r.readDocumentType(parseInfo.documentId); - DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); - Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualset")); assertSame(WeightedSet.class, f.getClass()); WeightedSet<?> w = (WeightedSet<?>) f; @@ -526,16 +530,11 @@ public class JsonReaderTestCase { @Test public void testArray() throws IOException { - JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testarray::whee',", + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testarray::whee',", " 'fields': {", " 'actualarray': [", " 'nalle',", " 'tralle' ]}}")); - DocumentParseInfo parseInfo = r.parseDocument().get(); - DocumentType docType = r.readDocumentType(parseInfo.documentId); - DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); - Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualarray")); assertSame(Array.class, f.getClass()); Array<?> a = (Array<?>) f; @@ -546,16 +545,11 @@ public class JsonReaderTestCase { @Test public void testMap() throws IOException { - JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testmap::whee',", + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testmap::whee',", " 'fields': {", " 'actualmap': {", " 'nalle': 'kalle',", " 'tralle': 'skalle' }}}")); - DocumentParseInfo parseInfo = r.parseDocument().get(); - DocumentType docType = r.readDocumentType(parseInfo.documentId); - DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); - Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualmap")); assertSame(MapFieldValue.class, f.getClass()); MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) f; @@ -566,16 +560,11 @@ public class JsonReaderTestCase { @Test public void testOldMap() throws IOException { - JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testmap::whee',", + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testmap::whee',", " 'fields': {", " 'actualmap': [", " { 'key': 'nalle', 'value': 'kalle'},", " { 'key': 'tralle', 'value': 'skalle'} ]}}")); - DocumentParseInfo parseInfo = r.parseDocument().get(); - DocumentType docType = r.readDocumentType(parseInfo.documentId); - DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); - Document doc = put.getDocument(); FieldValue f = doc.getFieldValue(doc.getField("actualmap")); assertSame(MapFieldValue.class, f.getClass()); MapFieldValue<?, ?> m = (MapFieldValue<?, ?>) f; @@ -586,14 +575,42 @@ public class JsonReaderTestCase { @Test public void testPositionPositive() throws IOException { - JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", " 'fields': {", " 'singlepos': 'N63.429722;E10.393333' }}")); - DocumentParseInfo parseInfo = r.parseDocument().get(); - DocumentType docType = r.readDocumentType(parseInfo.documentId); - DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); - Document doc = put.getDocument(); + FieldValue f = doc.getFieldValue(doc.getField("singlepos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + } + + @Test + public void testPositionOld() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'singlepos': {'x':10393333,'y':63429722} }}")); + FieldValue f = doc.getFieldValue(doc.getField("singlepos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + } + + @Test + public void testGeoPosition() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'singlepos': {'lat':63.429722,'lng':10.393333} }}")); + FieldValue f = doc.getFieldValue(doc.getField("singlepos")); + assertSame(Struct.class, f.getClass()); + assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); + assertEquals(63429722, PositionDataType.getYValue(f).getInteger()); + } + + @Test + public void testGeoPositionNoAbbreviations() throws IOException { + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'singlepos': {'latitude':63.429722,'longitude':10.393333} }}")); FieldValue f = doc.getFieldValue(doc.getField("singlepos")); assertSame(Struct.class, f.getClass()); assertEquals(10393333, PositionDataType.getXValue(f).getInteger()); @@ -602,14 +619,9 @@ public class JsonReaderTestCase { @Test public void testPositionNegative() throws IOException { - JsonReader r = createReader(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", - " 'fields': {", - " 'singlepos': 'W46.63;S23.55' }}")); - DocumentParseInfo parseInfo = r.parseDocument().get(); - DocumentType docType = r.readDocumentType(parseInfo.documentId); - DocumentPut put = new DocumentPut(new Document(docType, parseInfo.documentId)); - new VespaJsonDocumentReader().readPut(parseInfo.fieldsBuffer, put); - Document doc = put.getDocument(); + Document doc = docFromJson(inputJson("{ 'put': 'id:unittest:testsinglepos::bamf',", + " 'fields': {", + " 'singlepos': 'W46.63;S23.55' }}")); FieldValue f = doc.getFieldValue(doc.getField("singlepos")); assertSame(Struct.class, f.getClass()); assertEquals(-46630000, PositionDataType.getXValue(f).getInteger()); diff --git a/eval/src/vespa/eval/eval/hamming_distance.h b/eval/src/vespa/eval/eval/hamming_distance.h index 50c59c46a60..e7cfc88661d 100644 --- a/eval/src/vespa/eval/eval/hamming_distance.h +++ b/eval/src/vespa/eval/eval/hamming_distance.h @@ -5,8 +5,8 @@ namespace vespalib::eval { inline double hamming_distance(double a, double b) { - uint8_t x = (uint8_t) a; - uint8_t y = (uint8_t) b; + uint8_t x = (uint8_t) (int8_t) a; + uint8_t y = (uint8_t) (int8_t) b; return __builtin_popcount(x ^ y); } diff --git a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp index a439520677a..43ed724e010 100644 --- a/eval/src/vespa/eval/eval/llvm/compile_cache.cpp +++ b/eval/src/vespa/eval/eval/llvm/compile_cache.cpp @@ -4,8 +4,7 @@ #include <vespa/eval/eval/key_gen.h> #include <thread> -namespace vespalib { -namespace eval { +namespace vespalib::eval { std::mutex CompileCache::_lock{}; CompileCache::Map CompileCache::_cached{}; @@ -148,5 +147,4 @@ CompileCache::CompileTask::run() result->cond.notify_all(); } -} // namespace vespalib::eval -} // namespace vespalib +} diff --git a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp index 8c1e2fb525c..512e12bec71 100644 --- a/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp +++ b/eval/src/vespa/eval/eval/llvm/llvm_wrapper.cpp @@ -725,7 +725,8 @@ LLVMWrapper::compile(llvm::raw_ostream * dumpStream) if (dumpStream) { _module->print(*dumpStream, nullptr); } - _engine.reset(llvm::EngineBuilder(std::move(_module)).setOptLevel(llvm::CodeGenOpt::Aggressive).create()); + // Set relocation model to silence valgrind on CentOS 8 / aarch64 + _engine.reset(llvm::EngineBuilder(std::move(_module)).setOptLevel(llvm::CodeGenOpt::Aggressive).setRelocationModel(llvm::Reloc::Static).create()); assert(_engine && "llvm jit not available for your platform"); _engine->finalizeObject(); } diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp index 0764c9b1b66..1382aef7386 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/fastos/src/vespa/fastos/file.cpp @@ -39,7 +39,7 @@ static const size_t MAX_WRITE_CHUNK_SIZE = 0x4000000; // 64 MB FastOS_FileInterface::FastOS_FileInterface(const char *filename) : _fAdviseOptions(_defaultFAdviseOptions), _writeChunkSize(MAX_WRITE_CHUNK_SIZE), - _filename(nullptr), + _filename(), _openFlags(0), _directIOEnabled(false), _syncWritesEnabled(false) @@ -49,10 +49,7 @@ FastOS_FileInterface::FastOS_FileInterface(const char *filename) } -FastOS_FileInterface::~FastOS_FileInterface() -{ - free(_filename); -} +FastOS_FileInterface::~FastOS_FileInterface() = default; bool FastOS_FileInterface::InitializeClass () { @@ -358,18 +355,14 @@ FastOS_FileInterface::MakeDirIfNotPresentOrExit(const char *name) void FastOS_FileInterface::SetFileName(const char *filename) { - if (_filename != nullptr) { - free(_filename); - } - - _filename = strdup(filename); + _filename = filename; } const char * FastOS_FileInterface::GetFileName() const { - return (_filename != nullptr) ? _filename : ""; + return _filename.c_str(); } @@ -502,11 +495,8 @@ void FastOS_FileInterface::dropFromCache() const } FastOS_DirectoryScanInterface::FastOS_DirectoryScanInterface(const char *path) - : _searchPath(strdup(path)) + : _searchPath(path) { } -FastOS_DirectoryScanInterface::~FastOS_DirectoryScanInterface() -{ - free(_searchPath); -} +FastOS_DirectoryScanInterface::~FastOS_DirectoryScanInterface() = default; diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h index 40b33e49b35..2d83a1766f0 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/fastos/src/vespa/fastos/file.h @@ -88,7 +88,7 @@ private: void WriteBufInternal(const void *buffer, size_t length); protected: - char *_filename; + std::string _filename; unsigned int _openFlags; bool _directIOEnabled; bool _syncWritesEnabled; @@ -726,7 +726,7 @@ private: FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); protected: - char *_searchPath; + std::string _searchPath; public: @@ -750,7 +750,7 @@ public: * This is an internal copy of the path specified in the constructor. * @return Search path string. */ - const char *GetSearchPath () { return _searchPath; } + const char *GetSearchPath () { return _searchPath.c_str(); } /** * Read the next entry in the directory scan. Failure indicates diff --git a/fastos/src/vespa/fastos/process.cpp b/fastos/src/vespa/fastos/process.cpp index 29c53fe9326..332d82c6aad 100644 --- a/fastos/src/vespa/fastos/process.cpp +++ b/fastos/src/vespa/fastos/process.cpp @@ -1,14 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "process.h" -#include <cstring> FastOS_ProcessInterface::FastOS_ProcessInterface (const char *cmdLine, bool pipeStdin, FastOS_ProcessRedirectListener *stdoutListener, FastOS_ProcessRedirectListener *stderrListener, int bufferSize) : - _cmdLine(nullptr), + _cmdLine(cmdLine), _pipeStdin(pipeStdin), _stdoutListener(stdoutListener), _stderrListener(stderrListener), @@ -16,10 +15,6 @@ FastOS_ProcessInterface::FastOS_ProcessInterface (const char *cmdLine, _next(nullptr), _prev(nullptr) { - _cmdLine = strdup(cmdLine); } -FastOS_ProcessInterface::~FastOS_ProcessInterface () -{ - free (_cmdLine); -} +FastOS_ProcessInterface::~FastOS_ProcessInterface () = default; diff --git a/fastos/src/vespa/fastos/process.h b/fastos/src/vespa/fastos/process.h index 99f045d2f56..25d5224817a 100644 --- a/fastos/src/vespa/fastos/process.h +++ b/fastos/src/vespa/fastos/process.h @@ -12,6 +12,7 @@ #include "types.h" #include <cstddef> +#include <string> /** * This class serves as a sink for redirected (piped) output from @@ -52,8 +53,8 @@ private: protected: - char *_cmdLine; - bool _pipeStdin; + std::string _cmdLine; + bool _pipeStdin; FastOS_ProcessRedirectListener *_stdoutListener; FastOS_ProcessRedirectListener *_stderrListener; @@ -179,10 +180,7 @@ public: * Get command line string. * @return Command line string */ - const char *GetCommandLine () - { - return _cmdLine; - } + const char *GetCommandLine () const { return _cmdLine.c_str(); } }; #include <vespa/fastos/unix_process.h> diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/fastos/src/vespa/fastos/unix_file.cpp index 2ef8c2f55ff..8dd589d5144 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/fastos/src/vespa/fastos/unix_file.cpp @@ -258,7 +258,7 @@ FastOS_UNIX_File::Open(unsigned int openFlags, const char *filename) } unsigned int accessFlags = CalcAccessFlags(openFlags); - _filedes = open(_filename, accessFlags, 0664); + _filedes = open(_filename.c_str(), accessFlags, 0664); rc = (_filedes != -1); @@ -386,10 +386,9 @@ FastOS_UNIX_File::Delete(const char *name) bool FastOS_UNIX_File::Delete(void) { - assert(!IsOpened()); - assert(_filename != nullptr); + assert( ! IsOpened()); - return (unlink(_filename) == 0); + return (unlink(_filename.c_str()) == 0); } bool FastOS_UNIX_File::Rename (const char *currentFileName, const char *newFileName) diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java index aab2f5a53fd..b2efd35e41e 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java @@ -29,7 +29,7 @@ public class FileDownloader implements AutoCloseable { private static final Logger log = Logger.getLogger(FileDownloader.class.getName()); private static final Duration defaultTimeout = Duration.ofMinutes(3); - private static final Duration defaultSleepBetweenRetries = Duration.ofSeconds(10); + private static final Duration defaultSleepBetweenRetries = Duration.ofSeconds(5); public static final File defaultDownloadDirectory = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution")); private final ConnectionPool connectionPool; @@ -47,8 +47,8 @@ public class FileDownloader implements AutoCloseable { this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries); } - public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory) { - this(connectionPool, supervisor, downloadDirectory, defaultTimeout, defaultSleepBetweenRetries); + public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) { + this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries); } public FileDownloader(ConnectionPool connectionPool, @@ -69,8 +69,8 @@ public class FileDownloader implements AutoCloseable { downloadDirectory); } - public Optional<File> getFile(FileReference fileReference) { - return getFile(new FileReferenceDownload(fileReference)); + public Optional<File> getFile(FileReference fileReference, String client) { + return getFile(new FileReferenceDownload(fileReference, client)); } public Optional<File> getFile(FileReferenceDownload fileReferenceDownload) { diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java index 21e35bf67af..796f6ad2ebf 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownload.java @@ -22,6 +22,10 @@ public class FileReferenceDownload { this(fileReference, true, "unknown"); } + public FileReferenceDownload(FileReference fileReference, String client) { + this(fileReference, true, client); + } + public FileReferenceDownload(FileReference fileReference, boolean downloadFromOtherSourceIfNotFound, String client) { Objects.requireNonNull(fileReference, "file reference cannot be null"); this.fileReference = fileReference; diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java index 7b24098526c..e3edee2956f 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java @@ -80,7 +80,7 @@ public class FileReferenceDownloader { Optional<FileReferenceDownload> inProgress = downloads.get(fileReference); if (inProgress.isPresent()) return inProgress.get().future(); - log.log(Level.FINE, () -> "Will download file reference '" + fileReference.value() + "' with timeout " + downloadTimeout); + log.log(Level.FINE, () -> "Will download " + fileReference + " with timeout " + downloadTimeout); downloads.add(fileReferenceDownload); downloadExecutor.submit(() -> waitUntilDownloadStarted(fileReferenceDownload)); return fileReferenceDownload.future(); @@ -92,23 +92,25 @@ public class FileReferenceDownloader { private boolean startDownloadRpc(FileReferenceDownload fileReferenceDownload, int retryCount, Connection connection) { Request request = createRequest(fileReferenceDownload); - connection.invokeSync(request, rpcTimeout(retryCount).getSeconds()); + Duration rpcTimeout = rpcTimeout(retryCount); + connection.invokeSync(request, rpcTimeout.getSeconds()); Level logLevel = (retryCount > 3 ? Level.INFO : Level.FINE); FileReference fileReference = fileReferenceDownload.fileReference(); if (validateResponse(request)) { log.log(Level.FINE, () -> "Request callback, OK. Req: " + request + "\nSpec: " + connection); if (request.returnValues().get(0).asInt32() == 0) { - log.log(Level.FINE, () -> "Found '" + fileReference + "' available at " + connection.getAddress()); + log.log(Level.FINE, () -> "Found " + fileReference + " available at " + connection.getAddress()); return true; } else { - log.log(logLevel, "'" + fileReference + "' not found at " + connection.getAddress()); + log.log(logLevel, fileReference + " not found at " + connection.getAddress()); return false; } } else { log.log(logLevel, "Downloading " + fileReference + " from " + connection.getAddress() + " failed: " + - request + ", error: " + request.errorMessage() + ", will switch config server for next request" + - " (retry " + retryCount + ", rpc timeout " + rpcTimeout(retryCount)); + request + ", error: " + request.errorCode() + "(" + request.errorMessage() + + "). Will switch config server for next request" + + " (retry " + retryCount + ", rpc timeout " + rpcTimeout + ")"); return false; } } diff --git a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java index 97b948ef5d4..460a1ee593a 100644 --- a/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java +++ b/filedistribution/src/test/java/com/yahoo/vespa/filedistribution/FileDownloaderTest.java @@ -79,7 +79,7 @@ public class FileDownloaderTest { fileDownloader.downloads().completedDownloading(fileReference, fileReferenceFullPath); // Check that we get correct path and content when asking for file reference - Optional<File> pathToFile = fileDownloader.getFile(fileReference); + Optional<File> pathToFile = getFile(fileReference); assertTrue(pathToFile.isPresent()); String downloadedFile = new File(fileReferenceFullPath, filename).getAbsolutePath(); assertEquals(new File(fileReferenceFullPath, filename).getAbsolutePath(), downloadedFile); @@ -96,7 +96,7 @@ public class FileDownloaderTest { FileReference fileReference = new FileReference("bar"); File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); - assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + assertFalse(fileReferenceFullPath.getAbsolutePath(), getFile(fileReference).isPresent()); // Verify download status when unable to download assertDownloadStatus(fileReference, 0.0); @@ -107,7 +107,7 @@ public class FileDownloaderTest { FileReference fileReference = new FileReference("baz"); File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); - assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + assertFalse(fileReferenceFullPath.getAbsolutePath(), getFile(fileReference).isPresent()); // Verify download status assertDownloadStatus(fileReference, 0.0); @@ -115,7 +115,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); - Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + Optional<File> downloadedFile = getFile(fileReference); assertTrue(downloadedFile.isPresent()); File downloadedFileFullPath = new File(fileReferenceFullPath, filename); @@ -132,7 +132,7 @@ public class FileDownloaderTest { FileReference fileReference = new FileReference("fileReferenceToDirWithManyFiles"); File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); - assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + assertFalse(fileReferenceFullPath.getAbsolutePath(), getFile(fileReference).isPresent()); // Verify download status assertDownloadStatus(fileReference, 0.0); @@ -150,7 +150,7 @@ public class FileDownloaderTest { File tarFile = CompressedFileReference.compress(tempPath.toFile(), Arrays.asList(fooFile, barFile), new File(tempPath.toFile(), filename)); byte[] tarredContent = IOUtils.readFileBytes(tarFile); receiveFile(fileReference, filename, FileReferenceData.Type.compressed, tarredContent); - Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + Optional<File> downloadedFile = getFile(fileReference); assertTrue(downloadedFile.isPresent()); File downloadedFoo = new File(fileReferenceFullPath, tempPath.relativize(fooFile.toPath()).toString()); @@ -174,7 +174,7 @@ public class FileDownloaderTest { FileReference fileReference = new FileReference("fileReference"); File fileReferenceFullPath = fileReferenceFullPath(downloadDir, fileReference); - assertFalse(fileReferenceFullPath.getAbsolutePath(), fileDownloader.getFile(fileReference).isPresent()); + assertFalse(fileReferenceFullPath.getAbsolutePath(), getFile(fileReference).isPresent()); // Getting file failed, verify download status and since there was an error is not downloading ATM assertDownloadStatus(fileReference, 0.0); @@ -183,7 +183,7 @@ public class FileDownloaderTest { // Receives fileReference, should return and make it available to caller String filename = "abc.jar"; receiveFile(fileReference, filename, FileReferenceData.Type.file, "some other content"); - Optional<File> downloadedFile = fileDownloader.getFile(fileReference); + Optional<File> downloadedFile = getFile(fileReference); assertTrue(downloadedFile.isPresent()); File downloadedFileFullPath = new File(fileReferenceFullPath, filename); assertEquals(downloadedFileFullPath.getAbsolutePath(), downloadedFile.get().getAbsolutePath()); @@ -244,13 +244,13 @@ public class FileDownloaderTest { // Should download since we do not have the file on disk fileDownloader.downloadIfNeeded(new FileReferenceDownload(xyzzy)); assertTrue(fileDownloader.isDownloading(xyzzy)); - assertFalse(fileDownloader.getFile(xyzzy).isPresent()); + assertFalse(getFile(xyzzy).isPresent()); // Receive files to simulate download receiveFile(xyzzy, "xyzzy.jar", FileReferenceData.Type.file, "content"); // Should not download, since file has already been downloaded fileDownloader.downloadIfNeeded(new FileReferenceDownload(xyzzy)); // and file should be available - assertTrue(fileDownloader.getFile(xyzzy).isPresent()); + assertTrue(getFile(xyzzy).isPresent()); } @Test @@ -296,6 +296,10 @@ public class FileDownloaderTest { fileDownloader.downloads().completedDownloading(fileReference, file); } + private Optional<File> getFile(FileReference fileReference) { + return fileDownloader.getFile(fileReference, "test"); + } + private static class MockConnection implements ConnectionPool, com.yahoo.vespa.config.Connection { private ResponseHandler responseHandler; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 685d2a351c5..89a97f3e17d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -362,7 +362,7 @@ public class Flags { public static final UnboundStringFlag JDK_VERSION = defineStringFlag( "jdk-version", "11", - List.of("hmusum"), "2021-10-25", "2021-11-25", + List.of("hmusum"), "2021-10-25", "2022-01-10", "JDK version to use on host and inside containers. Note application-id dimension only applies for container, " + "while hostname and node type applies for host.", "Takes effect on restart for Docker container and on next host-admin tick for host", @@ -384,6 +384,13 @@ public class Flags { "Takes effect on config server restart", ZONE_ID); + public static final UnboundBooleanFlag CONFIG_PROXY_USE_FILE_DISTRIBUTION_CONNECTION_POOL = defineFeatureFlag( + "config-proxy-use-file-distribution-connection-pool", false, + List.of("hmusum"), "2021-11-25", "2021-12-25", + "Whether to use FileDistributionConnectionPool instead of JRTConnectionPool for file downloads in config proxy", + "Takes effect on container reboot", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag USE_V8_GEO_POSITIONS = defineFeatureFlag( "use-v8-geo-positions", false, List.of("arnej"), "2021-11-15", "2022-12-31", diff --git a/fnet/src/tests/info/info.cpp b/fnet/src/tests/info/info.cpp index f2299df839e..4271546e647 100644 --- a/fnet/src/tests/info/info.cpp +++ b/fnet/src/tests/info/info.cpp @@ -77,10 +77,10 @@ TEST("size of important objects") #else constexpr size_t MUTEX_SIZE = 40u; #endif - EXPECT_EQUAL(MUTEX_SIZE + 128u, sizeof(FNET_IOComponent)); + EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 112u, sizeof(FNET_IOComponent)); EXPECT_EQUAL(32u, sizeof(FNET_Channel)); EXPECT_EQUAL(40u, sizeof(FNET_PacketQueue_NoLock)); - EXPECT_EQUAL(MUTEX_SIZE + 432u, sizeof(FNET_Connection)); + EXPECT_EQUAL(MUTEX_SIZE + sizeof(std::string) + 416u, sizeof(FNET_Connection)); EXPECT_EQUAL(48u, sizeof(std::condition_variable)); EXPECT_EQUAL(56u, sizeof(FNET_DataBuffer)); EXPECT_EQUAL(8u, sizeof(FNET_Context)); diff --git a/fnet/src/vespa/fnet/connection.h b/fnet/src/vespa/fnet/connection.h index 6efb147d37f..e86b670b7e5 100644 --- a/fnet/src/vespa/fnet/connection.h +++ b/fnet/src/vespa/fnet/connection.h @@ -53,7 +53,7 @@ public: class FNET_Connection : public FNET_IOComponent { public: - enum State { + enum State : uint8_t { FNET_CONNECTING, FNET_CONNECTED, FNET_CLOSING, @@ -118,9 +118,6 @@ private: static std::atomic<uint64_t> _num_connections; // total number of connections - FNET_Connection(const FNET_Connection &); - FNET_Connection &operator=(const FNET_Connection &); - /** * Get next ID that may be used for multiplexing on this connection. @@ -245,6 +242,8 @@ private: */ vespalib::string GetPeerSpec() const; public: + FNET_Connection(const FNET_Connection &) = delete; + FNET_Connection &operator=(const FNET_Connection &) = delete; /** * Construct a connection in server aspect. diff --git a/fnet/src/vespa/fnet/frt/reflection.cpp b/fnet/src/vespa/fnet/frt/reflection.cpp index 0719c8b4c71..211e681df94 100644 --- a/fnet/src/vespa/fnet/frt/reflection.cpp +++ b/fnet/src/vespa/fnet/frt/reflection.cpp @@ -9,42 +9,30 @@ FRT_Method::FRT_Method(const char * name, const char * paramSpec, const char * r FRT_METHOD_PT method, FRT_Invokable * handler) : _hashNext(nullptr), _listNext(nullptr), - _name(strdup(name)), - _paramSpec(strdup(paramSpec)), - _returnSpec(strdup(returnSpec)), + _name(name), + _paramSpec(paramSpec), + _returnSpec(returnSpec), _method(method), _handler(handler), - _docLen(0), - _doc(nullptr) + _doc() { - assert(_name != nullptr); - assert(_paramSpec != nullptr); - assert(_returnSpec != nullptr); } -FRT_Method::~FRT_Method() { - free(_name); - free(_paramSpec); - free(_returnSpec); - free(_doc); -} +FRT_Method::~FRT_Method() = default; void FRT_Method::SetDocumentation(FRT_Values *values) { - free(_doc); - _docLen = values->GetLength(); - _doc = (char *) malloc(_docLen); - assert(_doc != nullptr); + _doc.resize(values->GetLength()); - FNET_DataBuffer buf(_doc, _docLen); + FNET_DataBuffer buf(&_doc[0], _doc.size()); values->EncodeCopy(&buf); } void FRT_Method::GetDocumentation(FRT_Values *values) { - FNET_DataBuffer buf(_doc, _docLen); - buf.FreeToData(_docLen); - values->DecodeCopy(&buf, _docLen); + FNET_DataBuffer buf(&_doc[0], _doc.size()); + buf.FreeToData(_doc.size()); + values->DecodeCopy(&buf, _doc.size()); } FRT_ReflectionManager::FRT_ReflectionManager() diff --git a/fnet/src/vespa/fnet/frt/reflection.h b/fnet/src/vespa/fnet/frt/reflection.h index c867bbb45ec..6267cafeeb1 100644 --- a/fnet/src/vespa/fnet/frt/reflection.h +++ b/fnet/src/vespa/fnet/frt/reflection.h @@ -3,7 +3,8 @@ #pragma once #include "invokable.h" -#include <cstdint> +#include <string> +#include <vector> class FRT_Values; class FRT_Supervisor; @@ -14,20 +15,18 @@ class FRT_Method friend class FRT_ReflectionManager; private: - FRT_Method *_hashNext; // list of methods in hash bucket - FRT_Method *_listNext; // list of all methods - char *_name; // method name - char *_paramSpec; // method parameter spec - char *_returnSpec; // method return spec - FRT_METHOD_PT _method; // method pointer - FRT_Invokable *_handler; // method handler - uint32_t _docLen; // method documentation length - char *_doc; // method documentation - - FRT_Method(const FRT_Method &); - FRT_Method &operator=(const FRT_Method &); + FRT_Method *_hashNext; // list of methods in hash bucket + FRT_Method *_listNext; // list of all methods + std::string _name; // method name + std::string _paramSpec; // method parameter spec + std::string _returnSpec; // method return spec + FRT_METHOD_PT _method; // method pointer + FRT_Invokable *_handler; // method handler + std::vector<char> _doc; // method documentation public: + FRT_Method(const FRT_Method &) = delete; + FRT_Method &operator=(const FRT_Method &) = delete; FRT_Method(const char *name, const char *paramSpec, const char *returnSpec, @@ -37,9 +36,9 @@ public: ~FRT_Method(); FRT_Method *GetNext() { return _listNext; } - const char *GetName() { return _name; } - const char *GetParamSpec() { return _paramSpec; } - const char *GetReturnSpec() { return _returnSpec; } + const char *GetName() { return _name.c_str(); } + const char *GetParamSpec() { return _paramSpec.c_str(); } + const char *GetReturnSpec() { return _returnSpec.c_str(); } FRT_METHOD_PT GetMethod() { return _method; } FRT_Invokable *GetHandler() { return _handler; } void SetDocumentation(FRT_Values *values); diff --git a/fnet/src/vespa/fnet/iocomponent.cpp b/fnet/src/vespa/fnet/iocomponent.cpp index eeda3e12bea..f08718c0c5c 100644 --- a/fnet/src/vespa/fnet/iocomponent.cpp +++ b/fnet/src/vespa/fnet/iocomponent.cpp @@ -12,23 +12,20 @@ FNET_IOComponent::FNET_IOComponent(FNET_TransportThread *owner, : _ioc_next(nullptr), _ioc_prev(nullptr), _ioc_owner(owner), - _ioc_socket_fd(socket_fd), _ioc_selector(nullptr), - _ioc_spec(nullptr), + _ioc_spec(spec), _flags(shouldTimeOut), + _ioc_socket_fd(socket_fd), + _ioc_refcnt(1), _ioc_timestamp(vespalib::steady_clock::now()), _ioc_lock(), - _ioc_cond(), - _ioc_refcnt(1) + _ioc_cond() { - _ioc_spec = strdup(spec); - assert(_ioc_spec != nullptr); } FNET_IOComponent::~FNET_IOComponent() { - free(_ioc_spec); assert(_ioc_selector == nullptr); } diff --git a/fnet/src/vespa/fnet/iocomponent.h b/fnet/src/vespa/fnet/iocomponent.h index 9220b6dfe8f..b4f061e5bc0 100644 --- a/fnet/src/vespa/fnet/iocomponent.h +++ b/fnet/src/vespa/fnet/iocomponent.h @@ -21,9 +21,6 @@ class FNET_IOComponent { friend class FNET_TransportThread; - FNET_IOComponent(const FNET_IOComponent &); - FNET_IOComponent &operator=(const FNET_IOComponent &); - using Selector = vespalib::Selector<FNET_IOComponent>; struct Flags { @@ -44,16 +41,18 @@ protected: FNET_IOComponent *_ioc_next; // next in list FNET_IOComponent *_ioc_prev; // prev in list FNET_TransportThread *_ioc_owner; // owner(TransportThread) ref. - int _ioc_socket_fd; // source of events. Selector *_ioc_selector; // attached event selector - char *_ioc_spec; // connect/listen spec + std::string _ioc_spec; // connect/listen spec Flags _flags; // Compressed representation of boolean flags; + int _ioc_socket_fd; // source of events. + uint32_t _ioc_refcnt; // reference counter vespalib::steady_time _ioc_timestamp; // last I/O activity std::mutex _ioc_lock; // synchronization std::condition_variable _ioc_cond; // synchronization - uint32_t _ioc_refcnt; // reference counter public: + FNET_IOComponent(const FNET_IOComponent &) = delete; + FNET_IOComponent &operator=(const FNET_IOComponent &) = delete; /** * Construct an IOComponent with the given owner. The socket that @@ -80,7 +79,7 @@ public: /** * @return connect/listen spec **/ - const char *GetSpec() const { return _ioc_spec; } + const char *GetSpec() const { return _ioc_spec.c_str(); } /* * Get a guard to gain exclusive access. diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 66b8cb56443..0dc7aee7cd4 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -36,7 +36,7 @@ <target_jdk_version>11</target_jdk_version> <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> - <junit.version>5.7.0</junit.version> <!-- NOTE: this must be in sync with junit version specified in 'tenant-cd-api' --> + <junit.version>5.8.1</junit.version> <!-- NOTE: this must be in sync with junit version specified in 'tenant-cd-api' --> <test.categories>!integration</test.categories> <!-- To allow specialized base pom to include additional "test provided" dependencies --> @@ -265,9 +265,19 @@ </goals> <configuration> <tasks> - <!-- Creating a dummy file to support running tests with old test runner. Remove when it is no longer in use --> - <mkdir dir="target/application-test/artifacts" /> - <touch file="target/application-test/artifacts/.ignore" /> + <!-- Workaround to copy src/test/application/tests only when its parents exists: + Copy in two steps, eliminating the parents in the helper step--> + + <mkdir dir="target/application-test/src/test/application" /> + <copy todir="target/application-test/"> + <fileset dir="." includes="src/test/application/tests/**" /> + </copy> + + <copy todir="target/application-test/"> + <fileset dir="target/application-test/src/test/application" includes="tests/**" /> + </copy> + <delete dir="target/application-test/src" /> + <copy file="target/${project.artifactId}-tests.jar" todir="target/application-test/components/" /> <zip destfile="target/application-test.zip" basedir="target/application-test/" /> </tasks> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/test/MockMetric.java b/jdisc_core/src/main/java/com/yahoo/jdisc/test/MockMetric.java index cb89320e580..bca6e7082c9 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/test/MockMetric.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/test/MockMetric.java @@ -35,6 +35,13 @@ public class MockMetric implements Metric { public Map<String, Map<Map<String, ?>, Double>> metrics() { return metrics; } + @Override + public String toString() { + return "MockMetric{" + + "metrics=" + metrics + + '}'; + } + private static class MapContext implements Context { private static final MapContext empty = new MapContext(Map.of()); diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 15b5bf81670..ed2ce3d638e 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -425,8 +425,7 @@ RPCNetwork::shutdown() { _transport->ShutDown(true); _threadPool->Close(); - _executor->shutdown(); - _executor->sync(); + _executor->shutdown().sync(); } void diff --git a/parent/pom.xml b/parent/pom.xml index 22925af4502..97c991d9693 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -812,6 +812,11 @@ </dependency> <dependency> <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>${junit.version}</version> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> <version>${junit.version}</version> </dependency> @@ -897,7 +902,7 @@ <commons.math3.version>3.6.1</commons.math3.version> <gson.version>2.8.9</gson.version> <jna.version>5.9.0</jna.version> - <junit.version>5.7.0</junit.version> + <junit.version>5.8.1</junit.version> <maven-assembly-plugin.version>3.3.0</maven-assembly-plugin.version> <maven-bundle-plugin.version>5.1.2</maven-bundle-plugin.version> <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 997422f3822..714f4972e12 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -8,6 +8,7 @@ readonly NUM_THREADS=$(( $(nproc) + 2 )) source /etc/profile.d/enable-devtoolset-10.sh source /etc/profile.d/enable-rh-maven35.sh +source /etc/profile.d/enable-rh-git227.sh export MALLOC_ARENA_MAX=1 export MAVEN_OPTS="-Xss1m -Xms128m -Xmx2g" @@ -39,10 +40,10 @@ case $SHOULD_BUILD in mvn -V $VESPA_MAVEN_EXTRA_OPTS install ;; go) - make -C client/go -j ${NUM_THREADS} + make -C client/go ;; *) - make -C client/go -j ${NUM_THREADS} + make -C client/go ./bootstrap.sh java time mvn -V $VESPA_MAVEN_EXTRA_OPTS install cmake3 -DVESPA_UNPRIVILEGED=no . diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 8238eb21831..be9d394a2b6 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -3,15 +3,19 @@ #include <vespa/vespalib/testkit/testapp.h> #include <tests/proton/common/dummydbowner.h> +#include <vespa/config-attributes.h> +#include <vespa/config-bucketspaces.h> #include <vespa/config-imported-fields.h> +#include <vespa/config-indexschema.h> #include <vespa/config-rank-profiles.h> +#include <vespa/config-summary.h> #include <vespa/config-summarymap.h> #include <vespa/document/base/testdocman.h> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/fastos/file.h> #include <vespa/persistence/conformancetest/conformancetest.h> #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/test/make_bucket_space.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> @@ -28,13 +32,10 @@ #include <vespa/searchcore/proton/server/persistencehandlerproxy.h> #include <vespa/searchcore/proton/server/threading_service_config.h> #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> +#include <vespa/searchcore/proton/test/mock_shared_threading_service.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/config-bucketspaces.h> -#include <vespa/config-attributes.h> -#include <vespa/config-indexschema.h> -#include <vespa/config-summary.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/size_literals.h> @@ -174,6 +175,7 @@ private: mutable DummyWireService _metricsWireService; mutable MemoryConfigStores _config_stores; vespalib::ThreadStackExecutor _summaryExecutor; + MockSharedThreadingService _shared_service; storage::spi::dummy::DummyBucketExecutor _bucketExecutor; public: @@ -202,7 +204,7 @@ public: mgr.nextGeneration(0ms); return DocumentDB::create(_baseDir, mgr.getConfig(), _tlsSpec, _queryLimiter, _clock, docType, bucketSpace, *b->getProtonConfigSP(), const_cast<DocumentDBFactory &>(*this), - _summaryExecutor, _summaryExecutor, _bucketExecutor, _tls, _metricsWireService, + _shared_service, _bucketExecutor, _tls, _metricsWireService, _fileHeaderContext, _config_stores.getConfigStore(docType.toString()), std::make_shared<vespalib::ThreadStackExecutor>(16, 128_Ki), HwInfo()); } @@ -218,6 +220,7 @@ DocumentDBFactory::DocumentDBFactory(const vespalib::string &baseDir, int tlsLis _clock(), _metricsWireService(), _summaryExecutor(8, 128_Ki), + _shared_service(_summaryExecutor, _summaryExecutor), _bucketExecutor(2) {} DocumentDBFactory::~DocumentDBFactory() = default; diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index 5c3fe94a8d7..c5a01de6b3b 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -1,45 +1,46 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/proton/common/dummydbowner.h> +#include <vespa/config-bucketspaces.h> #include <vespa/config/helper/configgetter.hpp> +#include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/test/make_bucket_space.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/test/make_bucket_space.h> +#include <vespa/eval/eval/value.h> +#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/attribute_writer.h> -#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchcore/proton/docsummary/docsumcontext.h> #include <vespa/searchcore/proton/docsummary/documentstoreadapter.h> #include <vespa/searchcore/proton/docsummary/summarymanager.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/feedoperation/putoperation.h> +#include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> #include <vespa/searchcore/proton/server/documentdb.h> -#include <vespa/searchcore/proton/server/feedhandler.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> +#include <vespa/searchcore/proton/server/feedhandler.h> #include <vespa/searchcore/proton/server/idocumentsubdb.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/server/searchview.h> #include <vespa/searchcore/proton/server/summaryadapter.h> -#include <vespa/searchcore/proton/matching/querylimiter.h> -#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> -#include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> +#include <vespa/searchcore/proton/test/mock_shared_threading_service.h> #include <vespa/searchlib/engine/docsumapi.h> #include <vespa/searchlib/index/docbuilder.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <vespa/searchlib/transactionlog/nosyncproxy.h> #include <vespa/searchlib/transactionlog/translogserver.h> -#include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/data/slime/json_format.h> #include <vespa/vespalib/data/simple_buffer.h> +#include <vespa/vespalib/data/slime/json_format.h> +#include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/encoding/base64.h> -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/config-bucketspaces.h> #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/vespalib/util/size_literals.h> #include <regex> #include <vespa/log/log.h> @@ -176,6 +177,7 @@ public: DummyFileHeaderContext _fileHeaderContext; TransLogServer _tls; vespalib::ThreadStackExecutor _summaryExecutor; + MockSharedThreadingService _shared_service; storage::spi::dummy::DummyBucketExecutor _bucketExecutor; bool _mkdirOk; matching::QueryLimiter _queryLimiter; @@ -196,6 +198,7 @@ public: _fileHeaderContext(), _tls("tmp", 9013, ".", _fileHeaderContext), _summaryExecutor(8, 128_Ki), + _shared_service(_summaryExecutor, _summaryExecutor), _bucketExecutor(2), _mkdirOk(FastOS_File::MakeDirectory("tmpdb")), _queryLimiter(), @@ -224,7 +227,7 @@ public: } _ddb = DocumentDB::create("tmpdb", _configMgr.getConfig(), "tcp/localhost:9013", _queryLimiter, _clock, DocTypeName(docTypeName), makeBucketSpace(), *b->getProtonConfigSP(), *this, - _summaryExecutor, _summaryExecutor, _bucketExecutor, _tls, _dummy, _fileHeaderContext, + _shared_service, _bucketExecutor, _tls, _dummy, _fileHeaderContext, std::make_unique<MemoryConfigStore>(), std::make_shared<vespalib::ThreadStackExecutor>(16, 128_Ki), _hwInfo), _ddb->start(); diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index e3e6ac6321e..b80cd08ae8e 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -234,8 +234,7 @@ MySearchableContext::MySearchableContext(IThreadingService &writeService, IBucketDBHandlerInitializer & bucketDBHandlerInitializer) : _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer), _queryLimiter(), _clock(), - _ctx(_fastUpdCtx._ctx, _queryLimiter, - _clock, dynamic_cast<vespalib::SyncableThreadExecutor &>(writeService.shared())) + _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, writeService.shared()) {} MySearchableContext::~MySearchableContext() = default; diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 2c21a30396d..9bc374b8386 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -35,18 +35,18 @@ struct ControllerFixtureBase : public ::testing::Test test::BucketHandler _bucketHandler; MyBucketModifiedHandler _modifiedHandler; std::shared_ptr<bucketdb::BucketDBOwner> _bucketDB; - MySubDb _ready; - MySubDb _notReady; - BucketCreateNotifier _bucketCreateNotifier; - test::DiskMemUsageNotifier _diskMemUsageNotifier; - MonitoredRefCount _refCount; - ThreadStackExecutor _singleExecutor; - ExecutorThreadService _master; - DummyBucketExecutor _bucketExecutor; - MyMoveHandler _moveHandler; - DocumentDBTaggedMetrics _metrics; + MySubDb _ready; + MySubDb _notReady; + BucketCreateNotifier _bucketCreateNotifier; + test::DiskMemUsageNotifier _diskMemUsageNotifier; + MonitoredRefCount _refCount; + ThreadStackExecutor _singleExecutor; + SyncableExecutorThreadService _master; + DummyBucketExecutor _bucketExecutor; + MyMoveHandler _moveHandler; + DocumentDBTaggedMetrics _metrics; std::shared_ptr<BucketMoveJob> _bmj; - MyCountJobRunner _runner; + MyCountJobRunner _runner; ControllerFixtureBase(const BlockableMaintenanceJobConfig &blockableConfig, bool storeMoveDoneContexts); ~ControllerFixtureBase(); ControllerFixtureBase &addReady(const BucketId &bucket) { diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index a24eeb262ab..b31534c011c 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -1,10 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <tests/proton/common/dummydbowner.h> +#include <vespa/config-bucketspaces.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/repo/documenttyperepo.h> -#include <vespa/fastos/file.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/fastos/file.h> +#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/flushableattribute.h> #include <vespa/searchcore/proton/common/statusreport.h> #include <vespa/searchcore/proton/docsummary/summaryflushtarget.h> @@ -22,17 +24,16 @@ #include <vespa/searchcore/proton/server/feedhandler.h> #include <vespa/searchcore/proton/server/fileconfigmanager.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> -#include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> +#include <vespa/searchcore/proton/test/mock_shared_threading_service.h> #include <vespa/searchcorespi/index/indexflushtarget.h> #include <vespa/searchlib/attribute/attribute_read_guard.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/vespalib/data/slime/slime.h> -#include <vespa/vespalib/util/size_literals.h> -#include <vespa/config-bucketspaces.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/size_literals.h> #include <iostream> using namespace cloud::config::filedistribution; @@ -118,6 +119,7 @@ struct Fixture : public FixtureBase { DummyWireService _dummy; MyDBOwner _myDBOwner; vespalib::ThreadStackExecutor _summaryExecutor; + MockSharedThreadingService _shared_service; HwInfo _hwInfo; storage::spi::dummy::DummyBucketExecutor _bucketExecutor; DocumentDB::SP _db; @@ -142,6 +144,7 @@ Fixture::Fixture(bool file_config) _dummy(), _myDBOwner(), _summaryExecutor(8, 128_Ki), + _shared_service(_summaryExecutor, _summaryExecutor), _hwInfo(), _bucketExecutor(2), _db(), @@ -165,7 +168,7 @@ Fixture::Fixture(bool file_config) mgr.nextGeneration(0ms); _db = DocumentDB::create(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"), makeBucketSpace(), - *b->getProtonConfigSP(), _myDBOwner, _summaryExecutor, _summaryExecutor, _bucketExecutor, _tls, _dummy, + *b->getProtonConfigSP(), _myDBOwner, _shared_service, _bucketExecutor, _tls, _dummy, _fileHeaderContext, make_config_store(), std::make_shared<vespalib::ThreadStackExecutor>(16, 128_Ki), _hwInfo); _db->start(); diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index bffedfd8dab..f76b7d03d08 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -747,6 +747,20 @@ TEST_F("require that put with different document type repo is ok", FeedHandlerFi EXPECT_EQUAL(1, f.tls_writer.store_count); } +TEST_F("require that feed stats are updated", FeedHandlerFixture) +{ + DocumentContext doc_context("id:ns:searchdocument::foo", *f.schema.builder); + auto op =std::make_unique<PutOperation>(doc_context.bucketId, Timestamp(10), std::move(doc_context.doc)); + FeedTokenContext token_context; + f.handler.performOperation(std::move(token_context.token), std::move(op)); + f.syncMaster(); // wait for initateCommit + f.syncMaster(); // wait for onCommitDone + auto stats = f.handler.get_stats(false); + EXPECT_EQUAL(1u, stats.get_commits()); + EXPECT_EQUAL(1u, stats.get_operations()); + EXPECT_LESS(0.0, stats.get_total_latency()); +} + using namespace document; TEST_F("require that update with a fieldpath update will be rejected", SchemaContext) { diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index a030ce1c455..fb9b10aa5a2 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -514,7 +514,8 @@ struct FixtureBase template <typename FunctionType> void runInMasterAndSyncAll(FunctionType func) { - test::runInMasterAndSyncAll(_writeService, func); + test::runInMaster(_writeService, func); + _writeServiceReal.sync_all_executors(); } template <typename FunctionType> void runInMaster(FunctionType func) { diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp index 13955953eb5..8f88d678c0c 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.cpp @@ -54,7 +54,7 @@ JobTestBase::init(uint32_t allowedLidBloat, _job.reset(); _singleExecutor = std::make_unique<vespalib::ThreadStackExecutor>(1, 0x10000); - _master = std::make_unique<proton::ExecutorThreadService> (*_singleExecutor); + _master = std::make_unique<proton::SyncableExecutorThreadService> (*_singleExecutor); _bucketExecutor = std::make_unique<storage::spi::dummy::DummyBucketExecutor>(4); _job = lidspace::CompactionJob::create(compactCfg, RetainGuard(_refCount), _handler, _storer, *_master, *_bucketExecutor, _diskMemUsageNotifier, blockableCfg, _clusterStateHandler, nodeRetired, diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h index 14f2ff42dbe..5875910f4d9 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_jobtest.h @@ -14,7 +14,7 @@ struct JobTestBase : public ::testing::Test { test::DiskMemUsageNotifier _diskMemUsageNotifier; std::unique_ptr<storage::spi::dummy::DummyBucketExecutor> _bucketExecutor; std::unique_ptr<vespalib::SyncableThreadExecutor> _singleExecutor; - std::unique_ptr<searchcorespi::index::IThreadService> _master; + std::unique_ptr<searchcorespi::index::ISyncableThreadService> _master; std::shared_ptr<MyHandler> _handler; MyStorer _storer; std::shared_ptr<BlockableMaintenanceJob> _job; diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 8940b01b91d..227e885564d 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -326,7 +326,7 @@ class MaintenanceControllerFixture public: MyExecutor _executor; MyExecutor _genericExecutor; - ExecutorThreadService _threadService; + SyncableExecutorThreadService _threadService; DummyBucketExecutor _bucketExecutor; DocTypeName _docTypeName; test::UserDocumentsBuilder _builder; diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index 7ccaa6e9fbf..526b94a3525 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -242,7 +242,8 @@ struct FixtureBase { template <typename FunctionType> void runInMasterAndSyncAll(FunctionType func) { - test::runInMasterAndSyncAll(writeService, func); + test::runInMaster(writeService, func); + writeService.sync_all_executors(); } template <typename FunctionType> void runInMasterAndSync(FunctionType func) { diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 8265585a038..8d8872b8998 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -137,7 +137,8 @@ public: template <typename FunctionType> void runInMasterAndSyncAll(FunctionType func) { - test::runInMasterAndSyncAll(_writeService, func); + test::runInMaster(_writeService, func); + _writeServiceReal.sync_all_executors(); } template <typename FunctionType> diff --git a/searchcore/src/tests/proton/index/indexcollection_test.cpp b/searchcore/src/tests/proton/index/indexcollection_test.cpp index 07fbacde49a..70141f057bf 100644 --- a/searchcore/src/tests/proton/index/indexcollection_test.cpp +++ b/searchcore/src/tests/proton/index/indexcollection_test.cpp @@ -25,7 +25,7 @@ public: MockIndexSearchable() : _field_length_info() {} - MockIndexSearchable(const FieldLengthInfo& field_length_info) + explicit MockIndexSearchable(const FieldLengthInfo& field_length_info) : _field_length_info(field_length_info) {} FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const override { @@ -79,17 +79,17 @@ public: return std::make_unique<WarmupIndexCollection>(WarmupConfig(1s, false), prev, next, *_warmup, _executor, *this); } - virtual void warmupDone(ISearchableIndexCollection::SP current) override { + void warmupDone(std::shared_ptr<WarmupIndexCollection> current) override { (void) current; } IndexCollectionTest() - : _selector(new FixedSourceSelector(0, "fs1")), - _source1(new MockIndexSearchable({3, 5})), - _source2(new MockIndexSearchable({7, 11})), - _fusion_source(new FakeIndexSearchable), + : _selector(std::make_shared<FixedSourceSelector>(0, "fs1")), + _source1(std::make_shared<MockIndexSearchable>(FieldLengthInfo(3, 5))), + _source2(std::make_shared<MockIndexSearchable>(FieldLengthInfo(7, 11))), + _fusion_source(std::make_shared<FakeIndexSearchable>()), _executor(1, 128_Ki), - _warmup(new FakeIndexSearchable) + _warmup(std::make_shared<FakeIndexSearchable>()) {} ~IndexCollectionTest() = default; }; diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index d6bbc77aa09..1e33482b055 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -210,8 +210,8 @@ IndexManagerTest::resetIndexManager() { _index_manager.reset(); _index_manager = std::make_unique<IndexManager>(index_dir, IndexConfig(), getSchema(), 1, - _reconfigurer, _writeService, _writeService.master(), - TuneFileIndexManager(), TuneFileAttributes(),_fileHeaderContext); + _reconfigurer, _writeService, _sharedExecutor, + TuneFileIndexManager(), TuneFileAttributes(), _fileHeaderContext); } void diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index 0882153edd6..62d86ce895d 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -1,19 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "bm_node.h" #include "bm_cluster.h" #include "bm_cluster_params.h" #include "bm_message_bus.h" +#include "bm_node.h" #include "bm_node_stats.h" #include "bm_storage_chain_builder.h" #include "bm_storage_link_context.h" -#include "storage_api_chain_bm_feed_handler.h" -#include "storage_api_message_bus_bm_feed_handler.h" -#include "storage_api_rpc_bm_feed_handler.h" #include "document_api_message_bus_bm_feed_handler.h" #include "i_bm_distribution.h" #include "i_bm_feed_handler.h" #include "spi_bm_feed_handler.h" +#include "storage_api_chain_bm_feed_handler.h" +#include "storage_api_message_bus_bm_feed_handler.h" +#include "storage_api_rpc_bm_feed_handler.h" +#include <tests/proton/common/dummydbowner.h> #include <vespa/config-attributes.h> #include <vespa/config-bucketspaces.h> #include <vespa/config-imported-fields.h> @@ -39,18 +40,19 @@ #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metricswireservice.h> -#include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> +#include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h> #include <vespa/searchcore/proton/persistenceengine/persistenceengine.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> -#include <vespa/searchcore/proton/server/documentdb.h> #include <vespa/searchcore/proton/server/document_db_maintenance_config.h> #include <vespa/searchcore/proton/server/document_meta_store_read_guards.h> +#include <vespa/searchcore/proton/server/documentdb.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/fileconfigmanager.h> #include <vespa/searchcore/proton/server/memoryconfigstore.h> #include <vespa/searchcore/proton/server/persistencehandlerproxy.h> #include <vespa/searchcore/proton/test/disk_mem_usage_notifier.h> +#include <vespa/searchcore/proton/test/mock_shared_threading_service.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/translogserver.h> #include <vespa/searchsummary/config/config-juniperrc.h> @@ -75,7 +77,6 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> -#include <tests/proton/common/dummydbowner.h> #include <vespa/log/log.h> LOG_SETUP(".bmcluster.bm_node"); @@ -459,6 +460,7 @@ class MyBmNode : public BmNode proton::DummyWireService _metrics_wire_service; proton::MemoryConfigStores _config_stores; vespalib::ThreadStackExecutor _summary_executor; + proton::MockSharedThreadingService _shared_service; proton::DummyDBOwner _document_db_owner; BucketSpace _bucket_space; std::shared_ptr<DocumentDB> _document_db; @@ -523,6 +525,7 @@ MyBmNode::MyBmNode(const vespalib::string& base_dir, int base_port, uint32_t nod _metrics_wire_service(), _config_stores(), _summary_executor(8, 128_Ki), + _shared_service(_summary_executor, _summary_executor), _document_db_owner(), _bucket_space(document::test::makeBucketSpace(_doc_type_name.getName())), _document_db(), @@ -594,7 +597,7 @@ MyBmNode::create_document_db(const BmClusterParams& params) mgr.nextGeneration(0ms); _document_db = DocumentDB::create(_base_dir, mgr.getConfig(), _tls_spec, _query_limiter, _clock, _doc_type_name, _bucket_space, *bootstrap_config->getProtonConfigSP(), _document_db_owner, - _summary_executor, _summary_executor, *_persistence_engine, _tls, + _shared_service, *_persistence_engine, _tls, _metrics_wire_service, _file_header_context, _config_stores.getConfigStore(_doc_type_name.toString()), std::make_shared<vespalib::ThreadStackExecutor>(16, 128_Ki), HwInfo()); diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp index 07580817dc9..6d71b81cb8b 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.cpp @@ -2,7 +2,6 @@ #include "summarycompacttarget.h" #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/searchcorespi/index/i_thread_service.h> #include <future> using search::IDocumentStore; @@ -39,7 +38,7 @@ public: } -SummaryCompactTarget::SummaryCompactTarget(searchcorespi::index::IThreadService & summaryService, IDocumentStore & docStore) +SummaryCompactTarget::SummaryCompactTarget(vespalib::Executor & summaryService, IDocumentStore & docStore) : IFlushTarget("summary.compact", Type::GC, Component::DOCUMENT_STORE), _summaryService(summaryService), _docStore(docStore), @@ -82,10 +81,4 @@ SummaryCompactTarget::initFlush(SerialNum currentSerial, std::shared_ptr<search: return future.get(); } -uint64_t -SummaryCompactTarget::getApproxBytesToWriteToDisk() const -{ - return 0; -} - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h index a5f39e953a5..c8035a544f2 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarycompacttarget.h @@ -15,23 +15,22 @@ namespace proton { class SummaryCompactTarget : public searchcorespi::IFlushTarget { private: using FlushStats = searchcorespi::FlushStats; - searchcorespi::index::IThreadService &_summaryService; + vespalib::Executor &_summaryService; search::IDocumentStore & _docStore; FlushStats _lastStats; public: - SummaryCompactTarget(searchcorespi::index::IThreadService & summaryService, search::IDocumentStore & docStore); + SummaryCompactTarget(vespalib::Executor & summaryService, search::IDocumentStore & docStore); - // Implements IFlushTarget - virtual MemoryGain getApproxMemoryGain() const override; - virtual DiskGain getApproxDiskGain() const override; - virtual SerialNum getFlushedSerialNum() const override; - virtual Time getLastFlushTime() const override; + MemoryGain getApproxMemoryGain() const override; + DiskGain getApproxDiskGain() const override; + SerialNum getFlushedSerialNum() const override; + Time getLastFlushTime() const override; - virtual Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; + Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; - virtual FlushStats getLastFlushStats() const override { return _lastStats; } - virtual uint64_t getApproxBytesToWriteToDisk() const override; + FlushStats getLastFlushStats() const override { return _lastStats; } + uint64_t getApproxBytesToWriteToDisk() const override { return 0; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp index 7f164af7339..45fc23175bf 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "summaryflushtarget.h" -#include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/vespalib/util/lambdatask.h> using search::IDocumentStore; @@ -28,7 +27,7 @@ public: { _currSerial = _docStore.initFlush(currSerial); } - virtual void run() override { + void run() override { _docStore.flush(_currSerial); updateStats(); } @@ -37,17 +36,13 @@ public: _stats.setPath(_docStore.getBaseDir()); } - virtual SerialNum - getFlushSerial() const override - { - return _currSerial; - } + SerialNum getFlushSerial() const override { return _currSerial; } }; } SummaryFlushTarget::SummaryFlushTarget(IDocumentStore & docStore, - searchcorespi::index::IThreadService & summaryService) + vespalib::Executor & summaryService) : IFlushTarget("summary.flush", Type::SYNC, Component::DOCUMENT_STORE), _docStore(docStore), _summaryService(summaryService), @@ -62,12 +57,6 @@ SummaryFlushTarget::getApproxMemoryGain() const return MemoryGain(_docStore.memoryUsed(), _docStore.memoryMeta()); } -IFlushTarget::DiskGain -SummaryFlushTarget::getApproxDiskGain() const -{ - return DiskGain(0, 0); -} - IFlushTarget::Time SummaryFlushTarget::getLastFlushTime() const { @@ -97,11 +86,4 @@ SummaryFlushTarget::initFlush(SerialNum currentSerial, std::shared_ptr<search::I return future.get(); } -uint64_t -SummaryFlushTarget::getApproxBytesToWriteToDisk() const -{ - return 0; -} - - } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h index 99cfa1a2080..f864b922af8 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summaryflushtarget.h @@ -4,7 +4,6 @@ #include <vespa/searchlib/docstore/idocumentstore.h> #include <vespa/searchcorespi/flush/iflushtarget.h> -namespace searchcorespi::index { struct IThreadService; } namespace proton { /** @@ -14,25 +13,24 @@ class SummaryFlushTarget : public searchcorespi::IFlushTarget { private: using FlushStats = searchcorespi::FlushStats; search::IDocumentStore & _docStore; - searchcorespi::index::IThreadService & _summaryService; - FlushStats _lastStats; + vespalib::Executor & _summaryService; + FlushStats _lastStats; Task::UP internalInitFlush(SerialNum currentSerial); public: SummaryFlushTarget(search::IDocumentStore & docStore, - searchcorespi::index::IThreadService & summaryService); + vespalib::Executor & summaryService); - // Implements IFlushTarget - virtual MemoryGain getApproxMemoryGain() const override; - virtual DiskGain getApproxDiskGain() const override; - virtual SerialNum getFlushedSerialNum() const override; - virtual Time getLastFlushTime() const override; + MemoryGain getApproxMemoryGain() const override; + DiskGain getApproxDiskGain() const override { return DiskGain(0, 0); } + SerialNum getFlushedSerialNum() const override; + Time getLastFlushTime() const override; - virtual Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; + Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; - virtual FlushStats getLastFlushStats() const override { return _lastStats; } - virtual uint64_t getApproxBytesToWriteToDisk() const override; + FlushStats getLastFlushStats() const override { return _lastStats; } + uint64_t getApproxBytesToWriteToDisk() const override { return 0; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp index 4570151d3eb..6f65d76789a 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.cpp @@ -7,7 +7,6 @@ #include <vespa/config/print/ostreamconfigwriter.h> #include <vespa/document/repo/documenttyperepo.h> #include <vespa/juniper/rpinterface.h> -#include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/searchcore/proton/flushengine/shrink_lid_space_flush_target.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchsummary/docsummary/docsumconfig.h> @@ -45,12 +44,12 @@ namespace { class ShrinkSummaryLidSpaceFlushTarget : public ShrinkLidSpaceFlushTarget { using ICompactableLidSpace = search::common::ICompactableLidSpace; - searchcorespi::index::IThreadService & _summaryService; + vespalib::Executor & _summaryService; public: ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, Type type, Component component, SerialNum flushedSerialNum, vespalib::system_time lastFlushTime, - searchcorespi::index::IThreadService & summaryService, + vespalib::Executor & summaryService, std::shared_ptr<ICompactableLidSpace> target); ~ShrinkSummaryLidSpaceFlushTarget() override; Task::UP initFlush(SerialNum currentSerial, std::shared_ptr<search::IFlushToken> flush_token) override; @@ -59,7 +58,7 @@ public: ShrinkSummaryLidSpaceFlushTarget:: ShrinkSummaryLidSpaceFlushTarget(const vespalib::string &name, Type type, Component component, SerialNum flushedSerialNum, vespalib::system_time lastFlushTime, - searchcorespi::index::IThreadService & summaryService, + vespalib::Executor & summaryService, std::shared_ptr<ICompactableLidSpace> target) : ShrinkLidSpaceFlushTarget(name, type, component, flushedSerialNum, lastFlushTime, std::move(target)), _summaryService(summaryService) @@ -153,9 +152,7 @@ SummaryManager::SummaryManager(vespalib::ThreadExecutor & executor, const LogDoc search::IBucketizer::SP bucketizer) : _baseDir(baseDir), _docTypeName(docTypeName), - _docStore(), - _tuneFileSummary(tuneFileSummary), - _currentSerial(0u) + _docStore() { _docStore = std::make_shared<LogDocumentStore>(executor, baseDir, storeConfig, growStrategy, tuneFileSummary, fileHeaderContext, tlSyncer, std::move(bucketizer)); @@ -167,27 +164,24 @@ void SummaryManager::putDocument(uint64_t syncToken, search::DocumentIdT lid, const Document & doc) { _docStore->write(syncToken, lid, doc); - _currentSerial = syncToken; } void SummaryManager::putDocument(uint64_t syncToken, search::DocumentIdT lid, const vespalib::nbostream & doc) { _docStore->write(syncToken, lid, doc); - _currentSerial = syncToken; } void SummaryManager::removeDocument(uint64_t syncToken, search::DocumentIdT lid) { _docStore->remove(syncToken, lid); - _currentSerial = syncToken; } namespace { IFlushTarget::SP -createShrinkLidSpaceFlushTarget(searchcorespi::index::IThreadService & summaryService, IDocumentStore::SP docStore) +createShrinkLidSpaceFlushTarget(vespalib::Executor & summaryService, IDocumentStore::SP docStore) { return std::make_shared<ShrinkSummaryLidSpaceFlushTarget>("summary.shrink", IFlushTarget::Type::GC, @@ -200,7 +194,8 @@ createShrinkLidSpaceFlushTarget(searchcorespi::index::IThreadService & summarySe } -IFlushTarget::List SummaryManager::getFlushTargets(searchcorespi::index::IThreadService & summaryService) +IFlushTarget::List +SummaryManager::getFlushTargets(vespalib::Executor & summaryService) { IFlushTarget::List ret; ret.push_back(std::make_shared<SummaryFlushTarget>(getBackingStore(), summaryService)); @@ -211,7 +206,8 @@ IFlushTarget::List SummaryManager::getFlushTargets(searchcorespi::index::IThread return ret; } -void SummaryManager::reconfigure(const LogDocumentStore::Config & config) { +void +SummaryManager::reconfigure(const LogDocumentStore::Config & config) { auto & docStore = dynamic_cast<LogDocumentStore &> (*_docStore); docStore.reconfigure(config); } diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h index 9bff8723ff6..b3cbd399262 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/summarymanager.h @@ -12,7 +12,6 @@ #include <vespa/document/fieldvalue/document.h> #include <vespa/vespalib/util/threadexecutor.h> -namespace searchcorespi::index { struct IThreadService; } namespace search { class IBucketizer; } namespace search::common { class FileHeaderContext; } @@ -58,8 +57,6 @@ private: vespalib::string _baseDir; DocTypeName _docTypeName; std::shared_ptr<search::IDocumentStore> _docStore; - const search::TuneFileSummary _tuneFileSummary; - uint64_t _currentSerial; public: typedef std::shared_ptr<SummaryManager> SP; @@ -77,7 +74,7 @@ public: void putDocument(uint64_t syncToken, search::DocumentIdT lid, const document::Document & doc); void putDocument(uint64_t syncToken, search::DocumentIdT lid, const vespalib::nbostream & doc); void removeDocument(uint64_t syncToken, search::DocumentIdT lid); - searchcorespi::IFlushTarget::List getFlushTargets(searchcorespi::index::IThreadService & summaryService); + searchcorespi::IFlushTarget::List getFlushTargets(vespalib::Executor & summaryService); ISummarySetup::SP createSummarySetup(const vespa::config::search::SummaryConfig &summaryCfg, diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 4eaa722e0ba..632f4482654 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -119,7 +119,7 @@ public: /** * Returns the underlying executor. Only used for state explorers. */ - const vespalib::SyncableThreadExecutor& get_executor() const { return _executor; } + const vespalib::ThreadExecutor& get_executor() const { return _executor; } /** * Starts the scheduling thread of this manager. diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp index 9e915779d92..630c536a1ca 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.cpp @@ -15,7 +15,7 @@ IndexManagerInitializer(const vespalib::string &baseDir, search::SerialNum serialNum, searchcorespi::IIndexManager::Reconfigurer & reconfigurer, searchcorespi::index::IThreadingService & threadingService, - vespalib::SyncableThreadExecutor & warmupExecutor, + vespalib::Executor & warmupExecutor, const search::TuneFileIndexManager & tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, diff --git a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h index a7acfb61d54..3cf1daf631e 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h +++ b/searchcore/src/vespa/searchcore/proton/index/index_manager_initializer.h @@ -20,7 +20,7 @@ class IndexManagerInitializer : public initializer::InitializerTask search::SerialNum _serialNum; searchcorespi::IIndexManager::Reconfigurer &_reconfigurer; searchcorespi::index::IThreadingService &_threadingService; - vespalib::SyncableThreadExecutor &_warmupExecutor; + vespalib::Executor &_warmupExecutor; const search::TuneFileIndexManager _tuneFileIndexManager; const search::TuneFileAttributes _tuneFileAttributes; const search::common::FileHeaderContext &_fileHeaderContext; @@ -33,7 +33,7 @@ public: search::SerialNum serialNum, searchcorespi::IIndexManager::Reconfigurer & reconfigurer, searchcorespi::index::IThreadingService & threadingService, - vespalib::SyncableThreadExecutor & warmupExecutor, + vespalib::Executor & warmupExecutor, const search::TuneFileIndexManager & tuneFileIndexManager, const search::TuneFileAttributes & tuneFileAttributes, const search::common::FileHeaderContext & fileHeaderContext, diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp index 169ba149297..de397e81d76 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.cpp @@ -80,7 +80,7 @@ IndexManager::IndexManager(const vespalib::string &baseDir, SerialNum serialNum, Reconfigurer &reconfigurer, IThreadingService &threadingService, - vespalib::SyncableThreadExecutor & warmupExecutor, + vespalib::Executor & warmupExecutor, const search::TuneFileIndexManager &tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, const FileHeaderContext &fileHeaderContext) : diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 4113af30b0d..436b4127804 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -73,7 +73,7 @@ public: SerialNum serialNum, Reconfigurer &reconfigurer, searchcorespi::index::IThreadingService &threadingService, - vespalib::SyncableThreadExecutor & warmupExecutor, + vespalib::Executor & warmupExecutor, const search::TuneFileIndexManager &tuneFileIndexManager, const search::TuneFileAttributes &tuneFileAttributes, const search::common::FileHeaderContext &fileHeaderContext); diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 7cc0c97048b..3d3be775a4a 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -69,7 +69,7 @@ public: /** * Returns the underlying executor. Only used for state explorers. */ - const vespalib::SyncableThreadExecutor& get_executor() const { return _executor; } + const vespalib::ThreadExecutor& get_executor() const { return _executor; } /** * Closes the request handler interface. This will prevent any more data diff --git a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt index 43e01420a22..ba6e5fd1ea5 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt @@ -5,6 +5,8 @@ vespa_add_library(searchcore_proton_metrics STATIC content_proton_metrics.cpp documentdb_job_trackers.cpp documentdb_tagged_metrics.cpp + document_db_commit_metrics.cpp + document_db_feeding_metrics.cpp executor_metrics.cpp executor_threading_service_metrics.cpp executor_threading_service_stats.cpp diff --git a/searchcore/src/vespa/searchcore/proton/metrics/document_db_commit_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/document_db_commit_metrics.cpp new file mode 100644 index 00000000000..c5b7d71a982 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/metrics/document_db_commit_metrics.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_commit_metrics.h" + +namespace proton { + +DocumentDBCommitMetrics::DocumentDBCommitMetrics(metrics::MetricSet* parent) + : MetricSet("commit", {}, "commit metrics for feeding in a document database", parent), + operations("operations", {}, "Number of operations included in a commit", this), + latency("latency", {}, "Latency for commit", this) +{ +} + +DocumentDBCommitMetrics::~DocumentDBCommitMetrics() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/document_db_commit_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/document_db_commit_metrics.h new file mode 100644 index 00000000000..45c826a7ccf --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/metrics/document_db_commit_metrics.h @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/metrics/metricset.h> +#include <vespa/metrics/valuemetric.h> + +namespace proton { + +/* + * Metrics for commits during feeding within a document db. + */ +struct DocumentDBCommitMetrics : metrics::MetricSet +{ + metrics::DoubleAverageMetric operations; + metrics::DoubleAverageMetric latency; + + DocumentDBCommitMetrics(metrics::MetricSet* parent); + ~DocumentDBCommitMetrics() override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/document_db_feeding_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/document_db_feeding_metrics.cpp new file mode 100644 index 00000000000..0c7f8f6f039 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/metrics/document_db_feeding_metrics.cpp @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "document_db_feeding_metrics.h" + +namespace proton { + +DocumentDBFeedingMetrics::DocumentDBFeedingMetrics(metrics::MetricSet* parent) + : MetricSet("feeding", {}, "feeding metrics in a document database", parent), + commit(this) +{ +} + +DocumentDBFeedingMetrics::~DocumentDBFeedingMetrics() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/document_db_feeding_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/document_db_feeding_metrics.h new file mode 100644 index 00000000000..7353cfbdc04 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/metrics/document_db_feeding_metrics.h @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "document_db_commit_metrics.h" + +namespace proton { + +/* + * Metrics for feeding within a document db. + */ +struct DocumentDBFeedingMetrics : metrics::MetricSet +{ + DocumentDBCommitMetrics commit; + + DocumentDBFeedingMetrics(metrics::MetricSet* parent); + ~DocumentDBFeedingMetrics() override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp index 1d947bb003a..e895a03b190 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp @@ -268,6 +268,7 @@ DocumentDBTaggedMetrics::DocumentDBTaggedMetrics(const vespalib::string &docType sessionCache(this), documents(this), bucketMove(this), + feeding(this), totalMemoryUsage(this), totalDiskUsage("disk_usage", {}, "The total disk usage (in bytes) for this document db", this), heart_beat_age("heart_beat_age", {}, "How long ago (in seconds) heart beat maintenace job was run", this), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h index bdedfeea8b9..c9d321d6752 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h @@ -5,6 +5,7 @@ #include "memory_usage_metrics.h" #include "executor_threading_service_metrics.h" #include "sessionmanager_metrics.h" +#include "document_db_feeding_metrics.h" #include <vespa/metrics/metricset.h> #include <vespa/metrics/valuemetric.h> #include <vespa/searchcore/proton/matching/matching_stats.h> @@ -202,6 +203,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet SessionCacheMetrics sessionCache; DocumentsMetrics documents; BucketMoveMetrics bucketMove; + DocumentDBFeedingMetrics feeding; MemoryUsageMetrics totalMemoryUsage; metrics::LongValueMetric totalDiskUsage; metrics::DoubleValueMetric heart_beat_age; diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp index af24dcd976d..63644e5c7ab 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp @@ -7,14 +7,12 @@ namespace proton { ExecutorThreadingServiceStats::ExecutorThreadingServiceStats(Stats masterExecutorStats, Stats indexExecutorStats, Stats summaryExecutorStats, - Stats sharedExecutorStats, Stats indexFieldInverterExecutorStats, Stats indexFieldWriterExecutorStats, Stats attributeFieldWriterExecutorStats) : _masterExecutorStats(masterExecutorStats), _indexExecutorStats(indexExecutorStats), _summaryExecutorStats(summaryExecutorStats), - _sharedExecutorStats(sharedExecutorStats), _indexFieldInverterExecutorStats(indexFieldInverterExecutorStats), _indexFieldWriterExecutorStats(indexFieldWriterExecutorStats), _attributeFieldWriterExecutorStats(attributeFieldWriterExecutorStats) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h index e2c53af11b5..8015ec83ae9 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h @@ -16,7 +16,6 @@ private: Stats _masterExecutorStats; Stats _indexExecutorStats; Stats _summaryExecutorStats; - Stats _sharedExecutorStats; Stats _indexFieldInverterExecutorStats; Stats _indexFieldWriterExecutorStats; Stats _attributeFieldWriterExecutorStats; @@ -24,7 +23,6 @@ public: ExecutorThreadingServiceStats(Stats masterExecutorStats, Stats indexExecutorStats, Stats summaryExecutorStats, - Stats sharedExecutorStats, Stats indexFieldInverterExecutorStats, Stats indexFieldWriterExecutorStats, Stats attributeFieldWriterExecutorStats); @@ -33,7 +31,6 @@ public: const Stats &getMasterExecutorStats() const { return _masterExecutorStats; } const Stats &getIndexExecutorStats() const { return _indexExecutorStats; } const Stats &getSummaryExecutorStats() const { return _summaryExecutorStats; } - const Stats &getSharedExecutorStats() const { return _sharedExecutorStats; } const Stats &getIndexFieldInverterExecutorStats() const { return _indexFieldInverterExecutorStats; } const Stats &getIndexFieldWriterExecutorStats() const { return _indexFieldWriterExecutorStats; } const Stats &getAttributeFieldWriterExecutorStats() const { return _attributeFieldWriterExecutorStats; } diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 511adbe66e9..1daacc29fcb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -47,6 +47,7 @@ vespa_add_library(searchcore_server STATIC feedhandler.cpp feedstate.cpp feedstates.cpp + feed_handler_stats.cpp fileconfigmanager.cpp flushhandlerproxy.cpp forcecommitcontext.cpp @@ -97,6 +98,8 @@ vespa_add_library(searchcore_server STATIC searchhandlerproxy.cpp searchview.cpp simpleflush.cpp + shared_threading_service.cpp + shared_threading_service_config.cpp storeonlydocsubdb.cpp storeonlyfeedview.cpp summaryadapter.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 0d45a287fdc..5d7f841a909 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -7,6 +7,7 @@ #include "documentdb.h" #include "documentdbconfigscout.h" #include "feedhandler.h" +#include "i_shared_threading_service.h" #include "idocumentdbowner.h" #include "idocumentsubdb.h" #include "maintenance_jobs_injector.h" @@ -131,8 +132,7 @@ DocumentDB::create(const vespalib::string &baseDir, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, IDocumentDBOwner &owner, - vespalib::SyncableThreadExecutor &warmupExecutor, - vespalib::ThreadExecutor &sharedExecutor, + ISharedThreadingService& shared_service, storage::spi::BucketExecutor &bucketExecutor, const search::transactionlog::WriterFactory &tlsWriterFactory, MetricsWireService &metricsWireService, @@ -143,7 +143,7 @@ DocumentDB::create(const vespalib::string &baseDir, { return DocumentDB::SP( new DocumentDB(baseDir, std::move(currentSnapshot), tlsSpec, queryLimiter, clock, docTypeName, bucketSpace, - protonCfg, owner, warmupExecutor, sharedExecutor, bucketExecutor, tlsWriterFactory, + protonCfg, owner, shared_service, bucketExecutor, tlsWriterFactory, metricsWireService, fileHeaderContext, std::move(config_store), initializeThreads, hwInfo)); } DocumentDB::DocumentDB(const vespalib::string &baseDir, @@ -155,8 +155,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, IDocumentDBOwner &owner, - vespalib::SyncableThreadExecutor &warmupExecutor, - vespalib::ThreadExecutor &sharedExecutor, + ISharedThreadingService& shared_service, storage::spi::BucketExecutor & bucketExecutor, const search::transactionlog::WriterFactory &tlsWriterFactory, MetricsWireService &metricsWireService, @@ -176,7 +175,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _baseDir(baseDir + "/" + _docTypeName.toString()), // Only one thread per executor, or performDropFeedView() will fail. _writeServiceConfig(configSnapshot->get_threading_service_config()), - _writeService(sharedExecutor, _writeServiceConfig, indexing_thread_stack_size), + _writeService(shared_service.shared(), _writeServiceConfig, indexing_thread_stack_size), _initializeThreads(std::move(initializeThreads)), _initConfigSnapshot(), _initConfigSerialNum(0u), @@ -204,12 +203,12 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _writeFilter(), _transient_usage_provider(std::make_shared<DocumentDBResourceUsageProvider>(*this)), _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), - _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, warmupExecutor, fileHeaderContext, + _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, shared_service.warmup(), fileHeaderContext, metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, hwInfo), - _maintenanceController(_writeService.master(), sharedExecutor, _refCount, _docTypeName), + _maintenanceController(_writeService.master(), shared_service.shared(), _refCount, _docTypeName), _jobTrackers(), _calc(), - _metricsUpdater(_subDBs, _writeService, _jobTrackers, *_sessionManager, _writeFilter) + _metricsUpdater(_subDBs, _writeService, _jobTrackers, *_sessionManager, _writeFilter, *_feedHandler) { assert(configSnapshot); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 391c11df276..e829f477e8a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -47,12 +47,13 @@ namespace storage::spi { struct BucketExecutor; } namespace proton { class AttributeConfigInspector; +class ExecutorThreadingServiceStats; class IDocumentDBOwner; +class ISharedThreadingService; class ITransientResourceUsageProvider; -struct MetricsWireService; class StatusReport; -class ExecutorThreadingServiceStats; class TransientResourceUsageProvider; +struct MetricsWireService; namespace matching { class SessionManager; } @@ -71,10 +72,15 @@ class DocumentDB : public DocumentDBConfigOwner, public std::enable_shared_from_this<DocumentDB> { private: - using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; + using InitializeThreads = std::shared_ptr<vespalib::ThreadExecutor>; using IFlushTargetList = std::vector<std::shared_ptr<searchcorespi::IFlushTarget>>; using StatusReportUP = std::unique_ptr<StatusReport>; using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType; + using ConfigComparisonResult = DocumentDBConfig::ComparisonResult; + using lock_guard = std::lock_guard<std::mutex>; + using SerialNum = search::SerialNum; + using Schema = search::index::Schema; + DocTypeName _docTypeName; document::BucketSpace _bucketSpace; @@ -85,9 +91,6 @@ private: // threads for initializer tasks during proton startup InitializeThreads _initializeThreads; - typedef search::SerialNum SerialNum; - typedef search::index::Schema Schema; - using lock_guard = std::lock_guard<std::mutex>; // variables related to reconfig DocumentDBConfig::SP _initConfigSnapshot; SerialNum _initConfigSerialNum; @@ -97,10 +100,7 @@ private: DocumentDBConfig::SP _activeConfigSnapshot; int64_t _activeConfigSnapshotGeneration; const bool _validateAndSanitizeDocStore; - - vespalib::Gate _initGate; - - typedef DocumentDBConfig::ComparisonResult ConfigComparisonResult; + vespalib::Gate _initGate; ClusterStateHandler _clusterStateHandler; BucketHandler _bucketHandler; @@ -201,8 +201,7 @@ private: document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, IDocumentDBOwner &owner, - vespalib::SyncableThreadExecutor &warmupExecutor, - vespalib::ThreadExecutor &sharedExecutor, + ISharedThreadingService& shared_service, storage::spi::BucketExecutor &bucketExecutor, const search::transactionlog::WriterFactory &tlsWriterFactory, MetricsWireService &metricsWireService, @@ -233,8 +232,7 @@ public: document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, IDocumentDBOwner &owner, - vespalib::SyncableThreadExecutor &warmupExecutor, - vespalib::ThreadExecutor &sharedExecutor, + ISharedThreadingService& shared_service, storage::spi::BucketExecutor & bucketExecutor, const search::transactionlog::WriterFactory &tlsWriterFactory, MetricsWireService &metricsWireService, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp index d0bd7d4ee69..4e156539441 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp @@ -5,6 +5,7 @@ #include "documentdb_metrics_updater.h" #include "documentsubdbcollection.h" #include "executorthreadingservice.h" +#include "feedhandler.h" #include "idocumentsubdb.h" #include <vespa/searchcommon/attribute/status.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> @@ -34,12 +35,16 @@ DocumentDBMetricsUpdater::DocumentDBMetricsUpdater(const DocumentSubDBCollection ExecutorThreadingService &writeService, DocumentDBJobTrackers &jobTrackers, matching::SessionManager &sessionManager, - const AttributeUsageFilter &writeFilter) + const AttributeUsageFilter &writeFilter, + FeedHandler& feed_handler) : _subDBs(subDBs), _writeService(writeService), _jobTrackers(jobTrackers), _sessionManager(sessionManager), - _writeFilter(writeFilter) + _writeFilter(writeFilter), + _feed_handler(feed_handler), + _lastDocStoreCacheStats(), + _last_feed_handler_stats() { } @@ -280,6 +285,27 @@ updateLidSpaceMetrics(MetricSetType &metrics, const search::IDocumentMetaStore & metrics.lidFragmentationFactor.set(stats.getLidFragmentationFactor()); } +void +update_feeding_metrics(DocumentDBFeedingMetrics& metrics, FeedHandlerStats stats, std::optional<FeedHandlerStats>& last_stats) +{ + auto delta_stats = stats; + if (last_stats.has_value()) { + delta_stats -= last_stats.value(); + } + last_stats = stats; + uint32_t commits = delta_stats.get_commits(); + if (commits != 0) { + double min_operations = delta_stats.get_min_operations().value_or(0); + double max_operations = delta_stats.get_max_operations().value_or(0); + double avg_operations = ((double) delta_stats.get_operations()) / commits; + metrics.commit.operations.addValueBatch(avg_operations, commits, min_operations, max_operations); + double min_latency = delta_stats.get_min_latency().value_or(0.0); + double max_latency = delta_stats.get_max_latency().value_or(0.0); + double avg_latency = delta_stats.get_total_latency() / commits; + metrics.commit.latency.addValueBatch(avg_latency, commits, min_latency, max_latency); + } +} + } void @@ -297,6 +323,7 @@ DocumentDBMetricsUpdater::updateMetrics(const metrics::MetricLockGuard & guard, metrics.totalMemoryUsage.update(totalStats.memoryUsage); metrics.totalDiskUsage.set(totalStats.diskUsage); + update_feeding_metrics(metrics.feeding, _feed_handler.get_stats(true), _last_feed_handler_stats); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h index b73fa3b4eb9..381d98b2199 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.h @@ -1,8 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include "feed_handler_stats.h" #include <vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h> #include <vespa/searchlib/docstore/cachestats.h> +#include <optional> namespace proton { @@ -14,6 +16,7 @@ class DocumentDBJobTrackers; class DocumentSubDBCollection; class ExecutorThreadingService; class ExecutorThreadingServiceStats; +class FeedHandler; /** * Class used to update metrics for a document db. @@ -34,8 +37,10 @@ private: DocumentDBJobTrackers &_jobTrackers; matching::SessionManager &_sessionManager; const AttributeUsageFilter &_writeFilter; + FeedHandler &_feed_handler; // Last updated document store cache statistics. Necessary due to metrics implementation is upside down. DocumentStoreCacheStats _lastDocStoreCacheStats; + std::optional<FeedHandlerStats> _last_feed_handler_stats; void updateMiscMetrics(DocumentDBTaggedMetrics &metrics, const ExecutorThreadingServiceStats &threadingServiceStats); void updateAttributeResourceUsageMetrics(DocumentDBTaggedMetrics::AttributeMetrics &metrics); @@ -45,7 +50,8 @@ public: ExecutorThreadingService &writeService, DocumentDBJobTrackers &jobTrackers, matching::SessionManager &sessionManager, - const AttributeUsageFilter &writeFilter); + const AttributeUsageFilter &writeFilter, + FeedHandler& feed_handler); ~DocumentDBMetricsUpdater(); void updateMetrics(const metrics::MetricLockGuard & guard, DocumentDBTaggedMetrics &metrics); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 6576e4ead97..3c9bdb2ec5f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -26,7 +26,7 @@ DocumentSubDBCollection::DocumentSubDBCollection( const IGetSerialNum &getSerialNum, const DocTypeName &docTypeName, searchcorespi::index::IThreadingService &writeService, - vespalib::SyncableThreadExecutor &warmupExecutor, + vespalib::Executor &warmupExecutor, const search::common::FileHeaderContext &fileHeaderContext, MetricsWireService &metricsWireService, DocumentDBTaggedMetrics &metrics, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 515a886969c..ca092bb0957 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -12,7 +12,7 @@ namespace vespalib { class Clock; - class SyncableThreadExecutor; + class Executor; class ThreadStackExecutorBase; } @@ -86,7 +86,7 @@ public: const IGetSerialNum &getSerialNum, const DocTypeName &docTypeName, searchcorespi::index::IThreadingService &writeService, - vespalib::SyncableThreadExecutor &warmupExecutor, + vespalib::Executor &warmupExecutor, const search::common::FileHeaderContext &fileHeaderContext, MetricsWireService &metricsWireService, DocumentDBTaggedMetrics &metrics, diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp index 684132b34e7..74f6a622661 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp @@ -9,6 +9,7 @@ using vespalib::makeLambdaTask; using vespalib::Executor; using vespalib::Gate; using vespalib::Runnable; +using vespalib::ThreadExecutor; using vespalib::SyncableThreadExecutor; namespace proton { @@ -29,11 +30,15 @@ sampleThreadId(FastOS_ThreadId *threadId) } std::unique_ptr<internal::ThreadId> -getThreadId(SyncableThreadExecutor &executor) +getThreadId(ThreadExecutor &executor) { std::unique_ptr<internal::ThreadId> id = std::make_unique<internal::ThreadId>(); - executor.execute(makeLambdaTask([threadId=&id->_id] { sampleThreadId(threadId);})); - executor.sync(); + vespalib::Gate gate; + executor.execute(makeLambdaTask([threadId=&id->_id, &gate] { + sampleThreadId(threadId); + gate.countDown(); + })); + gate.await(); return id; } @@ -46,7 +51,7 @@ runRunnable(Runnable *runnable, Gate *gate) } // namespace -ExecutorThreadService::ExecutorThreadService(SyncableThreadExecutor &executor) +ExecutorThreadService::ExecutorThreadService(ThreadExecutor &executor) : _executor(executor), _threadId(getThreadId(executor)) { @@ -90,4 +95,51 @@ ExecutorThreadService::wakeup() { _executor.wakeup(); } +SyncableExecutorThreadService::SyncableExecutorThreadService(SyncableThreadExecutor &executor) + : _executor(executor), + _threadId(getThreadId(executor)) +{ +} + +SyncableExecutorThreadService::~SyncableExecutorThreadService() = default; + +void +SyncableExecutorThreadService::run(Runnable &runnable) +{ + if (isCurrentThread()) { + runnable.run(); + } else { + Gate gate; + _executor.execute(makeLambdaTask([runnablePtr=&runnable, gatePtr=&gate] { runRunnable(runnablePtr, gatePtr); })); + gate.await(); + } +} + +bool +SyncableExecutorThreadService::isCurrentThread() const +{ + FastOS_ThreadId currentThreadId = FastOS_Thread::GetCurrentThreadId(); + return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId); +} + +vespalib::ExecutorStats +SyncableExecutorThreadService::getStats() { + return _executor.getStats(); +} + +void +SyncableExecutorThreadService::setTaskLimit(uint32_t taskLimit) { + _executor.setTaskLimit(taskLimit); +} + +uint32_t +SyncableExecutorThreadService::getTaskLimit() const { + return _executor.getTaskLimit(); +} + +void +SyncableExecutorThreadService::wakeup() { + _executor.wakeup(); +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h index 44a330ca696..7298b81611a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h @@ -14,11 +14,11 @@ namespace internal { struct ThreadId; } class ExecutorThreadService : public searchcorespi::index::IThreadService { private: - vespalib::SyncableThreadExecutor &_executor; + vespalib::ThreadExecutor &_executor; std::unique_ptr<internal::ThreadId> _threadId; public: - ExecutorThreadService(vespalib::SyncableThreadExecutor &executor); + ExecutorThreadService(vespalib::ThreadExecutor &executor); ~ExecutorThreadService(); vespalib::ExecutorStats getStats() override; @@ -27,14 +27,36 @@ public: return _executor.execute(std::move(task)); } void run(vespalib::Runnable &runnable) override; + + bool isCurrentThread() const override; + size_t getNumThreads() const override { return _executor.getNumThreads(); } + + void setTaskLimit(uint32_t taskLimit) override; + uint32_t getTaskLimit() const override; + void wakeup() override; +}; + +class SyncableExecutorThreadService : public searchcorespi::index::ISyncableThreadService +{ +private: + vespalib::SyncableThreadExecutor &_executor; + std::unique_ptr<internal::ThreadId> _threadId; + +public: + SyncableExecutorThreadService(vespalib::SyncableThreadExecutor &executor); + ~SyncableExecutorThreadService(); + + vespalib::ExecutorStats getStats() override; + + vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override { + return _executor.execute(std::move(task)); + } + void run(vespalib::Runnable &runnable) override; vespalib::Syncable &sync() override { _executor.sync(); return *this; } - ExecutorThreadService & shutdown() override { - _executor.shutdown(); - return *this; - } + bool isCurrentThread() const override; size_t getNumThreads() const override { return _executor.getNumThreads(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp index 89de400216e..daf54eac859 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.cpp @@ -59,7 +59,7 @@ convert_executor_to_slime(const ISequencedTaskExecutor* executor, Cursor& object } -ExecutorThreadingServiceExplorer::ExecutorThreadingServiceExplorer(ExecutorThreadingService& service) +ExecutorThreadingServiceExplorer::ExecutorThreadingServiceExplorer(searchcorespi::index::IThreadingService& service) : _service(service) { } @@ -71,9 +71,9 @@ ExecutorThreadingServiceExplorer::get_state(const vespalib::slime::Inserter& ins { auto& object = inserter.insertObject(); if (full) { - convert_executor_to_slime(&_service.getMasterExecutor(), object.setObject("master")); - convert_executor_to_slime(&_service.getIndexExecutor(), object.setObject("index")); - convert_executor_to_slime(&_service.getSummaryExecutor(), object.setObject("summary")); + convert_executor_to_slime(&_service.master(), object.setObject("master")); + convert_executor_to_slime(&_service.index(), object.setObject("index")); + convert_executor_to_slime(&_service.summary(), object.setObject("summary")); convert_executor_to_slime(&_service.indexFieldInverter(), object.setObject("index_field_inverter")); convert_executor_to_slime(&_service.indexFieldWriter(), object.setObject("index_field_writer")); convert_executor_to_slime(&_service.attributeFieldWriter(), object.setObject("attribute_field_writer")); diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h index 374a0e6b494..70ed23c6271 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/net/state_explorer.h> +namespace searchcorespi::index { struct IThreadingService; } namespace proton { class ExecutorThreadingService; @@ -13,10 +14,10 @@ class ExecutorThreadingService; */ class ExecutorThreadingServiceExplorer : public vespalib::StateExplorer { private: - ExecutorThreadingService& _service; + searchcorespi::index::IThreadingService & _service; public: - ExecutorThreadingServiceExplorer(ExecutorThreadingService& service); + ExecutorThreadingServiceExplorer(searchcorespi::index::IThreadingService & service); ~ExecutorThreadingServiceExplorer(); void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 145dc48a57a..86530f235fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -54,7 +54,6 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadExecutor& sha _summaryExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), summary_executor)), _masterService(_masterExecutor), _indexService(*_indexExecutor), - _summaryService(*_summaryExecutor), _indexFieldInverter(), _indexFieldWriter(), _attributeFieldWriter(), @@ -155,21 +154,20 @@ ExecutorThreadingService::getStats() auto master_stats = _masterExecutor.getStats(); auto index_stats = _indexExecutor->getStats(); auto summary_stats = _summaryExecutor->getStats(); - auto shared_stats = _sharedExecutor.getStats(); if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) { auto field_writer_stats = _field_writer->getStats(); - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats, + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, field_writer_stats, field_writer_stats, _attribute_field_writer_ptr->getStats()); } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { auto field_writer_stats = _field_writer->getStats(); - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats, + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, field_writer_stats, field_writer_stats, field_writer_stats); } else { - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, shared_stats, + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, _index_field_inverter_ptr->getStats(), _index_field_writer_ptr->getStats(), _attribute_field_writer_ptr->getStats()); diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 629c5043ed7..2a4c57ef57d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -23,9 +23,8 @@ private: std::atomic<uint32_t> _master_task_limit; std::unique_ptr<vespalib::SyncableThreadExecutor> _indexExecutor; std::unique_ptr<vespalib::SyncableThreadExecutor> _summaryExecutor; - ExecutorThreadService _masterService; + SyncableExecutorThreadService _masterService; ExecutorThreadService _indexService; - ExecutorThreadService _summaryService; std::unique_ptr<vespalib::ISequencedTaskExecutor> _indexFieldInverter; std::unique_ptr<vespalib::ISequencedTaskExecutor> _indexFieldWriter; std::unique_ptr<vespalib::ISequencedTaskExecutor> _attributeFieldWriter; @@ -47,7 +46,7 @@ public: uint32_t stackSize = 128 * 1024); ~ExecutorThreadingService() override; - void sync_all_executors() override; + void sync_all_executors(); void blocking_master_execute(vespalib::Executor::Task::UP task) override; @@ -60,27 +59,15 @@ public: uint32_t field_task_limit, uint32_t summary_task_limit); - // Expose the underlying executors for stats fetching and testing. - // TOD: Remove - This is only used for casting to check the underlying type - vespalib::ThreadExecutor &getMasterExecutor() { - return _masterExecutor; - } - vespalib::ThreadExecutor &getIndexExecutor() { - return *_indexExecutor; - } - vespalib::ThreadExecutor &getSummaryExecutor() { - return *_summaryExecutor; - } - - searchcorespi::index::IThreadService &master() override { + searchcorespi::index::ISyncableThreadService &master() override { return _masterService; } searchcorespi::index::IThreadService &index() override { return _indexService; } - searchcorespi::index::IThreadService &summary() override { - return _summaryService; + vespalib::ThreadExecutor &summary() override { + return *_summaryExecutor; } vespalib::ThreadExecutor &shared() override { return _sharedExecutor; diff --git a/searchcore/src/vespa/searchcore/proton/server/feed_handler_stats.cpp b/searchcore/src/vespa/searchcore/proton/server/feed_handler_stats.cpp new file mode 100644 index 00000000000..0db388d2644 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/feed_handler_stats.cpp @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "feed_handler_stats.h" + +namespace proton { + +namespace { + +template <typename T> +void update_min_max(T value, std::optional<T>& min, std::optional<T>& max) +{ + if (!min.has_value() || value < min.value()) { + min = value; + } + if (!max.has_value() || value > max.value()) { + max = value; + } +} + +} + +FeedHandlerStats::FeedHandlerStats(uint64_t commits, uint64_t operations, double total_latency) noexcept + : _commits(commits), + _operations(operations), + _total_latency(total_latency), + _min_operations(), + _max_operations(), + _min_latency(), + _max_latency() +{ +} + +FeedHandlerStats::FeedHandlerStats() noexcept + : FeedHandlerStats(0, 0, 0.0) +{ +} + +FeedHandlerStats::~FeedHandlerStats() = default; + + +FeedHandlerStats& +FeedHandlerStats::operator-=(const FeedHandlerStats& rhs) noexcept +{ + _commits -= rhs._commits; + _operations -= rhs._operations; + _total_latency -= rhs._total_latency; + return *this; +} + +void +FeedHandlerStats::add_commit(uint32_t operations, double latency) noexcept +{ + ++_commits; + _operations += operations; + _total_latency += latency; + update_min_max(operations, _min_operations, _max_operations); + update_min_max(latency, _min_latency, _max_latency); +} + +void +FeedHandlerStats::reset_min_max() noexcept +{ + _min_operations.reset(); + _max_operations.reset(); + _min_latency.reset(); + _max_latency.reset(); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/feed_handler_stats.h b/searchcore/src/vespa/searchcore/proton/server/feed_handler_stats.h new file mode 100644 index 00000000000..9c8d1b9190b --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/feed_handler_stats.h @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <optional> + +namespace proton { + +/* + * Stats for feed handler. + */ +class FeedHandlerStats +{ + uint64_t _commits; + uint64_t _operations; + double _total_latency; + std::optional<uint32_t> _min_operations; + std::optional<uint32_t> _max_operations; + std::optional<double> _min_latency; + std::optional<double> _max_latency; + +public: + FeedHandlerStats(uint64_t commits, uint64_t operations, double total_latency) noexcept; + FeedHandlerStats() noexcept; + ~FeedHandlerStats(); + FeedHandlerStats& operator-=(const FeedHandlerStats& rhs) noexcept; + void add_commit(uint32_t operations, double latency) noexcept; + void reset_min_max() noexcept; + uint64_t get_commits() noexcept { return _commits; } + uint64_t get_operations() noexcept { return _operations; } + double get_total_latency() noexcept { return _total_latency; } + const std::optional<uint32_t>& get_min_operations() noexcept { return _min_operations; } + const std::optional<uint32_t>& get_max_operations() noexcept { return _max_operations; } + const std::optional<double>& get_min_latency() noexcept { return _min_latency; } + const std::optional<double>& get_max_latency() noexcept { return _max_latency; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index bb03f48882f..51ea6425622 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "ddbstate.h" #include "feedhandler.h" +#include "ddbstate.h" #include "feedstates.h" #include "i_feed_handler_owner.h" #include "ifeedview.h" @@ -429,7 +429,9 @@ FeedHandler::FeedHandler(IThreadingService &writeService, _syncLock(), _syncedSerialNum(0), _allowSync(false), - _heart_beat_time(vespalib::steady_time()) + _heart_beat_time(vespalib::steady_time()), + _stats_lock(), + _stats() { } @@ -515,7 +517,7 @@ FeedHandler::getTransactionLogReplayDone() const { } void -FeedHandler::onCommitDone(size_t numPendingAtStart) { +FeedHandler::onCommitDone(size_t numPendingAtStart, vespalib::steady_time start_time) { assert(numPendingAtStart <= _numOperationsPendingCommit); _numOperationsPendingCommit -= numPendingAtStart; _numOperationsCompleted += numPendingAtStart; @@ -525,18 +527,22 @@ FeedHandler::onCommitDone(size_t numPendingAtStart) { } LOG(spam, "%zu: onCommitDone(%zu) total=%zu left=%zu", _numCommitsCompleted, numPendingAtStart, _numOperationsCompleted, _numOperationsPendingCommit); + vespalib::steady_time now = vespalib::steady_clock::now(); + auto latency = vespalib::to_s(now - start_time); + std::lock_guard guard(_stats_lock); + _stats.add_commit(numPendingAtStart, latency); } void FeedHandler::enqueCommitTask() { - _writeService.master().execute(makeLambdaTask([this]() { initiateCommit(); })); + _writeService.master().execute(makeLambdaTask([this, start_time(vespalib::steady_clock::now())]() { initiateCommit(start_time); })); } void -FeedHandler::initiateCommit() { +FeedHandler::initiateCommit(vespalib::steady_time start_time) { auto onCommitDoneContext = std::make_shared<OnCommitDone>( _writeService.master(), - makeLambdaTask([this, numPendingAtStart=_numOperationsPendingCommit]() { - onCommitDone(numPendingAtStart); + makeLambdaTask([this, numPendingAtStart=_numOperationsPendingCommit, start_time]() { + onCommitDone(numPendingAtStart, start_time); })); auto commitResult = _tlsWriter->startCommit(onCommitDoneContext); if (_activeFeedView) { @@ -822,4 +828,14 @@ FeedHandler::get_heart_beat_time() const return _heart_beat_time.load(std::memory_order_relaxed); } +FeedHandlerStats +FeedHandler::get_stats(bool reset_min_max) const { + std::lock_guard guard(_stats_lock); + auto result = _stats; + if (reset_min_max) { + _stats.reset_min_max(); + } + return result; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index ef15b268086..39d1f0f47fb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -2,6 +2,7 @@ #pragma once +#include "feed_handler_stats.h" #include "i_inc_serial_num.h" #include "i_operation_storer.h" #include "idocumentmovehandler.h" @@ -97,6 +98,8 @@ private: SerialNum _syncedSerialNum; bool _allowSync; // Sanity check std::atomic<vespalib::steady_time> _heart_beat_time; + mutable std::mutex _stats_lock; + mutable FeedHandlerStats _stats; /** * Delayed handling of feed operations, in master write thread. @@ -134,8 +137,8 @@ private: FeedStateSP getFeedState() const; void changeFeedState(FeedStateSP newState); void doChangeFeedState(FeedStateSP newState); - void onCommitDone(size_t numPendingAtStart); - void initiateCommit(); + void onCommitDone(size_t numPendingAtStart, vespalib::steady_time start_time); + void initiateCommit(vespalib::steady_time start_time); void enqueCommitTask(); public: FeedHandler(const FeedHandler &) = delete; @@ -245,6 +248,7 @@ public: [[nodiscard]] CommitResult storeOperationSync(const FeedOperation & op); void considerDelayedPrune(); vespalib::steady_time get_heart_beat_time() const; + FeedHandlerStats get_stats(bool reset_min_max) const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h index 0d6bb07b173..704b54dc566 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_proton_configurer_owner.h @@ -6,8 +6,6 @@ #include <vespa/vespalib/stllike/string.h> #include <memory> -namespace vespalib { class ThreadStackExecutorBase; } - namespace proton { class DocumentDBConfigOwner; @@ -19,7 +17,7 @@ class DocumentDBConfigOwner; class IProtonConfigurerOwner { public: - using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; + using InitializeThreads = std::shared_ptr<vespalib::ThreadExecutor>; virtual ~IProtonConfigurerOwner() { } virtual std::shared_ptr<DocumentDBConfigOwner> addDocumentDB(const DocTypeName &docTypeName, document::BucketSpace bucketSpace, diff --git a/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h new file mode 100644 index 00000000000..5145dbec43e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace vespalib { class ThreadExecutor; } + +namespace proton { + +/** + * Interface containing the thread executors that are shared across all document dbs. + */ +class ISharedThreadingService { +public: + virtual ~ISharedThreadingService() {} + + /** + * Returns the executor used for warmup (e.g. index warmup). + */ + virtual vespalib::ThreadExecutor& warmup() = 0; + + /** + * Returns the shared executor used for various assisting tasks in a document db. + * + * Example usages include: + * - Disk index fusion. + * - Updating nearest neighbor index (in DenseTensorAttribute). + * - Loading nearest neighbor index (in DenseTensorAttribute). + * - Writing of data in the document store. + */ + virtual vespalib::ThreadExecutor& shared() = 0; +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index 6b606298026..0d75464a161 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -39,7 +39,7 @@ isRunnable(const MaintenanceJobRunner & job, const Executor * master) { } -MaintenanceController::MaintenanceController(IThreadService &masterThread, +MaintenanceController::MaintenanceController(ISyncableThreadService &masterThread, vespalib::Executor & defaultExecutor, MonitoredRefCount & refCount, const DocTypeName &docTypeName) @@ -140,6 +140,11 @@ MaintenanceController::stop() _masterThread.sync(); // Wait for already scheduled maintenance jobs and performHoldJobs } +searchcorespi::index::IThreadService & +MaintenanceController::masterThread() { + return _masterThread; +} + void MaintenanceController::kill() { diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h index 8c8cc3e2d43..f2c425b2fd0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h @@ -17,7 +17,10 @@ class MonitoredRefCount; class Timer; } -namespace searchcorespi::index { struct IThreadService; } +namespace searchcorespi::index { + struct IThreadService; + struct ISyncableThreadService; +} namespace proton { @@ -33,12 +36,13 @@ class MaintenanceController { public: using IThreadService = searchcorespi::index::IThreadService; + using ISyncableThreadService = searchcorespi::index::ISyncableThreadService; using DocumentDBMaintenanceConfigSP = std::shared_ptr<DocumentDBMaintenanceConfig>; using JobList = std::vector<std::shared_ptr<MaintenanceJobRunner>>; using UP = std::unique_ptr<MaintenanceController>; enum class State {INITIALIZING, STARTED, PAUSED, STOPPING}; - MaintenanceController(IThreadService &masterThread, vespalib::Executor & defaultExecutor, vespalib::MonitoredRefCount & refCount, const DocTypeName &docTypeName); + MaintenanceController(ISyncableThreadService &masterThread, vespalib::Executor & defaultExecutor, vespalib::MonitoredRefCount & refCount, const DocTypeName &docTypeName); ~MaintenanceController(); void registerJobInMasterThread(IMaintenanceJob::UP job); @@ -70,14 +74,14 @@ public: const MaintenanceDocumentSubDB & getReadySubDB() const { return _readySubDB; } const MaintenanceDocumentSubDB & getRemSubDB() const { return _remSubDB; } const MaintenanceDocumentSubDB & getNotReadySubDB() const { return _notReadySubDB; } - IThreadService & masterThread() { return _masterThread; } + IThreadService & masterThread(); const DocTypeName & getDocTypeName() const { return _docTypeName; } vespalib::RetainGuard retainDB() { return vespalib::RetainGuard(_refCount); } private: using Mutex = std::mutex; using Guard = std::lock_guard<Mutex>; - IThreadService &_masterThread; + ISyncableThreadService &_masterThread; vespalib::Executor &_defaultExecutor; vespalib::MonitoredRefCount &_refCount; MaintenanceDocumentSubDB _readySubDB; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 5b21242e397..e056325e0d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -111,15 +111,6 @@ diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo) hwInfo); } -size_t -derive_shared_threads(const ProtonConfig &proton, const HwInfo::Cpu &cpuInfo) { - size_t scaledCores = (size_t)std::ceil(cpuInfo.cores() * proton.feeding.concurrency); - - // We need at least 1 guaranteed free worker in order to ensure progress so #documentsdbs + 1 should suffice, - // but we will not be cheap and give it one extra. - return std::max(scaledCores, proton.documentdb.size() + proton.flush.maxconcurrent + 1); -} - uint32_t computeRpcTransportThreads(const ProtonConfig & cfg, const HwInfo::Cpu &cpuInfo) { bool areSearchAndDocsumAsync = cfg.docsum.async && cfg.search.async; @@ -144,8 +135,6 @@ struct MetricsUpdateHook : metrics::UpdateHook const vespalib::string CUSTOM_COMPONENT_API_PATH = "/state/v1/custom/component"; -VESPA_THREAD_STACK_TAG(proton_shared_executor) -VESPA_THREAD_STACK_TAG(index_warmup_executor) VESPA_THREAD_STACK_TAG(initialize_executor) VESPA_THREAD_STACK_TAG(close_executor) @@ -240,8 +229,7 @@ Proton::Proton(const config::ConfigUri & configUri, _protonDiskLayout(), _protonConfigurer(_executor, *this, _protonDiskLayout), _protonConfigFetcher(configUri, _protonConfigurer, subscribeTimeout), - _warmupExecutor(), - _sharedExecutor(), + _shared_service(), _compile_cache_executor_binding(), _queryLimiter(), _clock(0.001), @@ -333,11 +321,8 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) protonConfig.visit.ignoremaxbytes); vespalib::string fileConfigId; - _warmupExecutor = std::make_unique<vespalib::ThreadStackExecutor>(4, 128_Ki, index_warmup_executor); - - const size_t sharedThreads = derive_shared_threads(protonConfig, hwInfo.cpu()); - _sharedExecutor = std::make_shared<vespalib::BlockingThreadStackExecutor>(sharedThreads, 128_Ki, sharedThreads*16, proton_shared_executor); - _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_sharedExecutor); + _shared_service = std::make_unique<SharedThreadingService>(SharedThreadingServiceConfig::make(protonConfig, hwInfo.cpu())); + _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_shared_service->shared_raw()); InitializeThreads initializeThreads; if (protonConfig.initialize.threads > 0) { initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(protonConfig.initialize.threads, 128_Ki, initialize_executor); @@ -460,11 +445,9 @@ Proton::~Proton() if (_flushEngine) { _flushEngine->close(); } - if (_warmupExecutor) { - _warmupExecutor->sync(); - } - if (_sharedExecutor) { - _sharedExecutor->sync(); + if (_shared_service) { + _shared_service->warmup_raw().sync(); + _shared_service->shared_raw()->sync(); } if ( ! _documentDBMap.empty()) { @@ -483,9 +466,8 @@ Proton::~Proton() _documentDBMap.clear(); _persistenceEngine.reset(); _tls.reset(); - _warmupExecutor.reset(); _compile_cache_executor_binding.reset(); - _sharedExecutor.reset(); + _shared_service.reset(); _clock.stop(); LOG(debug, "Explicit destructor done"); } @@ -619,11 +601,23 @@ Proton::addDocumentDB(const document::DocumentType &docType, // 1 thread per document type. initializeThreads = std::make_shared<vespalib::ThreadStackExecutor>(1, 128_Ki); } - auto ret = DocumentDB::create(config.basedir + "/documents", documentDBConfig, config.tlsspec, - _queryLimiter, _clock, docTypeName, bucketSpace, config, *this, - *_warmupExecutor, *_sharedExecutor, *_persistenceEngine, *_tls->getTransLogServer(), - *_metricsEngine, _fileHeaderContext, std::move(config_store), - initializeThreads, bootstrapConfig->getHwInfo()); + auto ret = DocumentDB::create(config.basedir + "/documents", + documentDBConfig, + config.tlsspec, + _queryLimiter, + _clock, + docTypeName, + bucketSpace, + config, + *this, + *_shared_service, + *_persistenceEngine, + *_tls->getTransLogServer(), + *_metricsEngine, + _fileHeaderContext, + std::move(config_store), + initializeThreads, + bootstrapConfig->getHwInfo()); try { ret->start(); } catch (vespalib::Exception &e) { @@ -791,11 +785,9 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) if (_summaryEngine) { updateExecutorMetrics(metrics.docsum, _summaryEngine->getExecutorStats()); } - if (_sharedExecutor) { - metrics.shared.update(_sharedExecutor->getStats()); - } - if (_warmupExecutor) { - metrics.warmup.update(_warmupExecutor->getStats()); + if (_shared_service) { + metrics.shared.update(_shared_service->shared().getStats()); + metrics.warmup.update(_shared_service->warmup().getStats()); } } } @@ -947,12 +939,12 @@ Proton::get_child(vespalib::stringref name) const return std::make_unique<ResourceUsageExplorer>(_diskMemUsageSampler->writeFilter(), _persistenceEngine->get_resource_usage_tracker()); } else if (name == THREAD_POOLS) { - return std::make_unique<ProtonThreadPoolsExplorer>(_sharedExecutor.get(), + return std::make_unique<ProtonThreadPoolsExplorer>((_shared_service) ? _shared_service->shared_raw().get() : nullptr, (_matchEngine) ? &_matchEngine->get_executor() : nullptr, (_summaryEngine) ? &_summaryEngine->get_executor() : nullptr, (_flushEngine) ? &_flushEngine->get_executor() : nullptr, &_executor, - _warmupExecutor.get()); + (_shared_service) ? &_shared_service->warmup() : nullptr); } return Explorer_UP(nullptr); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 6fd31352051..91635dc7497 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -12,6 +12,8 @@ #include "proton_config_fetcher.h" #include "proton_configurer.h" #include "rpc_hooks.h" +#include "shared_threading_service.h" +#include <vespa/eval/eval/llvm/compile_cache.h> #include <vespa/searchcore/proton/matching/querylimiter.h> #include <vespa/searchcore/proton/metrics/metrics_engine.h> #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h> @@ -24,7 +26,6 @@ #include <vespa/vespalib/net/json_handler_repo.h> #include <vespa/vespalib/net/state_explorer.h> #include <vespa/vespalib/util/varholder.h> -#include <vespa/eval/eval/llvm/compile_cache.h> #include <mutex> #include <shared_mutex> @@ -58,8 +59,7 @@ private: using MonitorReply = search::engine::MonitorReply; using MonitorClient = search::engine::MonitorClient; using DocumentDBMap = std::map<DocTypeName, DocumentDB::SP>; - using ProtonConfigSP = BootstrapConfig::ProtonConfigSP; - using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; + using InitializeThreads = std::shared_ptr<vespalib::ThreadExecutor>; using BucketSpace = document::BucketSpace; class ProtonFileHeaderContext : public search::common::FileHeaderContext @@ -101,8 +101,7 @@ private: std::unique_ptr<IProtonDiskLayout> _protonDiskLayout; ProtonConfigurer _protonConfigurer; ProtonConfigFetcher _protonConfigFetcher; - std::unique_ptr<vespalib::ThreadStackExecutorBase> _warmupExecutor; - std::shared_ptr<vespalib::ThreadStackExecutorBase> _sharedExecutor; + std::unique_ptr<SharedThreadingService> _shared_service; vespalib::eval::CompileCache::ExecutorBinding::UP _compile_cache_executor_binding; matching::QueryLimiter _queryLimiter; vespalib::Clock _clock; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp index 7c998ceca7c..2c891927fa3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.cpp @@ -12,6 +12,7 @@ #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/config-bucketspaces.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/util/retain_guard.h> #include <vespa/vespalib/stllike/asciistream.h> #include <future> @@ -42,7 +43,7 @@ getBucketSpace(const BootstrapConfig &bootstrapConfig, const DocTypeName &name) } -ProtonConfigurer::ProtonConfigurer(vespalib::SyncableThreadExecutor &executor, +ProtonConfigurer::ProtonConfigurer(vespalib::ThreadExecutor &executor, IProtonConfigurerOwner &owner, const std::unique_ptr<IProtonDiskLayout> &diskLayout) : IProtonConfigurer(), @@ -58,9 +59,22 @@ ProtonConfigurer::ProtonConfigurer(vespalib::SyncableThreadExecutor &executor, { } -ProtonConfigurer::~ProtonConfigurer() -{ -} +class ProtonConfigurer::ReconfigureTask : public vespalib::Executor::Task { +public: + ReconfigureTask(ProtonConfigurer & configurer) + : _configurer(configurer), + _retainGuard(configurer._pendingReconfigureTasks) + {} + + void run() override { + _configurer.performReconfigure(); + } +private: + ProtonConfigurer & _configurer; + vespalib::RetainGuard _retainGuard; +}; + +ProtonConfigurer::~ProtonConfigurer() = default; void ProtonConfigurer::setAllowReconfig(bool allowReconfig) @@ -72,11 +86,12 @@ ProtonConfigurer::setAllowReconfig(bool allowReconfig) _allowReconfig = allowReconfig; if (allowReconfig) { // Ensure that pending config is applied - _executor.execute(makeLambdaTask([this]() { performReconfigure(); })); + _executor.execute(std::make_unique<ReconfigureTask>(*this)); } } if (!allowReconfig) { - _executor.sync(); // drain queued performReconfigure tasks + // drain queued performReconfigure tasks + _pendingReconfigureTasks.waitForZeroRefCount(); } } @@ -102,7 +117,7 @@ ProtonConfigurer::reconfigure(std::shared_ptr<ProtonConfigSnapshot> configSnapsh std::lock_guard<std::mutex> guard(_mutex); _pendingConfigSnapshot = configSnapshot; if (_allowReconfig) { - _executor.execute(makeLambdaTask([&]() { performReconfigure(); })); + _executor.execute(std::make_unique<ReconfigureTask>(*this)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index 829da0756f8..ddb9c1bed92 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -7,6 +7,7 @@ #include <vespa/document/bucket/bucketspace.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/util/monitored_refcount.h> #include <map> #include <mutex> @@ -25,17 +26,19 @@ class IProtonDiskLayout; class ProtonConfigurer : public IProtonConfigurer { using DocumentDBs = std::map<DocTypeName, std::pair<std::weak_ptr<IDocumentDBConfigOwner>, std::weak_ptr<DocumentDBDirectoryHolder>>>; - using InitializeThreads = std::shared_ptr<vespalib::SyncableThreadExecutor>; + using InitializeThreads = std::shared_ptr<vespalib::ThreadExecutor>; + class ReconfigureTask; - ExecutorThreadService _executor; - IProtonConfigurerOwner &_owner; - DocumentDBs _documentDBs; - std::shared_ptr<ProtonConfigSnapshot> _pendingConfigSnapshot; - std::shared_ptr<ProtonConfigSnapshot> _activeConfigSnapshot; - mutable std::mutex _mutex; - bool _allowReconfig; - vespalib::SimpleComponentConfigProducer _componentConfig; + ExecutorThreadService _executor; + IProtonConfigurerOwner &_owner; + DocumentDBs _documentDBs; + std::shared_ptr<ProtonConfigSnapshot> _pendingConfigSnapshot; + std::shared_ptr<ProtonConfigSnapshot> _activeConfigSnapshot; + mutable std::mutex _mutex; + bool _allowReconfig; + vespalib::SimpleComponentConfigProducer _componentConfig; const std::unique_ptr<IProtonDiskLayout> &_diskLayout; + vespalib::MonitoredRefCount _pendingReconfigureTasks; void performReconfigure(); bool skipConfig(const ProtonConfigSnapshot *configSnapshot, bool initialConfig); @@ -48,7 +51,7 @@ class ProtonConfigurer : public IProtonConfigurer void pruneInitialDocumentDBDirs(const ProtonConfigSnapshot &configSnapshot); public: - ProtonConfigurer(vespalib::SyncableThreadExecutor &executor, + ProtonConfigurer(vespalib::ThreadExecutor &executor, IProtonConfigurerOwner &owner, const std::unique_ptr<IProtonDiskLayout> &diskLayout); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp index 73c63da622d..70195980376 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp @@ -5,18 +5,18 @@ #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/util/threadexecutor.h> -using vespalib::SyncableThreadExecutor; +using vespalib::ThreadExecutor; namespace proton { using explorer::convert_executor_to_slime; -ProtonThreadPoolsExplorer::ProtonThreadPoolsExplorer(const SyncableThreadExecutor* shared, - const SyncableThreadExecutor* match, - const SyncableThreadExecutor* docsum, - const SyncableThreadExecutor* flush, - const SyncableThreadExecutor* proton, - const SyncableThreadExecutor* warmup) +ProtonThreadPoolsExplorer::ProtonThreadPoolsExplorer(const ThreadExecutor* shared, + const ThreadExecutor* match, + const ThreadExecutor* docsum, + const ThreadExecutor* flush, + const ThreadExecutor* proton, + const ThreadExecutor* warmup) : _shared(shared), _match(match), _docsum(docsum), diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h index 7f0873a750d..76c5fa8cfb0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h @@ -4,7 +4,7 @@ #include <vespa/vespalib/net/state_explorer.h> -namespace vespalib { class SyncableThreadExecutor; } +namespace vespalib { class ThreadExecutor; } namespace proton { @@ -13,20 +13,20 @@ namespace proton { */ class ProtonThreadPoolsExplorer : public vespalib::StateExplorer { private: - const vespalib::SyncableThreadExecutor* _shared; - const vespalib::SyncableThreadExecutor* _match; - const vespalib::SyncableThreadExecutor* _docsum; - const vespalib::SyncableThreadExecutor* _flush; - const vespalib::SyncableThreadExecutor* _proton; - const vespalib::SyncableThreadExecutor* _warmup; + const vespalib::ThreadExecutor* _shared; + const vespalib::ThreadExecutor* _match; + const vespalib::ThreadExecutor* _docsum; + const vespalib::ThreadExecutor* _flush; + const vespalib::ThreadExecutor* _proton; + const vespalib::ThreadExecutor* _warmup; public: - ProtonThreadPoolsExplorer(const vespalib::SyncableThreadExecutor* shared, - const vespalib::SyncableThreadExecutor* match, - const vespalib::SyncableThreadExecutor* docsum, - const vespalib::SyncableThreadExecutor* flush, - const vespalib::SyncableThreadExecutor* proton, - const vespalib::SyncableThreadExecutor* warmup); + ProtonThreadPoolsExplorer(const vespalib::ThreadExecutor* shared, + const vespalib::ThreadExecutor* match, + const vespalib::ThreadExecutor* docsum, + const vespalib::ThreadExecutor* flush, + const vespalib::ThreadExecutor* proton, + const vespalib::ThreadExecutor* warmup); void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; }; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index ea758443cd4..e6da0e958e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -45,12 +45,12 @@ public: const FastAccessDocSubDB::Context _fastUpdCtx; matching::QueryLimiter &_queryLimiter; const vespalib::Clock &_clock; - vespalib::SyncableThreadExecutor &_warmupExecutor; + vespalib::Executor &_warmupExecutor; Context(const FastAccessDocSubDB::Context &fastUpdCtx, matching::QueryLimiter &queryLimiter, const vespalib::Clock &clock, - vespalib::SyncableThreadExecutor &warmupExecutor) + vespalib:: Executor &warmupExecutor) : _fastUpdCtx(fastUpdCtx), _queryLimiter(queryLimiter), _clock(clock), @@ -70,7 +70,7 @@ private: vespalib::eval::ConstantValueCache _constantValueCache; matching::ConstantValueRepo _constantValueRepo; SearchableDocSubDBConfigurer _configurer; - vespalib::SyncableThreadExecutor &_warmupExecutor; + vespalib::Executor &_warmupExecutor; std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; DocumentDBFlushConfig _flushConfig; diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp new file mode 100644 index 00000000000..04e775674b4 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "shared_threading_service.h" +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/vespalib/util/blockingthreadstackexecutor.h> + +VESPA_THREAD_STACK_TAG(proton_shared_executor) +VESPA_THREAD_STACK_TAG(proton_warmup_executor) + +namespace proton { + +SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfig& cfg) + : _warmup(cfg.warmup_threads(), 128_Ki, proton_warmup_executor), + _shared(std::make_shared<vespalib::BlockingThreadStackExecutor>(cfg.shared_threads(), 128_Ki, + cfg.shared_task_limit(), proton_shared_executor)) +{ +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h new file mode 100644 index 00000000000..ef0ff31c389 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "i_shared_threading_service.h" +#include "shared_threading_service_config.h" +#include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/syncable.h> +#include <memory> + +namespace proton { + +/** + * Class containing the thread executors that are shared across all document dbs. + */ +class SharedThreadingService : public ISharedThreadingService { +private: + vespalib::ThreadStackExecutor _warmup; + std::shared_ptr<vespalib::SyncableThreadExecutor> _shared; + +public: + SharedThreadingService(const SharedThreadingServiceConfig& cfg); + + vespalib::SyncableThreadExecutor& warmup_raw() { return _warmup; } + std::shared_ptr<vespalib::SyncableThreadExecutor> shared_raw() { return _shared; } + + vespalib::ThreadExecutor& warmup() override { return _warmup; } + vespalib::ThreadExecutor& shared() override { return *_shared; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp new file mode 100644 index 00000000000..cf62cf3b76c --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "shared_threading_service_config.h" +#include <vespa/searchcore/config/config-proton.h> +#include <cmath> + +namespace proton { + +using ProtonConfig = SharedThreadingServiceConfig::ProtonConfig; + +SharedThreadingServiceConfig::SharedThreadingServiceConfig(uint32_t shared_threads_in, + uint32_t shared_task_limit_in, + uint32_t warmup_threads_in) + : _shared_threads(shared_threads_in), + _shared_task_limit(shared_task_limit_in), + _warmup_threads(warmup_threads_in) +{ +} + +namespace { + +size_t +derive_shared_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) +{ + size_t scaled_cores = (size_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); + + // We need at least 1 guaranteed free worker in order to ensure progress. + return std::max(scaled_cores, cfg.documentdb.size() + cfg.flush.maxconcurrent + 1); +} + +} + +SharedThreadingServiceConfig +SharedThreadingServiceConfig::make(const proton::SharedThreadingServiceConfig::ProtonConfig& cfg, + const proton::HwInfo::Cpu& cpu_info) +{ + size_t shared_threads = derive_shared_threads(cfg, cpu_info); + return proton::SharedThreadingServiceConfig(shared_threads, shared_threads * 16, 4); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h new file mode 100644 index 00000000000..02966e0efeb --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/searchcore/proton/common/hw_info.h> + +namespace vespa::config::search::core::internal { class InternalProtonType; } + +namespace proton { + +/** + * Config for the thread executors that are shared across all document dbs. + */ +class SharedThreadingServiceConfig { +public: + using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType; + +private: + uint32_t _shared_threads; + uint32_t _shared_task_limit; + uint32_t _warmup_threads; + +public: + SharedThreadingServiceConfig(uint32_t shared_threads_in, + uint32_t shared_task_limit_in, + uint32_t warmup_threads_in); + + static SharedThreadingServiceConfig make(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info); + + uint32_t shared_threads() const { return _shared_threads; } + uint32_t shared_task_limit() const { return _shared_task_limit; } + uint32_t warmup_threads() const { return _warmup_threads; } + +}; + +} + diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h index 4e7a62548c2..c55e451d923 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlyfeedview.h @@ -153,7 +153,7 @@ protected: IGidToLidChangeHandler &_gidToLidChangeHandler; private: - searchcorespi::index::IThreadService & summaryExecutor() { + vespalib::Executor & summaryExecutor() { return _writeService.summary(); } void putSummary(SerialNum serialNum, Lid lid, FutureStream doc, OnOperationDoneType onDone); diff --git a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h index 7f6d9328491..c49649de1e3 100644 --- a/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h +++ b/searchcore/src/vespa/searchcore/proton/summaryengine/summaryengine.h @@ -72,7 +72,7 @@ public: /** * Returns the underlying executor. Only used for state explorers. */ - const vespalib::SyncableThreadExecutor& get_executor() const { return _executor; } + const vespalib::ThreadExecutor& get_executor() const { return _executor; } /** * Starts the underlying threads. This will throw a vespalib::Exception if diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h new file mode 100644 index 00000000000..f21f43ed5ad --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/searchcore/proton/server/i_shared_threading_service.h> + +namespace proton { + +class MockSharedThreadingService : public ISharedThreadingService { +private: + vespalib::ThreadExecutor& _warmup; + vespalib::ThreadExecutor& _shared; + +public: + MockSharedThreadingService(vespalib::ThreadExecutor& warmup_in, + vespalib::ThreadExecutor& shared_in) + : _warmup(warmup_in), + _shared(shared_in) + {} + vespalib::ThreadExecutor& warmup() override { return _warmup; } + vespalib::ThreadExecutor& shared() override { return _shared; } +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h index 26a92841999..0f199e10cb1 100644 --- a/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/thread_service_observer.h @@ -5,6 +5,45 @@ namespace proton::test { +class ThreadExecutorObserver : public vespalib::ThreadExecutor +{ +private: + vespalib::ThreadExecutor &_service; + uint32_t _executeCnt; + +public: + ThreadExecutorObserver(vespalib::ThreadExecutor &service) + : _service(service), + _executeCnt(0) + { + } + + uint32_t getExecuteCnt() const { return _executeCnt; } + + vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override { + ++_executeCnt; + return _service.execute(std::move(task)); + } + + size_t getNumThreads() const override { return _service.getNumThreads(); } + + vespalib::ExecutorStats getStats() override { + return _service.getStats(); + } + + void setTaskLimit(uint32_t taskLimit) override { + _service.setTaskLimit(taskLimit); + } + + uint32_t getTaskLimit() const override { + return _service.getTaskLimit(); + } + + void wakeup() override { + _service.wakeup(); + } +}; + class ThreadServiceObserver : public searchcorespi::index::IThreadService { private: @@ -27,14 +66,56 @@ public: void run(vespalib::Runnable &runnable) override { _service.run(runnable); } + + bool isCurrentThread() const override { + return _service.isCurrentThread(); + } + size_t getNumThreads() const override { return _service.getNumThreads(); } + + vespalib::ExecutorStats getStats() override { + return _service.getStats(); + } + + void setTaskLimit(uint32_t taskLimit) override { + _service.setTaskLimit(taskLimit); + } + + uint32_t getTaskLimit() const override { + return _service.getTaskLimit(); + } + + void wakeup() override { + _service.wakeup(); + } +}; + +class SyncableThreadServiceObserver : public searchcorespi::index::ISyncableThreadService +{ +private: + searchcorespi::index::ISyncableThreadService &_service; + uint32_t _executeCnt; + +public: + SyncableThreadServiceObserver(searchcorespi::index::ISyncableThreadService &service) + : _service(service), + _executeCnt(0) + { + } + + uint32_t getExecuteCnt() const { return _executeCnt; } + + vespalib::Executor::Task::UP execute(vespalib::Executor::Task::UP task) override { + ++_executeCnt; + return _service.execute(std::move(task)); + } + void run(vespalib::Runnable &runnable) override { + _service.run(runnable); + } vespalib::Syncable &sync() override { _service.sync(); return *this; } - ThreadServiceObserver &shutdown() override { - _service.shutdown(); - return *this; - } + bool isCurrentThread() const override { return _service.isCurrentThread(); } @@ -55,7 +136,6 @@ public: void wakeup() override { _service.wakeup(); } - }; } diff --git a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h index 0c6af0a4d25..d193a9555c3 100644 --- a/searchcore/src/vespa/searchcore/proton/test/thread_utils.h +++ b/searchcore/src/vespa/searchcore/proton/test/thread_utils.h @@ -7,17 +7,6 @@ namespace proton::test { /** - * Run the given function in the master thread and wait until all threads done. - */ -template <typename FunctionType> -void -runInMasterAndSyncAll(searchcorespi::index::IThreadingService &writeService, FunctionType func) -{ - writeService.master().execute(vespalib::makeLambdaTask(std::move(func))); - writeService.sync_all_executors(); -} - -/** * Run the given function in the master thread and wait until done. */ template <typename FunctionType> diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h index 46527362091..e93b1632b3f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h @@ -12,10 +12,10 @@ class ThreadingServiceObserver : public searchcorespi::index::IThreadingService { private: searchcorespi::index::IThreadingService &_service; - ThreadServiceObserver _master; - ThreadServiceObserver _index; - ThreadServiceObserver _summary; - vespalib::ThreadExecutor & _shared; + SyncableThreadServiceObserver _master; + ThreadServiceObserver _index; + ThreadExecutorObserver _summary; + vespalib::ThreadExecutor & _shared; vespalib::SequencedTaskExecutorObserver _indexFieldInverter; vespalib::SequencedTaskExecutorObserver _indexFieldWriter; vespalib::SequencedTaskExecutorObserver _attributeFieldWriter; @@ -23,31 +23,27 @@ private: public: ThreadingServiceObserver(searchcorespi::index::IThreadingService &service); ~ThreadingServiceObserver() override; - const ThreadServiceObserver &masterObserver() const { + const SyncableThreadServiceObserver &masterObserver() const { return _master; } const ThreadServiceObserver &indexObserver() const { return _index; } - const ThreadServiceObserver &summaryObserver() const { + const ThreadExecutorObserver &summaryObserver() const { return _summary; } - void sync_all_executors() override { - _service.sync_all_executors(); - } - void blocking_master_execute(vespalib::Executor::Task::UP task) override { _service.blocking_master_execute(std::move(task)); } - searchcorespi::index::IThreadService &master() override { + searchcorespi::index::ISyncableThreadService &master() override { return _master; } searchcorespi::index::IThreadService &index() override { return _index; } - searchcorespi::index::IThreadService &summary() override { + vespalib::ThreadExecutor &summary() override { return _summary; } vespalib::ThreadExecutor &shared() override { diff --git a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h b/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h index b4e51e2dd1b..f973908b62d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h +++ b/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h @@ -9,7 +9,7 @@ namespace searchcorespi::index { /** * Interface for a single thread used for write tasks. */ -struct IThreadService : public vespalib::SyncableThreadExecutor +struct IThreadService : public vespalib::ThreadExecutor { IThreadService(const IThreadService &) = delete; IThreadService & operator = (const IThreadService &) = delete; @@ -25,6 +25,9 @@ struct IThreadService : public vespalib::SyncableThreadExecutor * Returns whether the current thread is the underlying thread. */ virtual bool isCurrentThread() const = 0; +}; + +struct ISyncableThreadService : public IThreadService, vespalib::Syncable { }; diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index 19469d59d1b..a2bd19c3d29 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -328,10 +328,8 @@ IndexMaintainer::flushMemoryIndex(IMemoryIndex &memoryIndex, updateDiskIndexSchema(flushDir, *prunedSchema, noSerialNumHigh); } IndexWriteUtilities::writeSourceSelector(saveInfo, indexId, getAttrTune(), - _ctx.getFileHeaderContext(), - serialNum); - IndexWriteUtilities::writeSerialNum(serialNum, flushDir, - _ctx.getFileHeaderContext()); + _ctx.getFileHeaderContext(), serialNum); + IndexWriteUtilities::writeSerialNum(serialNum, flushDir, _ctx.getFileHeaderContext()); return loadDiskIndex(flushDir); } @@ -696,7 +694,7 @@ IndexMaintainer::doneFusion(FusionArgs *args, IDiskIndex::SP *new_index) } bool -IndexMaintainer::makeSureAllRemainingWarmupIsDone(ISearchableIndexCollection::SP keepAlive) +IndexMaintainer::makeSureAllRemainingWarmupIsDone(std::shared_ptr<WarmupIndexCollection> keepAlive) { // called by warmupDone via reconfigurer, warmupDone() doesn't wait for us assert(_ctx.getThreadingService().master().isCurrentThread()); @@ -713,13 +711,13 @@ IndexMaintainer::makeSureAllRemainingWarmupIsDone(ISearchableIndexCollection::SP LOG(info, "New index warmed up and switched in : %s", warmIndex->toString().c_str()); } LOG(info, "Sync warmupExecutor."); - _ctx.getWarmupExecutor().sync(); + keepAlive->drainPending(); LOG(info, "Now the keep alive of the warmupindexcollection should be gone."); return true; } void -IndexMaintainer::warmupDone(ISearchableIndexCollection::SP current) +IndexMaintainer::warmupDone(std::shared_ptr<WarmupIndexCollection> current) { // Called by a search thread LockGuard lock(_new_search_lock); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h index 6e4eb32ee50..8213c02b90c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h @@ -257,8 +257,8 @@ class IndexMaintainer : public IIndexManager, * result. */ bool reconfigure(std::unique_ptr<Configure> configure); - void warmupDone(ISearchableIndexCollection::SP current) override; - bool makeSureAllRemainingWarmupIsDone(ISearchableIndexCollection::SP keepAlive); + void warmupDone(std::shared_ptr<WarmupIndexCollection> current) override; + bool makeSureAllRemainingWarmupIsDone(std::shared_ptr<WarmupIndexCollection> keepAlive); void commit_and_wait(); void commit(vespalib::Gate& gate); void pruneRemovedFields(const Schema &schema, SerialNum serialNum); diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp index 522789e7fe8..efd7827fc3d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp @@ -11,7 +11,7 @@ namespace searchcorespi::index { IndexMaintainerContext::IndexMaintainerContext(IThreadingService &threadingService, IIndexManager::Reconfigurer &reconfigurer, const FileHeaderContext &fileHeaderContext, - vespalib::SyncableThreadExecutor & warmupExecutor) + vespalib::Executor & warmupExecutor) : _threadingService(threadingService), _reconfigurer(reconfigurer), _fileHeaderContext(fileHeaderContext), diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h index c90659c55bf..2c7aa4af48e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h @@ -17,13 +17,13 @@ private: IThreadingService &_threadingService; IIndexManager::Reconfigurer &_reconfigurer; const search::common::FileHeaderContext &_fileHeaderContext; - vespalib::SyncableThreadExecutor & _warmupExecutor; + vespalib::Executor & _warmupExecutor; public: IndexMaintainerContext(IThreadingService &threadingService, IIndexManager::Reconfigurer &reconfigurer, const search::common::FileHeaderContext &fileHeaderContext, - vespalib::SyncableThreadExecutor & warmupExecutor); + vespalib::Executor & warmupExecutor); /** * Returns the treading service that encapsulates the thread model used for writing. @@ -49,7 +49,7 @@ public: /** * @return The executor that should be used for warmup. */ - vespalib::SyncableThreadExecutor & getWarmupExecutor() const { return _warmupExecutor; } + vespalib::Executor & getWarmupExecutor() const { return _warmupExecutor; } }; } diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h index 0660f3ab495..c95a42f601b 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h +++ b/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h @@ -63,17 +63,15 @@ struct IThreadingService IThreadingService() = default; virtual ~IThreadingService() = default; - virtual void sync_all_executors() = 0; - /** * Block the calling thread until the master thread has capacity to handle more tasks, * and then execute the given task in the master thread. */ virtual void blocking_master_execute(vespalib::Executor::Task::UP task) = 0; - virtual IThreadService &master() = 0; + virtual ISyncableThreadService &master() = 0; virtual IThreadService &index() = 0; - virtual IThreadService &summary() = 0; + virtual vespalib::ThreadExecutor &summary() = 0; virtual vespalib::ThreadExecutor &shared() = 0; virtual vespalib::ISequencedTaskExecutor &indexFieldInverter() = 0; virtual vespalib::ISequencedTaskExecutor &indexFieldWriter() = 0; diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp index d6aba7c6ff1..b9cbdab1c0a 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/stllike/hash_set.h> +#include <thread> #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.warmupindexcollection"); @@ -29,7 +30,7 @@ WarmupIndexCollection::WarmupIndexCollection(const WarmupConfig & warmupConfig, ISearchableIndexCollection::SP prev, ISearchableIndexCollection::SP next, IndexSearchable & warmup, - vespalib::SyncableThreadExecutor & executor, + vespalib::Executor & executor, IWarmupDone & warmupDone) : _warmupConfig(warmupConfig), _prev(std::move(prev)), @@ -38,7 +39,8 @@ WarmupIndexCollection::WarmupIndexCollection(const WarmupConfig & warmupConfig, _executor(executor), _warmupDone(warmupDone), _warmupEndTime(vespalib::steady_clock::now() + warmupConfig.getDuration()), - _handledTerms(std::make_unique<FieldTermMap>()) + _handledTerms(std::make_unique<FieldTermMap>()), + _pendingTasks() { if (_next->valid()) { setCurrentIndex(_next->getCurrentIndex()); @@ -79,7 +81,7 @@ WarmupIndexCollection::~WarmupIndexCollection() if (_warmupEndTime != vespalib::steady_time()) { LOG(info, "Warmup aborted due to new state change or application shutdown"); } - _executor.sync(); + assert(_pendingTasks.has_zero_ref_count()); } const ISourceSelector & @@ -164,7 +166,7 @@ WarmupIndexCollection::createBlueprint(const IRequestContext & requestContext, needWarmUp = needWarmUp || ! handledBefore(fs.getFieldId(), term); } if (needWarmUp) { - auto task = std::make_unique<WarmupTask>(mdl.createMatchData(), *this); + auto task = std::make_unique<WarmupTask>(mdl.createMatchData(), shared_from_this()); task->createBlueprint(fsl, term); fireWarmup(std::move(task)); } @@ -216,25 +218,32 @@ WarmupIndexCollection::getSearchableSP(uint32_t i) const return _next->getSearchableSP(i); } -WarmupIndexCollection::WarmupTask::WarmupTask(std::unique_ptr<MatchData> md, WarmupIndexCollection & warmup) - : _warmup(warmup), +void +WarmupIndexCollection::drainPending() { + _pendingTasks.waitForZeroRefCount(); +} + +WarmupIndexCollection::WarmupTask::WarmupTask(std::unique_ptr<MatchData> md, std::shared_ptr<WarmupIndexCollection> warmup) + : _warmup(std::move(warmup)), + _retainGuard(_warmup->_pendingTasks), _matchData(std::move(md)), _bluePrint(), _requestContext() -{ } +{ +} WarmupIndexCollection::WarmupTask::~WarmupTask() = default; void WarmupIndexCollection::WarmupTask::run() { - if (_warmup._warmupEndTime != vespalib::steady_time()) { + if (_warmup->_warmupEndTime != vespalib::steady_time()) { LOG(debug, "Warming up %s", _bluePrint->asString().c_str()); _bluePrint->fetchPostings(search::queryeval::ExecuteInfo::TRUE); SearchIterator::UP it(_bluePrint->createSearch(*_matchData, true)); it->initFullRange(); for (uint32_t docId = it->seekFirst(1); !it->isAtEnd(); docId = it->seekNext(docId+1)) { - if (_warmup.doUnpack()) { + if (_warmup->doUnpack()) { it->unpack(docId); } } diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h index d18e43b56a7..b0b2952bee8 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h @@ -5,16 +5,19 @@ #include "isearchableindexcollection.h" #include "warmupconfig.h" #include <vespa/vespalib/util/threadexecutor.h> +#include <vespa/vespalib/util/monitored_refcount.h> +#include <vespa/vespalib/util/retain_guard.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> namespace searchcorespi { class FieldTermMap; +class WarmupIndexCollection; class IWarmupDone { public: virtual ~IWarmupDone() { } - virtual void warmupDone(ISearchableIndexCollection::SP current) = 0; + virtual void warmupDone(std::shared_ptr<WarmupIndexCollection> current) = 0; }; /** * Index collection that holds a reference to the active one and a new one that @@ -30,7 +33,7 @@ public: ISearchableIndexCollection::SP prev, ISearchableIndexCollection::SP next, IndexSearchable & warmup, - vespalib::SyncableThreadExecutor & executor, + vespalib::Executor & executor, IWarmupDone & warmupDone); ~WarmupIndexCollection() override; // Implements IIndexCollection @@ -64,28 +67,30 @@ public: const ISearchableIndexCollection::SP & getNextIndexCollection() const { return _next; } vespalib::string toString() const override; bool doUnpack() const { return _warmupConfig.getUnpack(); } + void drainPending(); private: typedef search::fef::MatchData MatchData; typedef search::queryeval::FakeRequestContext FakeRequestContext; typedef vespalib::Executor::Task Task; class WarmupTask : public Task { public: - WarmupTask(std::unique_ptr<MatchData> md, WarmupIndexCollection & warmup); + WarmupTask(std::unique_ptr<MatchData> md, std::shared_ptr<WarmupIndexCollection> warmup); ~WarmupTask() override; WarmupTask &createBlueprint(const FieldSpec &field, const Node &term) { - _bluePrint = _warmup.createBlueprint(_requestContext, field, term); + _bluePrint = _warmup->createBlueprint(_requestContext, field, term); return *this; } WarmupTask &createBlueprint(const FieldSpecList &fields, const Node &term) { - _bluePrint = _warmup.createBlueprint(_requestContext, fields, term); + _bluePrint = _warmup->createBlueprint(_requestContext, fields, term); return *this; } private: void run() override; - WarmupIndexCollection & _warmup; - std::unique_ptr<MatchData> _matchData; - Blueprint::UP _bluePrint; - FakeRequestContext _requestContext; + std::shared_ptr<WarmupIndexCollection> _warmup; + vespalib::RetainGuard _retainGuard; + std::unique_ptr<MatchData> _matchData; + Blueprint::UP _bluePrint; + FakeRequestContext _requestContext; }; void fireWarmup(Task::UP task); @@ -95,11 +100,12 @@ private: ISearchableIndexCollection::SP _prev; ISearchableIndexCollection::SP _next; IndexSearchable & _warmup; - vespalib::SyncableThreadExecutor & _executor; + vespalib::Executor & _executor; IWarmupDone & _warmupDone; vespalib::steady_time _warmupEndTime; std::mutex _lock; std::unique_ptr<FieldTermMap> _handledTerms; + vespalib::MonitoredRefCount _pendingTasks; }; } // namespace searchcorespi diff --git a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp index a54f981352b..f0e156a96ed 100644 --- a/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp +++ b/searchlib/src/tests/tensor/distance_functions/distance_functions_test.cpp @@ -71,7 +71,6 @@ TEST(DistanceFunctionsTest, euclidean_int8_smoketest) auto euclid = make_distance_function(DistanceMetric::Euclidean, ct); - std::vector<double> p00{0.0, 0.0, 0.0}; std::vector<Int8Float> p0{0.0, 0.0, 0.0}; std::vector<Int8Float> p1{1.0, 0.0, 0.0}; std::vector<Int8Float> p5{0.0,-1.0, 0.0}; @@ -85,9 +84,6 @@ TEST(DistanceFunctionsTest, euclidean_int8_smoketest) EXPECT_DOUBLE_EQ(12.0, euclid->calc(t(p1), t(p7))); EXPECT_DOUBLE_EQ(14.0, euclid->calc(t(p5), t(p7))); - EXPECT_DOUBLE_EQ(1.0, euclid->calc(t(p00), t(p1))); - EXPECT_DOUBLE_EQ(1.0, euclid->calc(t(p00), t(p5))); - EXPECT_DOUBLE_EQ(9.0, euclid->calc(t(p00), t(p7))); } TEST(DistanceFunctionsTest, angular_gives_expected_score) diff --git a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp index 315d4c8535c..96dfc580d87 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp +++ b/searchlib/src/vespa/searchlib/tensor/distance_function_factory.cpp @@ -21,6 +21,7 @@ make_distance_function(DistanceMetric variant, CellType cell_type) switch (cell_type) { case CellType::FLOAT: return std::make_unique<SquaredEuclideanDistanceHW<float>>(); case CellType::DOUBLE: return std::make_unique<SquaredEuclideanDistanceHW<double>>(); + case CellType::INT8: return std::make_unique<SquaredEuclideanDistanceHW<vespalib::eval::Int8Float>>(); default: return std::make_unique<SquaredEuclideanDistance>(CellType::FLOAT); } case DistanceMetric::Angular: diff --git a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h index 517ef68511b..6505ea119ea 100644 --- a/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h +++ b/searchlib/src/vespa/searchlib/tensor/euclidean_distance.h @@ -44,6 +44,9 @@ public: assert(expected_cell_type() == vespalib::eval::get_cell_type<FloatType>()); } + static const double *cast(const double * p) { return p; } + static const float *cast(const float * p) { return p; } + static const int8_t *cast(const vespalib::eval::Int8Float * p) { return reinterpret_cast<const int8_t *>(p); } double calc(const vespalib::eval::TypedCells& lhs, const vespalib::eval::TypedCells& rhs) const override { constexpr vespalib::eval::CellType expected = vespalib::eval::get_cell_type<FloatType>(); assert(lhs.type == expected && rhs.type == expected); @@ -51,7 +54,7 @@ public: auto rhs_vector = rhs.typify<FloatType>(); size_t sz = lhs_vector.size(); assert(sz == rhs_vector.size()); - return _computer.squaredEuclideanDistance(&lhs_vector[0], &rhs_vector[0], sz); + return _computer.squaredEuclideanDistance(cast(&lhs_vector[0]), cast(&rhs_vector[0]), sz); } double calc_with_limit(const vespalib::eval::TypedCells& lhs, diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp index 6ab5d42d350..07dbc9a247d 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.cpp +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.cpp @@ -15,17 +15,16 @@ static int cmpname(const void *av, const void *bv) *(const DirectoryTraverse::Name *const *) av; const DirectoryTraverse::Name *const b = *(const DirectoryTraverse::Name *const *) bv; - return strcmp(a->_name, b->_name); + return a->_name.compare(b->_name.c_str()); } } DirectoryTraverse::Name::Name(const char *name) - : _name(nullptr), + : _name(name), _next(nullptr) { - _name = strdup(name); } -DirectoryTraverse::Name::~Name() { free(_name); } +DirectoryTraverse::Name::~Name() = default; DirectoryTraverse::Name * DirectoryTraverse::Name::sort(Name *head, int count) @@ -132,19 +131,15 @@ DirectoryTraverse::ScanSingleDir() assert(_nameHead == nullptr); assert(_nameCount == 0); delete _curDir; - free(_fullDirName); - _fullDirName = nullptr; + _fullDirName.clear(); _curDir = UnQueueDir(); if (_curDir == nullptr) return; - _fullDirName = (char *) malloc(strlen(_baseDir) + 1 + - strlen(_curDir->_name) + 1); - strcpy(_fullDirName, _baseDir); - if (_curDir->_name[0] != '\0') { - strcat(_fullDirName, "/"); - strcat(_fullDirName, _curDir->_name); + _fullDirName = _baseDir; + if ( ! _curDir->_name.empty()) { + _fullDirName += "/" + _curDir->_name; } - FastOS_DirectoryScan *dirscan = new FastOS_DirectoryScan(_fullDirName); + FastOS_DirectoryScan *dirscan = new FastOS_DirectoryScan(_fullDirName.c_str()); while (dirscan->ReadNext()) { const char *name = dirscan->GetName(); if (strcmp(name, ".") == 0 || @@ -171,13 +166,8 @@ DirectoryTraverse::NextName() if (_nameHead == nullptr) return false; _curName = UnQueueName(); - free(_fullName); - _fullName = (char *) malloc(strlen(_fullDirName) + 1 + - strlen(_curName->_name) + 1); - strcpy(_fullName, _fullDirName); - _relName = _fullName + strlen(_baseDir) + 1; - strcat(_fullName, "/"); - strcat(_fullName, _curName->_name); + _fullName = _fullDirName + "/" + _curName->_name; + _relName = _fullName.c_str() + (_baseDir.size() + 1); return true; } @@ -193,13 +183,8 @@ DirectoryTraverse::NextRemoveDir() return false; curName = _rdirHead; _rdirHead = curName->_next; - free(_fullName); - _fullName = (char *) malloc(strlen(_baseDir) + 1 + - strlen(curName->_name) + 1); - strcpy(_fullName, _baseDir); - _relName = _fullName + strlen(_baseDir) + 1; - strcat(_fullName, "/"); - strcat(_fullName, curName->_name); + _fullName = _baseDir + "/" + curName->_name; + _relName = _fullName.c_str() + _baseDir.size() + 1; delete curName; return true; } @@ -226,7 +211,7 @@ DirectoryTraverse::RemoveTree() const char *fullname = GetFullName(); FastOS_File::RemoveDirectory(fullname); } - FastOS_File::RemoveDirectory(_baseDir); + FastOS_File::RemoveDirectory(_baseDir.c_str()); return true; } @@ -252,7 +237,7 @@ DirectoryTraverse::GetTreeSize() } DirectoryTraverse::DirectoryTraverse(const char *baseDir) - : _baseDir(nullptr), + : _baseDir(baseDir), _nameHead(nullptr), _nameCount(0), _dirHead(nullptr), @@ -261,11 +246,10 @@ DirectoryTraverse::DirectoryTraverse(const char *baseDir) _rdirHead(nullptr), _curDir(nullptr), _curName(nullptr), - _fullDirName(nullptr), - _fullName(nullptr), + _fullDirName(), + _fullName(), _relName(nullptr) { - _baseDir = strdup(baseDir); QueueDir(""); ScanSingleDir(); } @@ -273,9 +257,6 @@ DirectoryTraverse::DirectoryTraverse(const char *baseDir) DirectoryTraverse::~DirectoryTraverse() { - free(_fullDirName); - free(_fullName); - free(_baseDir); delete _curDir; delete _curName; PushPushedDirs(); diff --git a/searchlib/src/vespa/searchlib/util/dirtraverse.h b/searchlib/src/vespa/searchlib/util/dirtraverse.h index bff7aae705a..4a96ad0935d 100644 --- a/searchlib/src/vespa/searchlib/util/dirtraverse.h +++ b/searchlib/src/vespa/searchlib/util/dirtraverse.h @@ -3,6 +3,7 @@ #pragma once #include <cstdint> +#include <string> namespace search { @@ -20,14 +21,14 @@ public: Name& operator=(const Name &); public: - char *_name; + std::string _name; Name *_next; explicit Name(const char *name); ~Name(); static Name *sort(Name *head, int count); }; private: - char *_baseDir; + std::string _baseDir; Name *_nameHead; int _nameCount; Name *_dirHead; @@ -36,11 +37,11 @@ private: Name *_rdirHead; Name *_curDir; Name *_curName; - char *_fullDirName; - char *_fullName; - char *_relName; + std::string _fullDirName; + std::string _fullName; + const char *_relName; public: - const char *GetFullName() const { return _fullName; } + const char *GetFullName() const { return _fullName.c_str(); } const char *GetRelName() const { return _relName; } void QueueDir(const char *name); void PushDir(const char *name); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp index bd05dd4b0f5..19f6a7eef01 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "docsumstate.h" #include "keywordextractor.h" #include "idocsumenvironment.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> @@ -20,7 +19,7 @@ bool useful(search::ParseItem::ItemCreator creator) KeywordExtractor::KeywordExtractor(IDocsumEnvironment * env) : _env(env), - _legalPrefixes(NULL), + _legalPrefixes(nullptr), _legalIndexes() { } @@ -28,7 +27,7 @@ KeywordExtractor::KeywordExtractor(IDocsumEnvironment * env) KeywordExtractor::~KeywordExtractor() { - while (_legalPrefixes != NULL) { + while (_legalPrefixes != nullptr) { IndexPrefix *tmp = _legalPrefixes; _legalPrefixes = tmp->_next; delete tmp; @@ -42,32 +41,25 @@ KeywordExtractor::IsLegalIndexName(const char *idxName) const } KeywordExtractor::IndexPrefix::IndexPrefix(const char *prefix, IndexPrefix **list) - : _prefix(NULL), - _prefixLen(0), - _next(NULL) + : _prefix(prefix), + _next(nullptr) { - _prefix = strdup(prefix); - assert(_prefix != NULL); - _prefixLen = strlen(prefix); _next = *list; *list = this; } -KeywordExtractor::IndexPrefix::~IndexPrefix() -{ - free(_prefix); -} +KeywordExtractor::IndexPrefix::~IndexPrefix() = default; bool KeywordExtractor::IndexPrefix::Match(const char *idxName) const { - return (strncmp(idxName, _prefix, _prefixLen) == 0); + return vespalib::starts_with(idxName, _prefix); } void KeywordExtractor::AddLegalIndexSpec(const char *spec) { - if (spec == NULL) + if (spec == nullptr) return; vespalib::string toks(spec); // tokens @@ -107,9 +99,9 @@ KeywordExtractor::GetLegalIndexSpec() { vespalib::string spec; - if (_legalPrefixes != NULL) { + if (_legalPrefixes != nullptr) { for (IndexPrefix *pt = _legalPrefixes; - pt != NULL; pt = pt->_next) { + pt != nullptr; pt = pt->_next) { if (spec.size() > 0) spec.append(';'); spec.append(pt->_prefix); @@ -131,7 +123,7 @@ KeywordExtractor::IsLegalIndex(vespalib::stringref idxS) const { vespalib::string resolvedIdxName; - if (_env != NULL) { + if (_env != nullptr) { resolvedIdxName = _env->lookupIndex(idxS); } else { @@ -238,7 +230,7 @@ KeywordExtractor::ExtractKeywords(vespalib::stringref buf) const // Must now allocate a string and copy the data from the rawbuf void *result = malloc(keywords.GetUsedLen()); - if (result != NULL) { + if (result != nullptr) { memcpy(result, keywords.GetDrainPos(), keywords.GetUsedLen()); } return static_cast<char *>(result); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h index 50d72f7a7d0..44c85121058 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/keywordextractor.h @@ -24,9 +24,8 @@ public: IndexPrefix& operator=(const IndexPrefix &); public: - char *_prefix; - int _prefixLen; - IndexPrefix *_next; + vespalib::string _prefix; + IndexPrefix *_next; IndexPrefix(const char *prefix, IndexPrefix **list); ~IndexPrefix(); @@ -42,7 +41,7 @@ private: bool IsLegalIndexPrefix(const char *idxName) const { for (const IndexPrefix *pt = _legalPrefixes; - pt != NULL; + pt != nullptr; pt = pt->_next) { if (pt->Match(idxName)) diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index 954a63978f3..7c4711b6802 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -167,7 +167,7 @@ SequencedTaskExecutor::getExecutorIdImPerfect(uint64_t componentId) const { return ExecutorId(executorId); } -const vespalib::SyncableThreadExecutor* +const vespalib::ThreadExecutor* SequencedTaskExecutor::first_executor() const { if (_executors.empty()) { diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h index 7bb56424849..db0723d16c8 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ b/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h @@ -8,6 +8,7 @@ namespace vespalib { +class ThreadExecutor; class SyncableThreadExecutor; /** @@ -41,7 +42,7 @@ public: */ uint32_t getComponentHashSize() const { return _component2IdImperfect.size(); } uint32_t getComponentEffectiveHashSize() const { return _nextId; } - const vespalib::SyncableThreadExecutor* first_executor() const; + const vespalib::ThreadExecutor* first_executor() const; private: explicit SequencedTaskExecutor(std::vector<std::unique_ptr<vespalib::SyncableThreadExecutor>> executor); diff --git a/storage/src/tests/persistence/apply_bucket_diff_state_test.cpp b/storage/src/tests/persistence/apply_bucket_diff_state_test.cpp index cdf7ac817fa..ec57d775f43 100644 --- a/storage/src/tests/persistence/apply_bucket_diff_state_test.cpp +++ b/storage/src/tests/persistence/apply_bucket_diff_state_test.cpp @@ -3,6 +3,7 @@ #include <vespa/storage/persistence/apply_bucket_diff_state.h> #include <vespa/storage/persistence/merge_bucket_info_syncer.h> #include <vespa/storage/persistence/filestorage/merge_handler_metrics.h> +#include <vespa/storageapi/message/bucket.h> #include <vespa/document/base/documentid.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/document/bucket/bucketidfactory.h> @@ -10,6 +11,8 @@ #include <vespa/metrics/metricset.h> #include <vespa/persistence/spi/result.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> +#include <tests/common/message_sender_stub.h> +#include <tests/persistence/persistencetestutils.h> #include <gtest/gtest.h> using document::DocumentId; @@ -73,22 +76,25 @@ void push_bad(ApplyBucketDiffState &state) } -class ApplyBucketDiffStateTestBase : public ::testing::Test +class ApplyBucketDiffStateTestBase : public PersistenceTestUtils { public: uint32_t sync_count; DummyMergeBucketInfoSyncer syncer; metrics::MetricSet merge_handler_metrics_owner; MergeHandlerMetrics merge_handler_metrics; + FileStorThreadMetrics::Op op_metrics; framework::defaultimplementation::FakeClock clock; + MessageSenderStub message_sender; MonitoredRefCount monitored_ref_count; ApplyBucketDiffStateTestBase() - : ::testing::Test(), + : PersistenceTestUtils(), sync_count(0u), syncer(sync_count), merge_handler_metrics_owner("owner", {}, "owner"), merge_handler_metrics(&merge_handler_metrics_owner), + op_metrics("op", "op", &merge_handler_metrics_owner), clock(), monitored_ref_count() { @@ -99,6 +105,13 @@ public: std::shared_ptr<ApplyBucketDiffState> make_state() { return ApplyBucketDiffState::create(syncer, merge_handler_metrics, clock, spi::Bucket(dummy_document_bucket), RetainGuard(monitored_ref_count)); } + + MessageTracker::UP + create_tracker(std::shared_ptr<api::StorageMessage> cmd, document::Bucket bucket) { + return MessageTracker::createForTesting(framework::MilliSecTimer(clock), getEnv(), + message_sender, NoBucketLock::make(bucket), std::move(cmd)); + } + }; ApplyBucketDiffStateTestBase::~ApplyBucketDiffStateTestBase() = default; @@ -128,8 +141,44 @@ public: check_failure("Failed put for id::test::1 in Bucket(0x0000000000000010): Result(5, write blocked)"); } + void test_delayed_reply(bool failed, bool async_failed, bool chained_reply); + }; +void +ApplyBucketDiffStateTest::test_delayed_reply(bool failed, bool async_failed, bool chained_reply) +{ + auto cmd = std::make_shared<api::MergeBucketCommand>(dummy_document_bucket, std::vector<api::MergeBucketCommand::Node>{}, 0); + std::shared_ptr<api::StorageReply> reply = cmd->makeReply(); + auto tracker = create_tracker(cmd, dummy_document_bucket); + if (failed) { + reply->setResult(api::ReturnCode::Result::INTERNAL_FAILURE); + } + tracker->setMetric(op_metrics); + tracker->setReply(reply); + if (chained_reply) { + state->set_delayed_reply(std::move(tracker), message_sender, &op_metrics, framework::MilliSecTimer(clock), std::move(reply)); + } else { + state->set_delayed_reply(std::move(tracker), std::move(reply)); + } + clock.addMilliSecondsToTime(16); + if (async_failed) { + push_bad(*state); + } + state.reset(); + if (failed || async_failed) { + EXPECT_EQ(0.0, op_metrics.latency.getLast()); + EXPECT_EQ(0, op_metrics.latency.getCount()); + EXPECT_EQ(1, op_metrics.failed.getValue()); + } else { + EXPECT_EQ(16.0, op_metrics.latency.getLast()); + EXPECT_EQ(1, op_metrics.latency.getCount()); + EXPECT_EQ(0, op_metrics.failed.getValue()); + } + ASSERT_EQ(1, message_sender.replies.size()); + EXPECT_NE(failed || async_failed, std::dynamic_pointer_cast<api::MergeBucketReply>(message_sender.replies.front())->getResult().success()); +} + TEST_F(ApplyBucketDiffStateTest, ok_results_can_be_checked) { push_ok(*state); @@ -203,4 +252,29 @@ TEST_F(ApplyBucketDiffStateTest, total_latency_is_updated) EXPECT_EQ(1, merge_handler_metrics.mergeLatencyTotal.getCount()); } +TEST_F(ApplyBucketDiffStateTest, delayed_ok_reply) +{ + test_delayed_reply(false, false, false); +} + +TEST_F(ApplyBucketDiffStateTest, delayed_failed_reply) +{ + test_delayed_reply(true, false, false); +} + +TEST_F(ApplyBucketDiffStateTest, delayed_ok_chained_reply) +{ + test_delayed_reply(false, false, true); +} + +TEST_F(ApplyBucketDiffStateTest, delayed_failed_chained_reply) +{ + test_delayed_reply(true, false, true); +} + +TEST_F(ApplyBucketDiffStateTest, delayed_async_failed_reply) +{ + test_delayed_reply(false, true, false); +} + } diff --git a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp index 3aa231e1790..a16eef0ab6f 100644 --- a/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/updateoperation.cpp @@ -155,13 +155,13 @@ UpdateOperation::onReceive(DistributorStripeMessageSender& sender, for (uint32_t i = 0; i < _results.size(); i++) { if (_results[i].oldTs < oldTs) { - LOG(error, "Update operation for '%s' in bucket %s updated documents with different timestamps. " - "This should not happen and may indicate undetected replica divergence. " - "Found ts=%" PRIu64 " on node %u, ts=%" PRIu64 " on node %u", - reply.getDocumentId().toString().c_str(), - reply.getBucket().toString().c_str(), - _results[i].oldTs, _results[i].nodeId, - _results[goodNode].oldTs, _results[goodNode].nodeId); + LOG(warning, "Update operation for '%s' in bucket %s updated documents with different timestamps. " + "This should not happen and may indicate undetected replica divergence. " + "Found ts=%" PRIu64 " on node %u, ts=%" PRIu64 " on node %u", + reply.getDocumentId().toString().c_str(), + reply.getBucket().toString().c_str(), + _results[i].oldTs, _results[i].nodeId, + _results[goodNode].oldTs, _results[goodNode].nodeId); _metrics.diverging_timestamp_updates.inc(); replyToSend.setNodeWithNewestTimestamp(_results[goodNode].nodeId); diff --git a/storage/src/vespa/storage/persistence/apply_bucket_diff_state.cpp b/storage/src/vespa/storage/persistence/apply_bucket_diff_state.cpp index e976cefbec3..07823792062 100644 --- a/storage/src/vespa/storage/persistence/apply_bucket_diff_state.cpp +++ b/storage/src/vespa/storage/persistence/apply_bucket_diff_state.cpp @@ -34,7 +34,10 @@ ApplyBucketDiffState::ApplyBucketDiffState(const MergeBucketInfoSyncer& merge_bu _tracker(), _delayed_reply(), _sender(nullptr), - _retain_guard(std::move(retain_guard)) + _op_metrics(nullptr), + _op_start_time(), + _retain_guard(std::move(retain_guard)), + _merge_start_time() { } @@ -59,6 +62,15 @@ ApplyBucketDiffState::~ApplyBucketDiffState() _delayed_reply->setResult(api::ReturnCode(api::ReturnCode::INTERNAL_FAILURE, _fail_message)); } if (_sender) { + if (_op_metrics != nullptr) { + if (_delayed_reply->getResult().success()) { + if (_op_start_time.has_value()) { + _op_metrics->latency.addValue(_op_start_time.value().getElapsedTimeAsDouble()); + } + } else { + _op_metrics->failed.inc(); + } + } _sender->sendReply(std::move(_delayed_reply)); } else { // _tracker->_reply and _delayed_reply points to the same reply. @@ -110,10 +122,12 @@ ApplyBucketDiffState::set_delayed_reply(std::unique_ptr<MessageTracker>&& tracke } void -ApplyBucketDiffState::set_delayed_reply(std::unique_ptr<MessageTracker>&& tracker, MessageSender& sender, std::shared_ptr<api::StorageReply>&& delayed_reply) +ApplyBucketDiffState::set_delayed_reply(std::unique_ptr<MessageTracker>&& tracker, MessageSender& sender, FileStorThreadMetrics::Op* op_metrics, const framework::MilliSecTimer& op_start_time, std::shared_ptr<api::StorageReply>&& delayed_reply) { _tracker = std::move(tracker); _sender = &sender; + _op_metrics = op_metrics; + _op_start_time = op_start_time; _delayed_reply = std::move(delayed_reply); } diff --git a/storage/src/vespa/storage/persistence/apply_bucket_diff_state.h b/storage/src/vespa/storage/persistence/apply_bucket_diff_state.h index 0433cd4e108..49625bbf8b5 100644 --- a/storage/src/vespa/storage/persistence/apply_bucket_diff_state.h +++ b/storage/src/vespa/storage/persistence/apply_bucket_diff_state.h @@ -4,6 +4,7 @@ #include <vespa/persistence/spi/bucket.h> #include <vespa/storageframework/generic/clock/timer.h> +#include <vespa/storage/persistence/filestorage/filestormetrics.h> #include <vespa/vespalib/util/retain_guard.h> #include <future> #include <memory> @@ -20,7 +21,6 @@ class ApplyBucketDiffEntryResult; struct MessageSender; class MessageTracker; class MergeBucketInfoSyncer; -struct MergeHandlerMetrics; /* * State of all bucket diff entry spi operation (putAsync or removeAsync) @@ -39,6 +39,8 @@ class ApplyBucketDiffState { std::unique_ptr<MessageTracker> _tracker; std::shared_ptr<api::StorageReply> _delayed_reply; MessageSender* _sender; + FileStorThreadMetrics::Op* _op_metrics; + std::optional<framework::MilliSecTimer> _op_start_time; vespalib::RetainGuard _retain_guard; std::optional<framework::MilliSecTimer> _merge_start_time; @@ -53,7 +55,7 @@ public: void sync_bucket_info(); std::future<vespalib::string> get_future(); void set_delayed_reply(std::unique_ptr<MessageTracker>&& tracker, std::shared_ptr<api::StorageReply>&& delayed_reply); - void set_delayed_reply(std::unique_ptr<MessageTracker>&& tracker, MessageSender& sender, std::shared_ptr<api::StorageReply>&& delayed_reply); + void set_delayed_reply(std::unique_ptr<MessageTracker>&& tracker, MessageSender& sender, FileStorThreadMetrics::Op* op_metrics, const framework::MilliSecTimer& op_start_time, std::shared_ptr<api::StorageReply>&& delayed_reply); void set_tracker(std::unique_ptr<MessageTracker>&& tracker); void set_merge_start_time(const framework::MilliSecTimer& merge_start_time); const spi::Bucket& get_bucket() const noexcept { return _bucket; } diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index ee62da95f55..0c9cecdb6a1 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -104,6 +104,29 @@ void check_apply_diff_sync(std::shared_ptr<ApplyBucketDiffState> async_results) } } +FileStorThreadMetrics::Op *get_op_metrics(FileStorThreadMetrics& metrics, const api::StorageReply &reply) { + switch (reply.getType().getId()) { + case api::MessageType::MERGEBUCKET_REPLY_ID: + return &metrics.mergeBuckets; + case api::MessageType::APPLYBUCKETDIFF_REPLY_ID: + return &metrics.applyBucketDiff; + default: + ; + } + return nullptr; +} + +void update_op_metrics(FileStorThreadMetrics& metrics, const api::StorageReply &reply, const framework::MilliSecTimer& start_time) { + auto op_metrics = get_op_metrics(metrics, reply); + if (op_metrics) { + if (reply.getResult().success()) { + op_metrics->latency.addValue(start_time.getElapsedTimeAsDouble()); + } else { + op_metrics->failed.inc(); + } + } +} + } // anonymous namespace void @@ -1223,6 +1246,7 @@ MergeHandler::handleGetBucketDiffReply(api::GetBucketDiffReply& reply, MessageSe } if (replyToSend.get()) { replyToSend->setResult(reply.getResult()); + update_op_metrics(_env._metrics, *replyToSend, s->startTime); sender.sendReply(replyToSend); } } @@ -1433,7 +1457,8 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, Messa if (async_results && replyToSend) { replyToSend->setResult(returnCode); - async_results->set_delayed_reply(std::move(tracker), sender, std::move(replyToSend)); + auto op_metrics = get_op_metrics(_env._metrics, *replyToSend); + async_results->set_delayed_reply(std::move(tracker), sender, op_metrics, s->startTime, std::move(replyToSend)); } if (clearState) { _env._fileStorHandler.clearMergeStatus(bucket.getBucket()); @@ -1441,6 +1466,7 @@ MergeHandler::handleApplyBucketDiffReply(api::ApplyBucketDiffReply& reply, Messa if (replyToSend.get()) { // Send on replyToSend->setResult(returnCode); + update_op_metrics(_env._metrics, *replyToSend, s->startTime); sender.sendReply(replyToSend); } } diff --git a/tenant-base/pom.xml b/tenant-base/pom.xml index acf62b1d6e6..f4923bf79f1 100644 --- a/tenant-base/pom.xml +++ b/tenant-base/pom.xml @@ -40,7 +40,7 @@ <target_jdk_version>11</target_jdk_version> <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.0</maven-surefire-plugin.version> - <junit.version>5.7.0</junit.version> <!-- Keep in sync with hosted-tenant-base and tenant-cd until all direct use is removed --> + <junit.version>5.8.1</junit.version> <!-- Keep in sync with hosted-tenant-base and tenant-cd until all direct use is removed --> <endpoint>https://api.vespa-external.aws.oath.cloud:4443</endpoint> <test.categories>!integration</test.categories> </properties> diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml index e45fd7e9586..5ac52748152 100644 --- a/tenant-cd-api/pom.xml +++ b/tenant-cd-api/pom.xml @@ -25,7 +25,7 @@ This version must match the string in all ExportPackage annotations in this module. It must also be in sync junit version specified in 'hosted-tenant-base'. --> - <hosted-tenant-base-junit-version>5.7.0</hosted-tenant-base-junit-version> + <hosted-tenant-base-junit-version>5.8.1</hosted-tenant-base-junit-version> </properties> diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java index 7d14bb7fd18..cc511f67528 100644 --- a/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/condition/package-info.java @@ -2,7 +2,7 @@ /** * @author bjorncs */ -@ExportPackage(version = @Version(major = 5, minor = 7, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) package org.junit.jupiter.api.condition; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java index 14080a1cb51..7ddb6761e06 100644 --- a/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/package-info.java @@ -2,7 +2,7 @@ /** * @author bjorncs */ -@ExportPackage(version = @Version(major = 5, minor = 7, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) package org.junit.jupiter.api.extension; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java index cd0efd44e3c..72f3fd82347 100644 --- a/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/function/package-info.java @@ -2,7 +2,7 @@ /** * @author bjorncs */ -@ExportPackage(version = @Version(major = 5, minor = 7, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) package org.junit.jupiter.api.function; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java index 84b43e8e243..374aa823308 100644 --- a/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/io/package-info.java @@ -2,7 +2,7 @@ /** * @author bjorncs */ -@ExportPackage(version = @Version(major = 5, minor = 7, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) package org.junit.jupiter.api.io; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java index da5e4f19c1f..75e33914d6c 100644 --- a/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/package-info.java @@ -2,7 +2,7 @@ /** * @author bjorncs */ -@ExportPackage(version = @Version(major = 5, minor = 7, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) package org.junit.jupiter.api; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java index be7342d2f29..2abd95827e4 100644 --- a/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/parallel/package-info.java @@ -2,7 +2,7 @@ /** * @author bjorncs */ -@ExportPackage(version = @Version(major = 5, minor = 7, micro = 0)) +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) package org.junit.jupiter.api.parallel; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/valgrind-suppressions.txt b/valgrind-suppressions.txt index 75a7b256b35..63fd7857e76 100644 --- a/valgrind-suppressions.txt +++ b/valgrind-suppressions.txt @@ -2,6 +2,13 @@ NPTL keeps a cache of thread stacks, and metadata for thread local storage is not freed for threads in that cache Memcheck:Leak fun:calloc + fun:_dl_allocate_tls + fun:pthread_create@@GLIBC_2.17 +} +{ + NPTL keeps a cache of thread stacks, and metadata for thread local storage is not freed for threads in that cache + Memcheck:Leak + fun:calloc fun:UnknownInlinedFun fun:allocate_dtv fun:_dl_allocate_tls diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 4a3dc30d7ed..ce12637ccb0 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -385,6 +385,13 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { return Set.copyOf(listResponse.entity); } + @Override + public void deleteRole(AthenzRole role) { + URI uri = zmsUrl.resolve(String.format("domain/%s/role/%s", role.domain().getName(), role.roleName())); + HttpUriRequest request = RequestBuilder.delete(uri).build(); + execute(request, response -> readEntity(response, Void.class)); + } + private static Header createCookieHeaderWithOktaTokens(OktaIdentityToken identityToken, OktaAccessToken accessToken) { return new BasicHeader("Cookie", String.format("okta_at=%s; okta_it=%s", accessToken.token(), identityToken.token())); } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 823b5843115..aa038b5bb23 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -79,5 +79,7 @@ public interface ZmsClient extends AutoCloseable { Set<String> listPolicies(AthenzDomain domain); + void deleteRole(AthenzRole athenzRole); + void close(); } diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index 6ec70b08d39..045ac050e4b 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -25,7 +25,7 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>5.7.0</version> + <version>5.8.1</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> @@ -36,7 +36,7 @@ <dependency> <groupId>org.junit.platform</groupId> <artifactId>junit-platform-launcher</artifactId> - <version>1.6.2</version> + <version>1.8.1</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java index cd5c109850b..8599babd0a5 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -118,6 +118,7 @@ public class VespaCliTestRunner implements TestRunner { case SYSTEM_TEST: return "system-test"; case STAGING_SETUP_TEST: return "staging-setup"; case STAGING_TEST: return "staging-test"; + case PRODUCTION_TEST: return "production-test"; default: throw new IllegalArgumentException("Unsupported test suite '" + suite + "'"); } } diff --git a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java index ff66b31dfa8..e232e523cbf 100644 --- a/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java +++ b/vespa-testrunner-components/src/main/java/com/yahoo/vespa/hosted/testrunner/PomXmlGenerator.java @@ -37,7 +37,7 @@ public class PomXmlGenerator { " <version>1.0.0</version>\n" + "\n" + " <properties>\n" + - " <junit_version>5.7.0</junit_version>\n" + + " <junit_version>5.8.1</junit_version>\n" + " <surefire_version>2.22.0</surefire_version>\n" + "%PROPERTIES%" + " </properties>\n" + diff --git a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests index 269bbd2a3c3..b07bc9a26cc 100644 --- a/vespa-testrunner-components/src/test/resources/pom.xml_system_tests +++ b/vespa-testrunner-components/src/test/resources/pom.xml_system_tests @@ -6,7 +6,7 @@ <version>1.0.0</version> <properties> - <junit_version>5.7.0</junit_version> + <junit_version>5.8.1</junit_version> <surefire_version>2.22.0</surefire_version> <my-comp.jar.path>components/my-comp.jar</my-comp.jar.path> <main.jar.path>main.jar</main.jar.path> diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java index 3faafd7b2e5..50f79c0a828 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/FeedReplyReader.java @@ -12,6 +12,7 @@ import com.yahoo.messagebus.Trace; import com.yahoo.vespa.http.client.core.ErrorCode; import com.yahoo.vespa.http.client.core.OperationStatus; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -25,10 +26,12 @@ public class FeedReplyReader implements ReplyHandler { private static final Logger log = Logger.getLogger(FeedReplyReader.class.getName()); private final Metric metric; private final DocumentApiMetrics metricsHelper; + private final Metric.Context testAndSetMetricCtx; public FeedReplyReader(Metric metric, DocumentApiMetrics metricsHelper) { this.metric = metric; this.metricsHelper = metricsHelper; + this.testAndSetMetricCtx = metric.createContext(Map.of("operationType", "testAndSet")); } @Override @@ -52,7 +55,7 @@ public class FeedReplyReader implements ReplyHandler { metricsHelper.reportSuccessful(type, latencyInSeconds); metric.add(MetricNames.SUCCEEDED, 1, null); if (!conditionMet) - metric.add(MetricNames.TEST_AND_SET_CONDITION_NOT_MET, 1, null); + metric.add(MetricNames.CONDITION_NOT_MET, 1, testAndSetMetricCtx); enqueue(context, "Document processed.", ErrorCode.OK, !conditionMet, reply.getTrace()); } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java index 4b49e3594f8..6ded410ff68 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/vespa/http/server/MetricNames.java @@ -18,7 +18,7 @@ public final class MetricNames { public static final String OPERATIONS_PER_SEC = PREFIX + "ops_per_sec"; public static final String LATENCY = PREFIX + "latency"; public static final String FAILED = PREFIX + "failed"; - public static final String TEST_AND_SET_CONDITION_NOT_MET = PREFIX + "test_and_set_condition_not_met"; + public static final String CONDITION_NOT_MET = PREFIX + "condition_not_met"; public static final String PARSE_ERROR = PREFIX + "parse_error"; public static final String SUCCEEDED = PREFIX + "succeeded"; public static final String PENDING = PREFIX + "pending"; diff --git a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp index 799f2c79dd4..3cd3d00645c 100644 --- a/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp +++ b/vespalib/src/tests/datastore/sharded_hash_map/sharded_hash_map_test.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/datastore/unique_store_allocator.hpp> +#include <thread> #include <vespa/log/log.h> LOG_SETUP("vespalib_datastore_shared_hash_test"); @@ -24,6 +25,18 @@ using MyHashMap = vespalib::datastore::ShardedHashMap; using GenerationHandler = vespalib::GenerationHandler; using vespalib::makeLambdaTask; +namespace { + +void consider_yield(uint32_t i) +{ + if ((i % 1000) == 0) { + // Need to yield sometimes to avoid livelock when running unit test with valgrind + std::this_thread::yield(); + } +} + +} + struct DataStoreShardedHashTest : public ::testing::Test { GenerationHandler _generationHandler; @@ -142,6 +155,7 @@ DataStoreShardedHashTest::read_work(uint32_t cnt) EXPECT_EQ(key, wrapped_entry.value()); ++found; } + consider_yield(i); } _done_read_work += i; _found_count += found; @@ -168,6 +182,7 @@ DataStoreShardedHashTest::write_work(uint32_t cnt) remove(key); } commit(); + consider_yield(i); } _done_write_work += cnt; _stop_read = 1; diff --git a/vespalib/src/tests/hwaccelrated/.gitignore b/vespalib/src/tests/hwaccelrated/.gitignore new file mode 100644 index 00000000000..42f73a39d78 --- /dev/null +++ b/vespalib/src/tests/hwaccelrated/.gitignore @@ -0,0 +1 @@ +vespalib_hwaccelrated_bench_app diff --git a/vespalib/src/tests/hwaccelrated/CMakeLists.txt b/vespalib/src/tests/hwaccelrated/CMakeLists.txt index 960ae840995..9edea9c4472 100644 --- a/vespalib/src/tests/hwaccelrated/CMakeLists.txt +++ b/vespalib/src/tests/hwaccelrated/CMakeLists.txt @@ -6,3 +6,10 @@ vespa_add_executable(vespalib_hwaccelrated_test_app TEST vespalib ) vespa_add_test(NAME vespalib_hwaccelrated_test_app COMMAND vespalib_hwaccelrated_test_app) + +vespa_add_executable(vespalib_hwaccelrated_bench_app + SOURCES + hwaccelrated_bench.cpp + DEPENDS + vespalib +) diff --git a/vespalib/src/tests/hwaccelrated/hwaccelrated_bench.cpp b/vespalib/src/tests/hwaccelrated/hwaccelrated_bench.cpp new file mode 100644 index 00000000000..9984cfca440 --- /dev/null +++ b/vespalib/src/tests/hwaccelrated/hwaccelrated_bench.cpp @@ -0,0 +1,59 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/hwaccelrated/iaccelrated.h> +#include <vespa/vespalib/hwaccelrated/generic.h> +#include <vespa/vespalib/util/time.h> +# +using namespace vespalib; + +template<typename T> +std::vector<T> createAndFill(size_t sz) { + std::vector<T> v(sz); + for (size_t i(0); i < sz; i++) { + v[i] = rand()%128; + } + return v; +} + +template<typename T> +void +benchmarkEuclideanDistance(const hwaccelrated::IAccelrated & accel, size_t sz, size_t count) { + srand(1); + std::vector<T> a = createAndFill<T>(sz); + std::vector<T> b = createAndFill<T>(sz); + steady_time start = steady_clock::now(); + double sumOfSums(0); + for (size_t j(0); j < count; j++) { + double sum = accel.squaredEuclideanDistance(&a[0], &b[0], sz); + sumOfSums += sum; + } + duration elapsed = steady_clock::now() - start; + printf("sum=%f of N=%zu and vector length=%zu took %ld\n", sumOfSums, count, sz, count_ms(elapsed)); +} + +void +benchMarkEuclidianDistance(const hwaccelrated::IAccelrated & accelrator, size_t sz, size_t count) { + printf("double : "); + benchmarkEuclideanDistance<double>(accelrator, sz, count); + printf("float : "); + benchmarkEuclideanDistance<float>(accelrator, sz, count); + printf("int8_t : "); + benchmarkEuclideanDistance<int8_t>(accelrator, sz, count); +} + +int main(int argc, char *argv[]) { + int length = 1000; + int count = 1000000; + if (argc > 1) { + length = atol(argv[1]); + } + if (argc > 2) { + count = atol(argv[2]); + } + printf("%s %d %d\n", argv[0], length, count); + printf("Squared Euclidian Distance - Generic\n"); + benchMarkEuclidianDistance(hwaccelrated::GenericAccelrator(), length, count); + printf("Squared Euclidian Distance - Optimized for this cpu\n"); + benchMarkEuclidianDistance(hwaccelrated::IAccelrated::getAccelerator(), length, count); + return 0; +} diff --git a/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp b/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp index 3d66769c15a..bbe0ff6663a 100644 --- a/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp +++ b/vespalib/src/tests/hwaccelrated/hwaccelrated_test.cpp @@ -3,6 +3,8 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/hwaccelrated/iaccelrated.h> #include <vespa/vespalib/hwaccelrated/generic.h> +#include <vespa/log/log.h> +LOG_SETUP("hwaccelrated_test"); using namespace vespalib; @@ -15,26 +17,34 @@ std::vector<T> createAndFill(size_t sz) { return v; } -template<typename T> -void verifyEuclideanDistance(const hwaccelrated::IAccelrated & accel) { - const size_t testLength(255); +template<typename T, typename P> +void verifyEuclideanDistance(const hwaccelrated::IAccelrated & accel, size_t testLength, double approxFactor) { srand(1); std::vector<T> a = createAndFill<T>(testLength); std::vector<T> b = createAndFill<T>(testLength); for (size_t j(0); j < 0x20; j++) { - T sum(0); + P sum(0); for (size_t i(j); i < testLength; i++) { - sum += (a[i] - b[i]) * (a[i] - b[i]); + P d = P(a[i]) - P(b[i]); + sum += d * d; } - T hwComputedSum(accel.squaredEuclideanDistance(&a[j], &b[j], testLength - j)); - EXPECT_EQUAL(sum, hwComputedSum); + P hwComputedSum(accel.squaredEuclideanDistance(&a[j], &b[j], testLength - j)); + EXPECT_APPROX(sum, hwComputedSum, sum*approxFactor); } } +void +verifyEuclideanDistance(const hwaccelrated::IAccelrated & accelrator, size_t testLength) { + verifyEuclideanDistance<int8_t, double>(accelrator, testLength, 0.0); + verifyEuclideanDistance<float, double>(accelrator, testLength, 0.0001); // Small deviation requiring EXPECT_APPROX + verifyEuclideanDistance<double, double>(accelrator, testLength, 0.0); +} + TEST("test euclidean distance") { hwaccelrated::GenericAccelrator genericAccelrator; - verifyEuclideanDistance<float>(genericAccelrator); - verifyEuclideanDistance<double >(genericAccelrator); + constexpr size_t TEST_LENGTH = 140000; // must be longer than 64k + TEST_DO(verifyEuclideanDistance(hwaccelrated::GenericAccelrator(), TEST_LENGTH)); + TEST_DO(verifyEuclideanDistance(hwaccelrated::IAccelrated::getAccelerator(), TEST_LENGTH)); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp index 6a6421ad016..590223ed13a 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.cpp @@ -11,6 +11,11 @@ Avx2Accelrator::populationCount(const uint64_t *a, size_t sz) const { } double +Avx2Accelrator::squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const { + return helper::squaredEuclideanDistance(a, b, sz); +} + +double Avx2Accelrator::squaredEuclideanDistance(const float * a, const float * b, size_t sz) const { return avx::euclideanDistanceSelectAlignment<float, 32>(a, b, sz); } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h index 44752dd9270..2949e81fd36 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx2.h @@ -13,6 +13,7 @@ class Avx2Accelrator : public GenericAccelrator { public: size_t populationCount(const uint64_t *a, size_t sz) const override; + double squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp index 94a6637a072..5878165bb6d 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.cpp @@ -23,6 +23,11 @@ Avx512Accelrator::populationCount(const uint64_t *a, size_t sz) const { } double +Avx512Accelrator::squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const { + return helper::squaredEuclideanDistance(a, b, sz); +} + +double Avx512Accelrator::squaredEuclideanDistance(const float * a, const float * b, size_t sz) const { return avx::euclideanDistanceSelectAlignment<float, 64>(a, b, sz); } diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h index 826cf63be70..4989f72e698 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avx512.h @@ -15,6 +15,7 @@ public: float dotProduct(const float * a, const float * b, size_t sz) const override; double dotProduct(const double * a, const double * b, size_t sz) const override; size_t populationCount(const uint64_t *a, size_t sz) const override; + double squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp index fb6ec167cf4..13946fa3398 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.cpp @@ -34,7 +34,7 @@ multiplyAdd(const T * a, const T * b, size_t sz) template <typename T, size_t UNROLL> double -euclideanDistanceT(const T * a, const T * b, size_t sz) +squaredEuclideanDistanceT(const T * a, const T * b, size_t sz) { T partial[UNROLL]; for (size_t i(0); i < UNROLL; i++) { @@ -43,11 +43,13 @@ euclideanDistanceT(const T * a, const T * b, size_t sz) size_t i(0); for (; i + UNROLL <= sz; i += UNROLL) { for (size_t j(0); j < UNROLL; j++) { - partial[j] += (a[i+j] - b[i+j]) * (a[i+j] - b[i+j]); + T d = a[i+j] - b[i+j]; + partial[j] += d * d; } } for (;i < sz; i++) { - partial[i%UNROLL] += (a[i] - b[i]) * (a[i] - b[i]); + T d = a[i] - b[i]; + partial[i%UNROLL] += d * d; } double sum(0); for (size_t j(0); j < UNROLL; j++) { @@ -156,13 +158,18 @@ GenericAccelrator::populationCount(const uint64_t *a, size_t sz) const { } double +GenericAccelrator::squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const { + return helper::squaredEuclideanDistance(a, b, sz); +} + +double GenericAccelrator::squaredEuclideanDistance(const float * a, const float * b, size_t sz) const { - return euclideanDistanceT<float, 8>(a, b, sz); + return squaredEuclideanDistanceT<float, 2>(a, b, sz); } double GenericAccelrator::squaredEuclideanDistance(const double * a, const double * b, size_t sz) const { - return euclideanDistanceT<double, 4>(a, b, sz); + return squaredEuclideanDistanceT<double, 2>(a, b, sz); } void diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h index c6b75bbcaf0..315e807da07 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/generic.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/generic.h @@ -23,6 +23,7 @@ public: void andNotBit(void * a, const void * b, size_t bytes) const override; void notBit(void * a, size_t bytes) const override; size_t populationCount(const uint64_t *a, size_t sz) const override; + double squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const override; double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const override; double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const override; void and64(size_t offset, const std::vector<std::pair<const void *, bool>> &src, void *dest) const override; diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h index afb2024b322..6eae41ead4b 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h +++ b/vespalib/src/vespa/vespalib/hwaccelrated/iaccelrated.h @@ -28,6 +28,7 @@ public: virtual void andNotBit(void * a, const void * b, size_t bytes) const = 0; virtual void notBit(void * a, size_t bytes) const = 0; virtual size_t populationCount(const uint64_t *a, size_t sz) const = 0; + virtual double squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) const = 0; virtual double squaredEuclideanDistance(const float * a, const float * b, size_t sz) const = 0; virtual double squaredEuclideanDistance(const double * a, const double * b, size_t sz) const = 0; // AND 64 bytes from multiple, optionally inverted sources diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp index 824e0e1ebd9..3b063ce6805 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/private_helpers.hpp @@ -74,5 +74,31 @@ orChunks(size_t offset, const std::vector<std::pair<const void *, bool>> & src, } } +template<typename TemporaryT=int32_t> +double squaredEuclideanDistanceT(const int8_t * a, const int8_t * b, size_t sz) __attribute__((noinline)); +template<typename TemporaryT> +double squaredEuclideanDistanceT(const int8_t * a, const int8_t * b, size_t sz) +{ + //Note that this is 3 times faster with int32_t than with int64_t and 16x faster than float + TemporaryT sum = 0; + for (size_t i(0); i < sz; i++) { + int16_t d = int16_t(a[i]) - int16_t(b[i]); + sum += d * d; + } + return sum; +} + +inline double +squaredEuclideanDistance(const int8_t * a, const int8_t * b, size_t sz) { + constexpr size_t LOOP_COUNT = 0x10000; + double sum(0); + size_t i=0; + for (; i + LOOP_COUNT <= sz; i += LOOP_COUNT) { + sum += squaredEuclideanDistanceT<int32_t>(a + i, b + i, LOOP_COUNT); + } + sum += squaredEuclideanDistanceT<int32_t>(a + i, b + i, sz - i); + return sum; +} + } } diff --git a/vespalog/src/logger/runserver.cpp b/vespalog/src/logger/runserver.cpp index c74806a8b5b..68950cef4b2 100644 --- a/vespalog/src/logger/runserver.cpp +++ b/vespalog/src/logger/runserver.cpp @@ -5,7 +5,6 @@ #include <fcntl.h> #include <cerrno> #include <unistd.h> -#include <csignal> #include <sys/select.h> #include <sys/types.h> @@ -54,13 +53,13 @@ bool whole_seconds(int cnt, int secs) { class PidFile { private: - char *_pidfile; + std::string _pidfile; int _fd; PidFile(const PidFile&); PidFile& operator= (const PidFile&); public: - PidFile(const char *pidfile) : _pidfile(strdup(pidfile)), _fd(-1) {} - ~PidFile() { free(_pidfile); if (_fd >= 0) close(_fd); } + PidFile(const char *pidfile) : _pidfile(pidfile), _fd(-1) {} + ~PidFile() { if (_fd >= 0) close(_fd); } int readPid(); void writePid(); bool writeOpen(); @@ -72,7 +71,7 @@ public: void PidFile::cleanUp() { - if (!anotherRunning()) remove(_pidfile); + if (!anotherRunning()) remove(_pidfile.c_str()); if (_fd >= 0) close(_fd); _fd = -1; } @@ -82,14 +81,14 @@ PidFile::writeOpen() { if (_fd >= 0) close(_fd); int flags = O_CREAT | O_WRONLY | O_NONBLOCK; - _fd = open(_pidfile, flags, 0644); + _fd = open(_pidfile.c_str(), flags, 0644); if (_fd < 0) { - fprintf(stderr, "could not create pidfile %s: %s\n", _pidfile, + fprintf(stderr, "could not create pidfile %s: %s\n", _pidfile.c_str(), strerror(errno)); return false; } if (flock(_fd, LOCK_EX | LOCK_NB) != 0) { - fprintf(stderr, "could not lock pidfile %s: %s\n", _pidfile, + fprintf(stderr, "could not lock pidfile %s: %s\n", _pidfile.c_str(), strerror(errno)); close(_fd); _fd = -1; @@ -106,7 +105,7 @@ PidFile::writePid() int didtruncate = ftruncate(_fd, (off_t)0); if (didtruncate != 0) { fprintf(stderr, "could not truncate pid file %s: %s\n", - _pidfile, strerror(errno)); + _pidfile.c_str(), strerror(errno)); std::_Exit(1); } char buf[100]; @@ -115,16 +114,16 @@ PidFile::writePid() ssize_t didw = write(_fd, buf, l); if (didw != l) { fprintf(stderr, "could not write pid to %s: %s\n", - _pidfile, strerror(errno)); + _pidfile.c_str(), strerror(errno)); std::_Exit(1); } - LOG(debug, "wrote '%s' to %s (fd %d)", buf, _pidfile, _fd); + LOG(debug, "wrote '%s' to %s (fd %d)", buf, _pidfile.c_str(), _fd); } int PidFile::readPid() { - FILE *pf = fopen(_pidfile, "r"); + FILE *pf = fopen(_pidfile.c_str(), "r"); if (pf == NULL) return 0; char buf[100]; strcpy(buf, "0"); @@ -151,7 +150,7 @@ bool PidFile::canStealLock() { int flags = O_WRONLY | O_NONBLOCK; - int desc = open(_pidfile, flags, 0644); + int desc = open(_pidfile.c_str(), flags, 0644); if (desc < 0) { return false; } diff --git a/vespalog/src/vespa/log/control-file.cpp b/vespalog/src/vespa/log/control-file.cpp index 77ad1d0ec73..2096dd1531c 100644 --- a/vespalog/src/vespa/log/control-file.cpp +++ b/vespalog/src/vespa/log/control-file.cpp @@ -5,7 +5,6 @@ #include <ctype.h> #include <cstdio> #include <sys/mman.h> -#include <sys/stat.h> #include <errno.h> #include <unistd.h> #include <memory> @@ -28,7 +27,7 @@ ControlFile::ControlFile(const char *file, Mode mode) : (O_RDWR | O_CREAT))), _fileSize(0), _mode(mode), - _fileName(strdup(file)), + _fileName(file), _prefix(0), _mapBase(0), _mappedSize(0), @@ -43,7 +42,6 @@ ControlFile::ControlFile(const char *file, Mode mode) ControlFile::~ControlFile() { freeMapping(); - free(_fileName); } void @@ -168,7 +166,7 @@ ControlFile::extendMapping() if (fileLen == -1) { _fileBacking.unlock(); - LOG(error, "Cannot get file size of '%s': %s", _fileName, + LOG(error, "Cannot get file size of '%s': %s", _fileName.c_str(), strerror(errno)); return false; } @@ -273,14 +271,14 @@ ControlFile::getLevels(const char *name) strcat(appendedString, "\n"); int len = strlen(appendedString); - int fd = open(_fileName, O_WRONLY | O_APPEND); + int fd = open(_fileName.c_str(), O_WRONLY | O_APPEND); int wlen = write(fd, appendedString, len); oldFileLength = lseek(fd, (off_t)0, SEEK_CUR) - wlen; close(fd); if (wlen != len) { _fileBacking.unlock(); LOG(error, "Writing to control file '%s' fails (%d/%d bytes): %s", - _fileName, wlen, len, strerror(errno)); + _fileName.c_str(), wlen, len, strerror(errno)); return reinterpret_cast<unsigned int *>(inheritLevels); } else { _fileSize = _fileBacking.size(); @@ -290,7 +288,7 @@ ControlFile::getLevels(const char *name) if (!extendMapping()) { _fileBacking.unlock(); // just for sure LOG(error, "Failed to extend mapping of '%s', losing runtime " - "configurability of component '%s'", _fileName, name); + "configurability of component '%s'", _fileName.c_str(), name); return defaultLevels(); } } diff --git a/vespalog/src/vespa/log/control-file.h b/vespalog/src/vespa/log/control-file.h index 6f302c7a97c..69e725dc465 100644 --- a/vespalog/src/vespa/log/control-file.h +++ b/vespalog/src/vespa/log/control-file.h @@ -19,7 +19,7 @@ private: Lock _fileBacking; int _fileSize; enum Mode _mode; - char *_fileName; + std::string _fileName; void ensureHeader(); bool hasPrefix() { return (_prefix != NULL && _prefix[0] != '\0' && diff --git a/vespalog/src/vespa/log/internal.h b/vespalog/src/vespa/log/internal.h index c9081b72ce9..4411d9fa6e6 100644 --- a/vespalog/src/vespa/log/internal.h +++ b/vespalog/src/vespa/log/internal.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <cstring> +#include <string> #include <cstdlib> #if !__GNUC__ && !defined(__attribute__) @@ -15,15 +15,14 @@ void throwInvalid(const char *fmt, ...) class InvalidLogException { private: - char *_what; - InvalidLogException& operator = (const InvalidLogException&); + std::string _what; public: - InvalidLogException(const InvalidLogException &x) : - _what(strdup(x._what)) {} - InvalidLogException(const char *s) : _what(strdup(s)) {} - ~InvalidLogException() { free(_what); } - const char *what() const { return _what; } + InvalidLogException& operator = (const InvalidLogException&) = delete; + InvalidLogException(const InvalidLogException &x) = default; + InvalidLogException(const char *s) : _what(s) {} + ~InvalidLogException() = default; + const char *what() const { return _what.c_str(); } }; } // end namespace ns_log |